From a48b9472e81002515e271c9cbcdf1dd66d7247dc Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 20 Oct 2026 15:44:58 -0700 Subject: [PATCH 1/3] Add skeleton profiles for naming device segments A device that describes rigid segments needs to say which skeleton those segments are posed by at run time. SkeletonBinding names each segment in a registered skeleton profile, optionally pinned to an asset identified by its RFC 6920 SHA-256 name. Three profiles are registered: openxr.hand (HandJoint names, needs a side), openxr.body.bd (BodyJoint names) and urdf.link (link names of the URDF in the asset). An unregistered profile id reads as a generic binding. validate_skeleton_binding() checks a binding against its device and profile in C++ and Python; check_skeleton_asset() checks the asset bytes and, for urdf.link, that every name is a link of the URDF. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lotus Li --- docs/source/index.rst | 1 + docs/source/references/skeleton_binding.rst | 77 +++++++ .../cpp/inc/schema/skeleton_binding.hpp | 208 ++++++++++++++++++ src/core/schema/fbs/skeleton_binding.fbs | 47 ++++ src/core/schema/golden/skeleton_binding.bfbs | Bin 0 -> 1104 bytes src/core/schema/python/CMakeLists.txt | 1 + src/core/schema/python/schema_module.cpp | 4 + .../schema/python/skeleton_binding_bindings.h | 160 ++++++++++++++ src/python/isaaccapture/schema/__init__.py | 27 +++ .../isaaccapture/schema/skeleton_assets.py | 47 ++++ tests/cpp/core/schema/CMakeLists.txt | 1 + .../cpp/core/schema/test_skeleton_binding.cpp | 114 ++++++++++ .../core/schema/test_skeleton_binding.py | 88 ++++++++ 13 files changed, 775 insertions(+) create mode 100644 docs/source/references/skeleton_binding.rst create mode 100644 src/core/schema/cpp/inc/schema/skeleton_binding.hpp create mode 100644 src/core/schema/fbs/skeleton_binding.fbs create mode 100644 src/core/schema/golden/skeleton_binding.bfbs create mode 100644 src/core/schema/python/skeleton_binding_bindings.h create mode 100644 src/python/isaaccapture/schema/skeleton_assets.py create mode 100644 tests/cpp/core/schema/test_skeleton_binding.cpp create mode 100644 tests/python/core/schema/test_skeleton_binding.py diff --git a/docs/source/index.rst b/docs/source/index.rst index 9fe57ba79f..3b66361a50 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -92,6 +92,7 @@ Table of Contents references/requirements references/build references/generated_trackers + references/skeleton_binding references/retargeting/index references/device_provider_monitoring references/camera_streaming diff --git a/docs/source/references/skeleton_binding.rst b/docs/source/references/skeleton_binding.rst new file mode 100644 index 0000000000..8128e29420 --- /dev/null +++ b/docs/source/references/skeleton_binding.rst @@ -0,0 +1,77 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Skeleton Binding +================ + +A device that describes rigid segments — the links a tactile glove's taxels are skinned to, +for example — carries a ``SkeletonBinding`` that names each segment in a skeleton whose pose +some other source provides at run time. The schema is +:code-file:`src/core/schema/fbs/skeleton_binding.fbs`; the registry and the checks below are +:code-file:`src/core/schema/cpp/inc/schema/skeleton_binding.hpp`. + +``segment_names[i]`` names segment ``i`` of the device. ``profile`` says which skeleton the names +come from; without it the binding is *generic* and the names are labels only. ``asset`` identifies +an external description, such as a URDF, that a profile names segments in. + +Registered profiles +------------------- + +.. list-table:: + :header-rows: 1 + + * - ``id`` + - ``version`` + - Segment names + - Requires + * - ``openxr.hand`` + - 1 + - ``HandJoint`` names (``PALM`` … ``LITTLE_TIP``), the XR_EXT_hand_tracking joints + - ``side`` LEFT or RIGHT + * - ``openxr.body.bd`` + - 1 + - ``BodyJoint`` names (``PELVIS`` … ``RIGHT_HAND``), the XR_BD_body_tracking joints + - — + * - ``urdf.link`` + - 1 + - ```` values of the URDF in ``asset`` + - ``asset`` + +A profile's names never change within a version; a change takes a new ``version``. Ids outside +this table are valid and read as generic bindings. Vendor profiles take a reverse-DNS prefix, +such as ``com.example.glove``. + +Rules +----- + +``validate_skeleton_binding(binding, segment_count, side)`` returns an empty string for a valid +binding, otherwise the first rule it breaks: + +#. ``segment_names`` has exactly ``segment_count`` entries, each non-empty and unique. +#. ``asset.digest``, when present, is an RFC 6920 SHA-256 name: ``ni:///sha-256;`` and the + 43-character unpadded base64url digest of the asset's bytes. +#. An ``asset`` needs a ``profile``. +#. ``profile.id`` is two or more dot-separated tokens of ``[a-z0-9_]``, each starting with a + letter; ``profile.version`` is at least 1. +#. For a registered profile: ``asset`` is present exactly when the profile requires one, ``side`` + is LEFT or RIGHT when the profile requires it, and every name is one of the profile's names. + +Checking names against an asset needs its bytes, so it is a separate step in Python: +``check_skeleton_asset(binding, data)`` verifies the digest and, for ``urdf.link``, that every name +is a link of the URDF. ``asset_digest(data)`` computes the digest to store. + +.. code-block:: python + + from isaaccapture.schema import ( + AssetRef, SkeletonBinding, SkeletonProfileRef, asset_digest, check_skeleton_asset, + validate_skeleton_binding, + ) + + urdf = open("glove.urdf", "rb").read() + binding = SkeletonBinding( + ["base_link", "index_tip"], + profile=SkeletonProfileRef("urdf.link", 1), + asset=AssetRef(asset_digest(urdf), "package://glove/glove.urdf", "application/xml"), + ) + assert validate_skeleton_binding(binding, segment_count=2) == "" + assert check_skeleton_asset(binding, urdf) == "" diff --git a/src/core/schema/cpp/inc/schema/skeleton_binding.hpp b/src/core/schema/cpp/inc/schema/skeleton_binding.hpp new file mode 100644 index 0000000000..cdbf10238c --- /dev/null +++ b/src/core/schema/cpp/inc/schema/skeleton_binding.hpp @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The registered skeleton profiles, and the checks every SkeletonBinding must pass. +// docs/source/references/skeleton_binding.rst is the registry's specification. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace core +{ + +//! XR_EXT_hand_tracking joints, named as HandJoint names them. +inline constexpr std::string_view SKELETON_PROFILE_OPENXR_HAND = "openxr.hand"; +//! XR_BD_body_tracking joints, named as BodyJoint names them. +inline constexpr std::string_view SKELETON_PROFILE_OPENXR_BODY_BD = "openxr.body.bd"; +//! Link frames of the URDF the binding's asset holds, named as the URDF names them. +inline constexpr std::string_view SKELETON_PROFILE_URDF_LINK = "urdf.link"; + +//! A registered skeleton profile. +struct SkeletonProfile +{ + std::string_view id; + uint32_t version; + //! The segment names the profile defines; empty when they come from the asset. + std::vector segment_names; + //! Whether a binding needs a device side of LEFT or RIGHT. + bool requires_side; + //! Whether a binding names its segments in an asset. + bool requires_asset; +}; + +//! Every registered profile. +inline const std::vector& skeleton_profiles() +{ + static const std::vector profiles = [] + { + std::vector hand_joints; + for (const HandJoint joint : EnumValuesHandJoint()) + { + if (joint != HandJoint_NUM_JOINTS) + { + hand_joints.emplace_back(EnumNameHandJoint(joint)); + } + } + std::vector body_joints; + for (const BodyJoint joint : EnumValuesBodyJoint()) + { + if (joint != BodyJoint_NUM_JOINTS) + { + body_joints.emplace_back(EnumNameBodyJoint(joint)); + } + } + return std::vector{ + { SKELETON_PROFILE_OPENXR_HAND, 1, std::move(hand_joints), true, false }, + { SKELETON_PROFILE_OPENXR_BODY_BD, 1, std::move(body_joints), false, false }, + { SKELETON_PROFILE_URDF_LINK, 1, {}, false, true }, + }; + }(); + return profiles; +} + +//! The registered profile `id` at `version`, or null when it is not registered. +inline const SkeletonProfile* find_skeleton_profile(std::string_view id, uint32_t version) +{ + const auto& profiles = skeleton_profiles(); + const auto it = + std::find_if(profiles.begin(), profiles.end(), + [&](const SkeletonProfile& profile) { return profile.id == id && profile.version == version; }); + return it != profiles.end() ? &*it : nullptr; +} + +//! Whether `id` is a registry id: two or more dot-separated tokens, each a lowercase letter +//! followed by lowercase letters, digits and '_'. +inline bool is_valid_registry_id(std::string_view id) +{ + size_t tokens = 0; + size_t start = 0; + while (start <= id.size()) + { + const size_t end = std::min(id.find('.', start), id.size()); + const std::string_view token = id.substr(start, end - start); + if (token.empty() || token[0] < 'a' || token[0] > 'z') + { + return false; + } + for (const char c : token) + { + if (!((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_')) + { + return false; + } + } + ++tokens; + start = end + 1; + } + return tokens >= 2; +} + +//! Whether `digest` is an RFC 6920 SHA-256 name: "ni:///sha-256;" and 43 base64url characters. +inline bool is_valid_asset_digest(std::string_view digest) +{ + constexpr std::string_view prefix = "ni:///sha-256;"; + if (digest.size() != prefix.size() + 43 || digest.substr(0, prefix.size()) != prefix) + { + return false; + } + return std::all_of( + digest.begin() + prefix.size(), digest.end(), + [](char c) + { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_'; }); +} + +/*! + * @brief Checks `binding` against the device it binds and against its profile. + * @param segment_count Number of segments of the bound device. + * @param side Side of the bound device. + * @return Empty when valid, otherwise what is wrong. A profile that is not registered is valid + * and reads as a generic binding. + */ +inline std::string validate_skeleton_binding(const SkeletonBinding& binding, size_t segment_count, BodySide side) +{ + const auto* names = binding.segment_names(); + const size_t name_count = names != nullptr ? names->size() : 0; + if (name_count != segment_count) + { + return "segment_names has " + std::to_string(name_count) + " names for " + std::to_string(segment_count) + + " segments"; + } + std::set seen; + for (size_t i = 0; i < name_count; ++i) + { + const auto* name = names->Get(static_cast(i)); + if (name == nullptr || name->size() == 0) + { + return "segment_names[" + std::to_string(i) + "] is empty"; + } + if (!seen.insert(std::string_view(name->c_str(), name->size())).second) + { + return "segment_names repeats '" + name->str() + "'"; + } + } + + const auto* asset = binding.asset(); + if (asset != nullptr && (asset->digest() == nullptr || + !is_valid_asset_digest(std::string_view(asset->digest()->c_str(), asset->digest()->size())))) + { + return "asset.digest is not an ni:///sha-256 name"; + } + + const auto* profile = binding.profile(); + if (profile == nullptr) + { + return asset != nullptr ? "an asset needs a profile that names segments in it" : ""; + } + const std::string id = profile->id() != nullptr ? profile->id()->str() : std::string(); + if (!is_valid_registry_id(id)) + { + return "profile.id '" + id + "' is not a registry id"; + } + if (profile->version() == 0) + { + return "profile.version must be at least 1"; + } + + const SkeletonProfile* known = find_skeleton_profile(id, profile->version()); + if (known == nullptr) + { + return ""; + } + if (known->requires_asset && asset == nullptr) + { + return "profile " + id + " needs an asset"; + } + if (!known->requires_asset && asset != nullptr) + { + return "profile " + id + " takes no asset"; + } + if (known->requires_side && side != BodySide_LEFT && side != BodySide_RIGHT) + { + return "profile " + id + " needs a side of LEFT or RIGHT"; + } + if (!known->segment_names.empty()) + { + for (const auto* name : *names) + { + if (std::find(known->segment_names.begin(), known->segment_names.end(), name->str()) == + known->segment_names.end()) + { + return "'" + name->str() + "' is not a segment of profile " + id; + } + } + } + return ""; +} + +} // namespace core diff --git a/src/core/schema/fbs/skeleton_binding.fbs b/src/core/schema/fbs/skeleton_binding.fbs new file mode 100644 index 0000000000..a1567fac49 --- /dev/null +++ b/src/core/schema/fbs/skeleton_binding.fbs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// How a device's rigid segments are named in a skeleton that poses them at run time. +// The registered profiles and their rules are in docs/source/references/skeleton_binding.rst. + +namespace core; + +// Side of the body a device is worn on or mounted to. +enum BodySide : ubyte { + UNSPECIFIED = 0, + LEFT = 1, + RIGHT = 2, +} + +// A registered skeleton profile: the skeleton definition segment names refer to. +table SkeletonProfileRef { + // Registered id, e.g. "openxr.hand"; vendor profiles start with a reverse-DNS prefix. + id: string (id: 0); + + // Major version of the profile definition. + version: uint (id: 1); +} + +// An external description a profile needs, such as a URDF, identified by its exact bytes. +table AssetRef { + // RFC 6920 name of the bytes: "ni:///sha-256;" and the unpadded base64url SHA-256 digest. + digest: string (id: 0); + + // Where the bytes can be fetched (RFC 3986 URI); the digest decides whether they are the right ones. + uri: string (id: 1); + + // Media type of the bytes, e.g. "application/xml". + media_type: string (id: 2); +} + +// Names the rigid segments of a device in a skeleton. +table SkeletonBinding { + // Absent for a generic binding, whose names are labels only. + profile: SkeletonProfileRef (id: 0); + + // The asset the profile names segments in; required by profiles defined over an asset. + asset: AssetRef (id: 1); + + // segment_names[i] names segment i of the device in the profile's namespace. + segment_names: [string] (id: 2); +} diff --git a/src/core/schema/golden/skeleton_binding.bfbs b/src/core/schema/golden/skeleton_binding.bfbs new file mode 100644 index 0000000000000000000000000000000000000000..c4d68fe6d69882059daae2fb7c2b073831f0e5a9 GIT binary patch literal 1104 zcmZvav2GJV5QfLWaV)UODY77i3l}L;LLTU#gWbL2aa@)jv5 z@(?@%4?#gmK}o>_P+`7*H)lB#BmK9#bF=f$KeJ6UYj@j&hBd8hRjXNqrvY~T_n6&2 zXafDIf@k1=R>FA>9)RNwvlloT9OSP7WfzBXMEJ|B!FJ%%pAaXe9rkBHT)W^W`68mW z*oU~t*Td1L5r-h|8}JtBULCxjFWg~!9$ycV*m3^8TrQ;C;mY-J3zb^Rn0Nbnmnz@Yg0s2@DD~27V~j3b)KumxzPKYz4U$tnpX@~^E?&dg&X733|PEQajM2IL3>>s55dgE$!M5g zU-1B%bWN31{U3g=Y|Ha-^m>b2f7WuTo{x*fPQ;1MBBmK?@O%|w;$-5oT=TjAvcoGj zJ0WIYL_qaE0Y^vK-4KVDVTy<2ONO9ck;)FhQ*dwI{m?vhB52aPtK9Xlva0WEsIOY| o{F7T`$vj1``k;=h>UG-;y+Gq0Zt2w9)x3fgZ^yi5)tE4#e=s|S*Z=?k literal 0 HcmV?d00001 diff --git a/src/core/schema/python/CMakeLists.txt b/src/core/schema/python/CMakeLists.txt index b34cac9c90..795ac84ff8 100644 --- a/src/core/schema/python/CMakeLists.txt +++ b/src/core/schema/python/CMakeLists.txt @@ -20,6 +20,7 @@ pybind11_add_module(schema_py schema_array_views.h schema_serialized.h se3_tracker_bindings.h + skeleton_binding_bindings.h soma_common_bindings.h soma_body_joint_poses_bindings.h soma_body_joint_rotations_bindings.h diff --git a/src/core/schema/python/schema_module.cpp b/src/core/schema/python/schema_module.cpp index 7d7a950be7..065d7cd3df 100644 --- a/src/core/schema/python/schema_module.cpp +++ b/src/core/schema/python/schema_module.cpp @@ -22,6 +22,7 @@ #include "plugin_device_status_bindings.h" #include "pose_bindings.h" #include "se3_tracker_bindings.h" +#include "skeleton_binding_bindings.h" #include "soma_body_joint_poses_bindings.h" #include "soma_body_joint_rotations_bindings.h" #include "soma_common_bindings.h" @@ -99,6 +100,9 @@ PYBIND11_MODULE(_schema, m) // Bind full body types (BodyJointPose, BodyJoints structs, FullBodyPose table). core::bind_full_body(m); + // Bind skeleton binding types (BodySide enum, SkeletonBinding table) and the profile registry. + core::bind_skeleton_binding(m); + // Bind shared SOMA primitives, then the body and hand pose contracts. core::bind_soma_common(m); core::bind_soma_body_joint_rotations(m); diff --git a/src/core/schema/python/skeleton_binding_bindings.h b/src/core/schema/python/skeleton_binding_bindings.h new file mode 100644 index 0000000000..614cf3b8ab --- /dev/null +++ b/src/core/schema/python/skeleton_binding_bindings.h @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Python bindings for the SkeletonBinding FlatBuffer schema and the skeleton profile registry. +// Types: BodySide (enum), SkeletonProfileRef, AssetRef and SkeletonBinding (tables, exposed as +// encoded views), and SkeletonProfile (a registry entry). + +#pragma once + +#include "schema_serialized.h" + +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +namespace core +{ + +inline void bind_skeleton_binding(py::module& m) +{ + py::enum_(m, "BodySide") + .value("UNSPECIFIED", BodySide_UNSPECIFIED) + .value("LEFT", BodySide_LEFT) + .value("RIGHT", BodySide_RIGHT); + + m.attr("SKELETON_PROFILE_OPENXR_HAND") = std::string(SKELETON_PROFILE_OPENXR_HAND); + m.attr("SKELETON_PROFILE_OPENXR_BODY_BD") = std::string(SKELETON_PROFILE_OPENXR_BODY_BD); + m.attr("SKELETON_PROFILE_URDF_LINK") = std::string(SKELETON_PROFILE_URDF_LINK); + + serialized_class(m, "SkeletonProfileRef", "Encoded reference to a skeleton profile.") + .def(py::init( + [](const std::string& id, uint32_t version) + { + SkeletonProfileRefT native; + native.id = id; + native.version = version; + return pack(native); + }), + py::arg("id"), py::arg("version"), "Encode a reference to profile `id` at major `version`.") + .def_property_readonly("id", string_field(&SkeletonProfileRef::id)) + .def_property_readonly("version", field(&SkeletonProfileRef::version)) + .def("__repr__", + [](const Serialized& self) + { + const auto* id = self->id(); + return "SkeletonProfileRef(id=" + (id != nullptr ? id->str() : std::string{}) + + ", version=" + std::to_string(self->version()) + ")"; + }); + + serialized_class(m, "AssetRef", "Encoded reference to an asset, identified by its digest.") + .def(py::init( + [](const std::string& digest, const std::string& uri, const std::string& media_type) + { + AssetRefT native; + native.digest = digest; + native.uri = uri; + native.media_type = media_type; + return pack(native); + }), + py::arg("digest"), py::arg("uri") = std::string{}, py::arg("media_type") = std::string{}, + "Encode an asset reference. `asset_digest()` computes `digest` from the asset's bytes.") + .def_property_readonly("digest", string_field(&AssetRef::digest)) + .def_property_readonly("uri", string_field(&AssetRef::uri)) + .def_property_readonly("media_type", string_field(&AssetRef::media_type)) + .def("__repr__", + [](const Serialized& self) + { + const auto* digest = self->digest(); + return "AssetRef(digest=" + (digest != nullptr ? digest->str() : std::string{}) + ")"; + }); + + serialized_class(m, "SkeletonBinding", "Encoded names of a device's segments in a skeleton.") + .def(py::init( + [](const std::vector& segment_names, const Serialized* profile, + const Serialized* asset) + { + SkeletonBindingT native; + native.segment_names = segment_names; + if (profile != nullptr && *profile) + { + native.profile = std::make_shared(); + (*profile)->UnPackTo(native.profile.get()); + } + if (asset != nullptr && *asset) + { + native.asset = std::make_shared(); + (*asset)->UnPackTo(native.asset.get()); + } + return pack(native); + }), + py::arg("segment_names"), py::arg("profile") = nullptr, py::arg("asset") = nullptr, + "Encode a binding. Without `profile` it is generic: the names are labels only.") + .def_property_readonly("segment_names", + [](const Serialized& self) + { + std::vector names; + if (const auto* encoded = self->segment_names()) + { + for (const auto* name : *encoded) + { + names.push_back(name->str()); + } + } + return names; + }) + .def_property_readonly( + "profile", + [](const Serialized& self) -> py::object + { + const auto* profile = self->profile(); + return profile != nullptr ? py::cast(self.narrow(profile)) : py::none(); + }, + "The profile the names refer to, or None for a generic binding.") + .def_property_readonly( + "asset", + [](const Serialized& self) -> py::object + { + const auto* asset = self->asset(); + return asset != nullptr ? py::cast(self.narrow(asset)) : py::none(); + }, + "The asset the profile names segments in, or None.") + .def("__repr__", + [](const Serialized& self) + { + const auto* profile = self->profile(); + const auto* names = self->segment_names(); + return "SkeletonBinding(profile=" + + (profile != nullptr && profile->id() != nullptr ? profile->id()->str() : std::string("None")) + + ", segments=" + std::to_string(names != nullptr ? names->size() : 0) + ")"; + }); + + py::class_(m, "SkeletonProfile", "A registered skeleton profile.") + .def_property_readonly("id", [](const SkeletonProfile& self) { return std::string(self.id); }) + .def_readonly("version", &SkeletonProfile::version) + .def_readonly("segment_names", &SkeletonProfile::segment_names, + "The names the profile defines; empty when they come from the asset.") + .def_readonly("requires_side", &SkeletonProfile::requires_side) + .def_readonly("requires_asset", &SkeletonProfile::requires_asset) + .def("__repr__", [](const SkeletonProfile& self) + { return "SkeletonProfile(id=" + std::string(self.id) + ", version=" + std::to_string(self.version) + ")"; }); + + m.def("skeleton_profiles", &skeleton_profiles, py::return_value_policy::reference, "Every registered profile."); + m.def("find_skeleton_profile", &find_skeleton_profile, py::arg("id"), py::arg("version"), + py::return_value_policy::reference, "The registered profile `id` at `version`, or None."); + m.def( + "validate_skeleton_binding", + [](const Serialized& binding, size_t segment_count, BodySide side) + { return validate_skeleton_binding(*binding, segment_count, side); }, + py::arg("binding"), py::arg("segment_count"), py::arg("side") = BodySide_UNSPECIFIED, + "Check `binding` against a device of `segment_count` segments on `side`. Returns \"\" when valid, " + "otherwise what is wrong. A profile that is not registered reads as a generic binding."); +} + +} // namespace core diff --git a/src/python/isaaccapture/schema/__init__.py b/src/python/isaaccapture/schema/__init__.py index 9b484fafb1..4823d57e72 100644 --- a/src/python/isaaccapture/schema/__init__.py +++ b/src/python/isaaccapture/schema/__init__.py @@ -95,6 +95,18 @@ BodyJoints, FullBodyPose, FullBodyPoseRecord, + # Skeleton binding types and the skeleton profile registry. + BodySide, + SkeletonProfileRef, + AssetRef, + SkeletonBinding, + SkeletonProfile, + SKELETON_PROFILE_OPENXR_HAND, + SKELETON_PROFILE_OPENXR_BODY_BD, + SKELETON_PROFILE_URDF_LINK, + skeleton_profiles, + find_skeleton_profile, + validate_skeleton_binding, # Shared SOMA types. SomaHandSide, # SOMA body joint-rotation types. @@ -116,6 +128,7 @@ SomaHandJointPoses, SomaHandJointPosesRecord, ) +from .skeleton_assets import asset_digest, check_skeleton_asset # Deprecated aliases, resolved lazily via __getattr__ so accessing them emits a # DeprecationWarning. Omitted from __all__. @@ -231,6 +244,20 @@ def __getattr__(name: str): "BodyJoints", "FullBodyPose", "FullBodyPoseRecord", + # Skeleton binding types and the skeleton profile registry. + "BodySide", + "SkeletonProfileRef", + "AssetRef", + "SkeletonBinding", + "SkeletonProfile", + "SKELETON_PROFILE_OPENXR_HAND", + "SKELETON_PROFILE_OPENXR_BODY_BD", + "SKELETON_PROFILE_URDF_LINK", + "skeleton_profiles", + "find_skeleton_profile", + "validate_skeleton_binding", + "asset_digest", + "check_skeleton_asset", # Shared SOMA types. "SomaHandSide", # SOMA body joint-rotation types. diff --git a/src/python/isaaccapture/schema/skeleton_assets.py b/src/python/isaaccapture/schema/skeleton_assets.py new file mode 100644 index 0000000000..eae929093e --- /dev/null +++ b/src/python/isaaccapture/schema/skeleton_assets.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Checks of a SkeletonBinding against the bytes of the asset it names.""" + +import base64 +import hashlib +import xml.etree.ElementTree as ET + +from ._schema import SKELETON_PROFILE_URDF_LINK, SkeletonBinding + +_NI_SHA256 = "ni:///sha-256;" +_URDF_LINK_V1 = (SKELETON_PROFILE_URDF_LINK, 1) + + +def asset_digest(data: bytes) -> str: + """RFC 6920 name of ``data``: ``ni:///sha-256;`` and its unpadded base64url SHA-256.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=") + return _NI_SHA256 + digest.decode("ascii") + + +def check_skeleton_asset(binding: SkeletonBinding, data: bytes) -> str: + """Check that ``data`` is the asset ``binding`` names, and that its segments are in it. + + Segment names are checked for ``urdf.link`` v1, where each must be a ```` of the + URDF. Returns "" when valid, otherwise what is wrong. + """ + asset = binding.asset + if asset is None: + return "binding has no asset" + if asset_digest(data) != asset.digest: + return "asset bytes do not match asset.digest" + + profile = binding.profile + if profile is None or (profile.id, profile.version) != _URDF_LINK_V1: + return "" + try: + robot = ET.fromstring(data) + except ET.ParseError as error: + return f"asset is not XML: {error}" + if robot.tag != "robot": + return f"asset root is <{robot.tag}>, not " + links = {link.get("name") for link in robot.findall("link")} + for name in binding.segment_names: + if name not in links: + return f"'{name}' is not a link of the URDF" + return "" diff --git a/tests/cpp/core/schema/CMakeLists.txt b/tests/cpp/core/schema/CMakeLists.txt index 3fa2548b78..5bdee9fc7b 100644 --- a/tests/cpp/core/schema/CMakeLists.txt +++ b/tests/cpp/core/schema/CMakeLists.txt @@ -11,6 +11,7 @@ add_executable(schema_tests test_full_body.cpp test_controller.cpp test_pedals.cpp + test_skeleton_binding.cpp test_steering_wheel.cpp test_spacemouse.cpp test_gamepad.cpp diff --git a/tests/cpp/core/schema/test_skeleton_binding.cpp b/tests/cpp/core/schema/test_skeleton_binding.cpp new file mode 100644 index 0000000000..3991e153a8 --- /dev/null +++ b/tests/cpp/core/schema/test_skeleton_binding.cpp @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit tests for the skeleton profile registry and validate_skeleton_binding(). + +#include +#include +#include + +#include +#include +#include + +namespace +{ + +const std::string kDigest = "ni:///sha-256;f4OxZX_x_FO5LcGBSKHWXfwtSx-j1ncoSt3SABJtkGk"; + +core::SkeletonBindingT binding(std::vector names, const std::string& profile_id = "", uint32_t version = 1) +{ + core::SkeletonBindingT native; + native.segment_names = std::move(names); + if (!profile_id.empty()) + { + native.profile = std::make_shared(); + native.profile->id = profile_id; + native.profile->version = version; + } + return native; +} + +core::SkeletonBindingT with_asset(core::SkeletonBindingT native, const std::string& digest = kDigest) +{ + native.asset = std::make_shared(); + native.asset->digest = digest; + native.asset->media_type = "application/xml"; + return native; +} + +std::string validate(const core::SkeletonBindingT& native, size_t segment_count, core::BodySide side) +{ + return core::validate_skeleton_binding(*core::pack(native), segment_count, side); +} + +} // namespace + +TEST_CASE("Registered profiles take their names from the joint enums", "[unit][skeleton_binding]") +{ + const auto* hand = core::find_skeleton_profile(core::SKELETON_PROFILE_OPENXR_HAND, 1); + REQUIRE(hand != nullptr); + REQUIRE(hand->segment_names.size() == core::HandJoint_NUM_JOINTS); + CHECK(hand->segment_names.front() == "PALM"); + CHECK(hand->segment_names.back() == "LITTLE_TIP"); + CHECK(hand->requires_side); + CHECK_FALSE(hand->requires_asset); + + const auto* body = core::find_skeleton_profile(core::SKELETON_PROFILE_OPENXR_BODY_BD, 1); + REQUIRE(body != nullptr); + CHECK(body->segment_names.size() == core::BodyJoint_NUM_JOINTS); + + const auto* urdf = core::find_skeleton_profile(core::SKELETON_PROFILE_URDF_LINK, 1); + REQUIRE(urdf != nullptr); + CHECK(urdf->segment_names.empty()); + CHECK(urdf->requires_asset); + + CHECK(core::find_skeleton_profile(core::SKELETON_PROFILE_OPENXR_HAND, 2) == nullptr); + CHECK(core::skeleton_profiles().size() == 3); +} + +TEST_CASE("Registry ids are dotted lowercase tokens", "[unit][skeleton_binding]") +{ + CHECK(core::is_valid_registry_id("openxr.hand")); + CHECK(core::is_valid_registry_id("com.example.glove_v2")); + CHECK_FALSE(core::is_valid_registry_id("openxr")); + CHECK_FALSE(core::is_valid_registry_id("OpenXR.hand")); + CHECK_FALSE(core::is_valid_registry_id("openxr..hand")); + CHECK_FALSE(core::is_valid_registry_id("openxr.hand.")); + CHECK_FALSE(core::is_valid_registry_id("1x.hand")); + CHECK_FALSE(core::is_valid_registry_id("openxr.hand-left")); +} + +TEST_CASE("Asset digests are RFC 6920 SHA-256 names", "[unit][skeleton_binding]") +{ + CHECK(core::is_valid_asset_digest(kDigest)); + CHECK_FALSE(core::is_valid_asset_digest("sha-256;f4OxZX_x_FO5LcGBSKHWXfwtSx-j1ncoSt3SABJtkGk")); + CHECK_FALSE(core::is_valid_asset_digest(kDigest + "=")); + CHECK_FALSE(core::is_valid_asset_digest("ni:///sha-256;f4OxZX/x+FO5LcGBSKHWXfwtSx-j1ncoSt3SABJtkGk")); +} + +TEST_CASE("validate_skeleton_binding accepts conforming bindings", "[unit][skeleton_binding]") +{ + CHECK(validate(binding({ "WRIST", "INDEX_TIP" }, "openxr.hand"), 2, core::BodySide_LEFT).empty()); + CHECK(validate(binding({ "LEFT_HAND", "RIGHT_HAND" }, "openxr.body.bd"), 2, core::BodySide_UNSPECIFIED).empty()); + CHECK(validate(with_asset(binding({ "base_link" }, "urdf.link")), 1, core::BodySide_UNSPECIFIED).empty()); + CHECK(validate(binding({ "palm", "pad" }), 2, core::BodySide_UNSPECIFIED).empty()); + // A profile this build does not know reads as a generic binding. + CHECK(validate(binding({ "anything" }, "com.example.hand", 3), 1, core::BodySide_UNSPECIFIED).empty()); +} + +TEST_CASE("validate_skeleton_binding rejects nonconforming bindings", "[unit][skeleton_binding]") +{ + CHECK_FALSE(validate(binding({ "WRIST" }, "openxr.hand"), 2, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(binding({ "WRIST", "" }, "openxr.hand"), 2, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(binding({ "WRIST", "WRIST" }, "openxr.hand"), 2, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(binding({ "WRIST" }, "openxr.hand"), 1, core::BodySide_UNSPECIFIED).empty()); + CHECK_FALSE(validate(binding({ "LEFT_HAND" }, "openxr.hand"), 1, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(binding({ "WRIST" }, "openxr.hand", 0), 1, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(binding({ "WRIST" }, "OpenXR.Hand"), 1, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(with_asset(binding({ "WRIST" }, "openxr.hand")), 1, core::BodySide_LEFT).empty()); + CHECK_FALSE(validate(binding({ "base_link" }, "urdf.link"), 1, core::BodySide_UNSPECIFIED).empty()); + CHECK_FALSE( + validate(with_asset(binding({ "base_link" }, "urdf.link"), "md5:abc"), 1, core::BodySide_UNSPECIFIED).empty()); + CHECK_FALSE(validate(with_asset(binding({ "base_link" })), 1, core::BodySide_UNSPECIFIED).empty()); +} diff --git a/tests/python/core/schema/test_skeleton_binding.py b/tests/python/core/schema/test_skeleton_binding.py new file mode 100644 index 0000000000..fb0bab6555 --- /dev/null +++ b/tests/python/core/schema/test_skeleton_binding.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for SkeletonBinding, the skeleton profile registry and the asset checks.""" + +from isaaccapture.schema import ( + SKELETON_PROFILE_OPENXR_HAND, + SKELETON_PROFILE_URDF_LINK, + AssetRef, + BodySide, + HandJoint, + SkeletonBinding, + SkeletonProfileRef, + asset_digest, + check_skeleton_asset, + find_skeleton_profile, + skeleton_profiles, + validate_skeleton_binding, +) + +URDF = b""" + + + +""" + + +def urdf_binding(names, data=URDF): + return SkeletonBinding( + names, + profile=SkeletonProfileRef(SKELETON_PROFILE_URDF_LINK, 1), + asset=AssetRef( + asset_digest(data), "package://glove/glove.urdf", "application/xml" + ), + ) + + +def test_binding_round_trips(): + binding = SkeletonBinding( + ["WRIST", "INDEX_TIP"], profile=SkeletonProfileRef("openxr.hand", 1) + ) + + assert binding.segment_names == ["WRIST", "INDEX_TIP"] + assert binding.profile.id == "openxr.hand" + assert binding.profile.version == 1 + assert binding.asset is None + assert SkeletonBinding(["a"]).profile is None + assert binding.profile.to_bytes() == SkeletonProfileRef("openxr.hand", 1).to_bytes() + + +def test_registry_lists_openxr_hand_joints(): + hand = find_skeleton_profile(SKELETON_PROFILE_OPENXR_HAND, 1) + + assert hand.requires_side + assert hand.segment_names == [ + name for name in HandJoint.__members__ if name != "NUM_JOINTS" + ] + assert find_skeleton_profile("openxr.hand", 2) is None + assert {p.id for p in skeleton_profiles()} == { + "openxr.hand", + "openxr.body.bd", + "urdf.link", + } + + +def test_validate_reports_the_first_problem(): + binding = SkeletonBinding(["WRIST"], profile=SkeletonProfileRef("openxr.hand", 1)) + + assert validate_skeleton_binding(binding, 1, BodySide.RIGHT) == "" + assert "side" in validate_skeleton_binding(binding, 1) + assert "2 segments" in validate_skeleton_binding(binding, 2, BodySide.RIGHT) + + +def test_asset_digest_matches_rfc6920_example(): + assert ( + asset_digest(b"Hello World!") + == "ni:///sha-256;f4OxZX_x_FO5LcGBSKHWXfwtSx-j1ncoSt3SABJtkGk" + ) + + +def test_check_skeleton_asset(): + binding = urdf_binding(["base_link", "index_tip"]) + + assert validate_skeleton_binding(binding, 2) == "" + assert check_skeleton_asset(binding, URDF) == "" + assert "digest" in check_skeleton_asset(binding, URDF + b" ") + assert "not a link" in check_skeleton_asset(urdf_binding(["index"]), URDF) + assert "no asset" in check_skeleton_asset(SkeletonBinding(["a"]), URDF) From 2f9743acd59a63684d022f19185a3c82f5f8f853 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 20 Oct 2026 15:58:25 -0700 Subject: [PATCH 2/3] Add the tactile schema and tracker A tactile device publishes a TactileMetadata once per session as the metadata of the generated TactileTracker, and a TactileData every frame. TactileMetadata describes the device (vendor, model, side, rate, calibration), its rigid segments with their bind poses, its taxels with their poses and four-slot linear blend skinning over the segments, the SkeletonBinding that names the segments in a skeleton, the TactileData vectors it fills, and vendor extensions. TactileData carries per-taxel normal force, shear force, raw readings and taxel state, and an optional frame counter. validate_tactile_metadata() and validate_tactile_data() check the rules in docs/source/references/tactile_schema.rst, in C++ and Python; the Python TactileMetadata constructor raises ValueError on the first one broken. Per-taxel fields read in Python as NumPy views into the buffer. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lotus Li --- docs/source/device/tactile/index.rst | 3 +- docs/source/index.rst | 1 + docs/source/references/generated_trackers.rst | 7 +- docs/source/references/tactile_schema.rst | 70 +++ src/core/deviceio_trackers/trackers.toml | 7 + .../cpp/inc/schema/tactile_validation.hpp | 215 ++++++++ src/core/schema/fbs/tactile.fbs | 49 ++ src/core/schema/fbs/tactile_metadata.fbs | 88 ++++ src/core/schema/golden/tactile.bfbs | Bin 0 -> 1632 bytes src/core/schema/golden/tactile_metadata.bfbs | Bin 0 -> 3688 bytes src/core/schema/python/CMakeLists.txt | 1 + src/core/schema/python/schema_module.cpp | 4 + src/core/schema/python/tactile_bindings.h | 494 ++++++++++++++++++ src/python/isaaccapture/schema/__init__.py | 22 + tests/cpp/core/schema/CMakeLists.txt | 1 + tests/cpp/core/schema/test_tactile.cpp | 220 ++++++++ tests/python/core/schema/test_tactile.py | 145 +++++ 17 files changed, 1324 insertions(+), 3 deletions(-) create mode 100644 docs/source/references/tactile_schema.rst create mode 100644 src/core/schema/cpp/inc/schema/tactile_validation.hpp create mode 100644 src/core/schema/fbs/tactile.fbs create mode 100644 src/core/schema/fbs/tactile_metadata.fbs create mode 100644 src/core/schema/golden/tactile.bfbs create mode 100644 src/core/schema/golden/tactile_metadata.bfbs create mode 100644 src/core/schema/python/tactile_bindings.h create mode 100644 tests/cpp/core/schema/test_tactile.cpp create mode 100644 tests/python/core/schema/test_tactile.py diff --git a/docs/source/device/tactile/index.rst b/docs/source/device/tactile/index.rst index 57b8522573..daf2fa0250 100644 --- a/docs/source/device/tactile/index.rst +++ b/docs/source/device/tactile/index.rst @@ -5,7 +5,8 @@ Tactile devices =============== Tactile devices capture contact measurements. For actuator output, see -:doc:`/references/haptic_feedback`. +:doc:`/references/haptic_feedback`. A device can publish through the vendor-neutral +:doc:`/references/tactile_schema`. .. toctree:: :maxdepth: 1 diff --git a/docs/source/index.rst b/docs/source/index.rst index 3b66361a50..cfba8f1013 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -93,6 +93,7 @@ Table of Contents references/build references/generated_trackers references/skeleton_binding + references/tactile_schema references/retargeting/index references/device_provider_monitoring references/camera_streaming diff --git a/docs/source/references/generated_trackers.rst b/docs/source/references/generated_trackers.rst index 1147ed9ce6..75b34ffee4 100644 --- a/docs/source/references/generated_trackers.rst +++ b/docs/source/references/generated_trackers.rst @@ -86,10 +86,10 @@ example) declares that description as the tracker's ``metadata`` table: [[tracker]] name = "tactile" table = "TactileData" - metadata = "TactileShape" + metadata = "TactileMetadata" The metadata table is the ``root_type`` of a schema of its own, named after the table in snake case -(``tactile_shape.fbs``), with no Record wrapper: MCAP records the table itself. Like the data table, +(``tactile_metadata.fbs``), with no Record wrapper: MCAP records the table itself. Like the data table, it needs a hand-written ``schema/python/*_bindings.h`` before ``get_metadata()`` can return it to Python. Only ``direction = "pull"`` supports ``metadata``. @@ -154,6 +154,9 @@ Generated today * - ``gamepad`` - ``GamepadTracker`` - Gamepad state pushed by the gamepad plugin (SDL3); only ``python_accessor`` is overridden + * - ``tactile`` + - ``TactileTracker`` + - Taxel readings with a ``TactileMetadata`` device description; see :doc:`tactile_schema` Still hand-written ------------------ diff --git a/docs/source/references/tactile_schema.rst b/docs/source/references/tactile_schema.rst new file mode 100644 index 0000000000..7102460cb2 --- /dev/null +++ b/docs/source/references/tactile_schema.rst @@ -0,0 +1,70 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Tactile Schema +============== + +A tactile device publishes through the generated ``TactileTracker``: a ``TactileMetadata`` once per +session as the tracker's :ref:`metadata `, and a ``TactileData`` every frame. The +schemas are :code-file:`src/core/schema/fbs/tactile_metadata.fbs` and +:code-file:`src/core/schema/fbs/tactile.fbs`; the checks below are +:code-file:`src/core/schema/cpp/inc/schema/tactile_validation.hpp`. + +Device description +------------------ + +``TactileMetadata`` holds: + +- ``device`` — vendor, model, serial number, firmware, ``side`` (a ``BodySide``), nominal rate and + calibration id. +- ``shape`` — the rigid ``segments`` of the device, each with its bind pose, and the ``taxels``, each + with its pose and up to four skinning weights over the segments. Both are in the device frame at + the reference configuration. ``taxel_areas`` optionally gives each taxel's area in m². +- ``binding`` — a :doc:`skeleton_binding` naming each segment in the skeleton that poses it. Without + one the segments are not posed by anything and the taxels stay at their bind poses. +- ``channels`` — which ``TactileData`` vectors the device fills. +- ``extensions`` — vendor data, each named by a registry id such as ``com.example.glove``. + +A taxel's frame has +Z along the outward surface normal and +X along the shear x axis. Its position +at run time is + +.. code-block:: text + + p = sum_k weights[k] * T_current[segments[k]] * inverse(T_bind[segments[k]]) * p_bind + +where ``T_current`` is the segment's pose from the bound skeleton, and ``p`` comes out in the frame +that skeleton is posed in. The taxel's axes blend the same way and are renormalized. + +Per-frame data +-------------- + +Entry *i* of each ``TactileData`` vector is taxel *i* of the shape. Forces are what the contacted +surface exerts on the taxel, in newtons: + +- ``normal_force`` — along the taxel normal; positive presses into the surface. +- ``shear_force`` — along the taxel frame's +X and +Y. +- ``raw`` — ``channels.raw_values_per_taxel`` uncalibrated readings per taxel. +- ``taxel_state`` — ``VALID``, ``SATURATED``, ``INVALID`` or ``UNAVAILABLE``; when absent, every + taxel is ``VALID``. +- ``sequence`` — the device's frame counter, when it has one. + +Rules +----- + +``validate_tactile_metadata(metadata)`` returns an empty string for a valid description, otherwise +the first rule it breaks. The Python ``TactileMetadata`` constructor raises ``ValueError`` with the +same message. + +#. ``shape`` has at least one segment and one taxel. +#. Every bind pose and taxel pose is finite, with an orientation of norm 1 within 1e-3. +#. Each taxel's weights are finite and nonnegative and sum to 1 within 1e-5. A slot with a nonzero + weight names an existing segment that no other slot of the taxel names; a zero-weight slot + holds segment 0. +#. ``taxel_areas`` is empty or holds one finite, positive value per taxel. +#. ``channels`` is present and declares normal force, shear force or raw readings. +#. ``binding``, when present, passes ``validate_skeleton_binding`` for the shape's segments and + ``device.side``. +#. Extension ids are registry ids and unique. + +``validate_tactile_data(data, metadata)`` checks that a frame fills exactly the declared vectors, +with one entry per taxel (``raw_values_per_taxel`` per taxel for ``raw``). diff --git a/src/core/deviceio_trackers/trackers.toml b/src/core/deviceio_trackers/trackers.toml index d27518d43c..bef7c9523b 100644 --- a/src/core/deviceio_trackers/trackers.toml +++ b/src/core/deviceio_trackers/trackers.toml @@ -111,3 +111,10 @@ python_accessor = "get_spacemouse_data" name = "gamepad" table = "GamepadSnapshot" python_accessor = "get_gamepad_data" + +[[tracker]] +name = "tactile" +table = "TactileData" +metadata = "TactileMetadata" +# 1000 taxels with normal force, shear, state and two raw values encode to about 21 KB. +max_flatbuffer_size = 32768 diff --git a/src/core/schema/cpp/inc/schema/tactile_validation.hpp b/src/core/schema/cpp/inc/schema/tactile_validation.hpp new file mode 100644 index 0000000000..4a33cd0ffa --- /dev/null +++ b/src/core/schema/cpp/inc/schema/tactile_validation.hpp @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The checks a TactileMetadata and each TactileData published under it must pass. +// docs/source/references/tactile_schema.rst lists the rules. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace core +{ + +//! Largest deviation from 1 a taxel's weights may sum to. +inline constexpr float TACTILE_WEIGHT_SUM_TOLERANCE = 1e-5f; +//! Largest deviation from 1 the norm of a bind or taxel orientation may have. +inline constexpr float TACTILE_QUATERNION_NORM_TOLERANCE = 1e-3f; + +namespace detail +{ + +inline bool is_rigid_pose(const Pose& pose) +{ + const auto& p = pose.position(); + const auto& q = pose.orientation(); + if (!std::isfinite(p.x()) || !std::isfinite(p.y()) || !std::isfinite(p.z())) + { + return false; + } + const float norm = std::sqrt(q.x() * q.x() + q.y() * q.y() + q.z() * q.z() + q.w() * q.w()); + return std::isfinite(norm) && std::fabs(norm - 1.0f) <= TACTILE_QUATERNION_NORM_TOLERANCE; +} + +inline std::string check_vector_size(const char* name, size_t present, size_t expected, bool declared) +{ + if (!declared) + { + return present == 0 ? "" : std::string(name) + " is filled but not declared in channels"; + } + if (present != expected) + { + return std::string(name) + " has " + std::to_string(present) + " values, expected " + std::to_string(expected); + } + return ""; +} + +} // namespace detail + +/*! + * @brief Checks `metadata` against the rules for a tactile device description. + * @return Empty when valid, otherwise the first rule it breaks. + */ +inline std::string validate_tactile_metadata(const TactileMetadata& metadata) +{ + const auto* shape = metadata.shape(); + if (shape == nullptr) + { + return "shape is missing"; + } + const auto* segments = shape->segments(); + const size_t segment_count = segments != nullptr ? segments->size() : 0; + if (segment_count == 0) + { + return "shape has no segments"; + } + for (size_t s = 0; s < segment_count; ++s) + { + if (!detail::is_rigid_pose(segments->Get(static_cast(s))->bind_pose())) + { + return "segments[" + std::to_string(s) + "].bind_pose is not finite with a unit orientation"; + } + } + + const auto* taxels = shape->taxels(); + const size_t taxel_count = taxels != nullptr ? taxels->size() : 0; + if (taxel_count == 0) + { + return "shape has no taxels"; + } + for (size_t i = 0; i < taxel_count; ++i) + { + const auto* taxel = taxels->Get(static_cast(i)); + const std::string at = "taxels[" + std::to_string(i) + "]"; + if (!detail::is_rigid_pose(taxel->pose())) + { + return at + ".pose is not finite with a unit orientation"; + } + float sum = 0.0f; + std::set weighted; + for (flatbuffers::uoffset_t k = 0; k < 4; ++k) + { + const float weight = taxel->weights()->Get(k); + const uint16_t segment = taxel->segments()->Get(k); + if (!std::isfinite(weight) || weight < 0.0f) + { + return at + ".weights[" + std::to_string(k) + "] is negative or not finite"; + } + if (weight == 0.0f) + { + if (segment != 0) + { + return at + ".segments[" + std::to_string(k) + "] has zero weight but is not 0"; + } + continue; + } + if (segment >= segment_count) + { + return at + ".segments[" + std::to_string(k) + "] is " + std::to_string(segment) + ", past the " + + std::to_string(segment_count) + " segments"; + } + if (!weighted.insert(segment).second) + { + return at + " weights segment " + std::to_string(segment) + " twice"; + } + sum += weight; + } + if (std::fabs(sum - 1.0f) > TACTILE_WEIGHT_SUM_TOLERANCE) + { + return at + ".weights sum to " + std::to_string(sum) + ", not 1"; + } + } + + if (const auto* areas = shape->taxel_areas()) + { + if (areas->size() != taxel_count) + { + return "taxel_areas has " + std::to_string(areas->size()) + " values for " + std::to_string(taxel_count) + + " taxels"; + } + for (const float area : *areas) + { + if (!std::isfinite(area) || area <= 0.0f) + { + return "taxel_areas must be finite and positive"; + } + } + } + + const auto* channels = metadata.channels(); + if (channels == nullptr) + { + return "channels is missing"; + } + if (!channels->normal_force() && !channels->shear_force() && channels->raw_values_per_taxel() == 0) + { + return "channels declares no force or raw data"; + } + + if (const auto* binding = metadata.binding()) + { + const BodySide side = metadata.device() != nullptr ? metadata.device()->side() : BodySide_UNSPECIFIED; + const std::string error = validate_skeleton_binding(*binding, segment_count, side); + if (!error.empty()) + { + return "binding: " + error; + } + } + + if (const auto* extensions = metadata.extensions()) + { + std::set ids; + for (const auto* extension : *extensions) + { + const std::string id = extension->id() != nullptr ? extension->id()->str() : std::string(); + if (!is_valid_registry_id(id)) + { + return "extension id '" + id + "' is not a registry id"; + } + if (!ids.insert(id).second) + { + return "extension id '" + id + "' appears twice"; + } + } + } + return ""; +} + +/*! + * @brief Checks that `data` fills exactly the channels `metadata` declares, one entry per taxel. + * + * Precondition: `metadata` passes validate_tactile_metadata(). + * @return Empty when valid, otherwise what is wrong. + */ +inline std::string validate_tactile_data(const TactileData& data, const TactileMetadata& metadata) +{ + const size_t taxel_count = metadata.shape()->taxels()->size(); + const auto* channels = metadata.channels(); + const auto size = [](const auto* vector) -> size_t { return vector != nullptr ? vector->size() : 0; }; + + std::string error = + detail::check_vector_size("normal_force", size(data.normal_force()), taxel_count, channels->normal_force()); + if (error.empty()) + { + error = detail::check_vector_size("shear_force", size(data.shear_force()), taxel_count, channels->shear_force()); + } + if (error.empty()) + { + error = detail::check_vector_size("raw", size(data.raw()), taxel_count * channels->raw_values_per_taxel(), + channels->raw_values_per_taxel() > 0); + } + if (error.empty()) + { + error = detail::check_vector_size("taxel_state", size(data.taxel_state()), taxel_count, channels->taxel_state()); + } + return error; +} + +} // namespace core diff --git a/src/core/schema/fbs/tactile.fbs b/src/core/schema/fbs/tactile.fbs new file mode 100644 index 0000000000..7d9322eed4 --- /dev/null +++ b/src/core/schema/fbs/tactile.fbs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// One frame of a tactile device. Entry i of each vector is taxel i of the TactileShape in +// the tracker's TactileMetadata, and a vector is filled exactly when its TactileChannels +// flag is set. Forces are what the contacted surface exerts on the taxel, in newtons. + +include "timestamp.fbs"; + +namespace core; + +enum TaxelState : ubyte { + VALID = 0, + // At the top of the sensing range: the true force is at least the reported one. + SATURATED = 1, + // Read, but known to be wrong, e.g. a damaged taxel. + INVALID = 2, + // Not read this frame. + UNAVAILABLE = 3, +} + +// Tangential force along the taxel frame's +X and +Y. +struct TactileShear { + x: float; + y: float; +} + +table TactileData { + // Force along the taxel normal; positive presses into the surface. + normal_force: [float] (id: 0); + + shear_force: [TactileShear] (id: 1); + + // Uncalibrated readings, TactileChannels.raw_values_per_taxel consecutive values per taxel. + raw: [int] (id: 2); + + // When absent, every taxel is VALID. + taxel_state: [TaxelState] (id: 3); + + // Device frame counter; gaps mean dropped frames. Null when the device has none. + sequence: uint = null (id: 4); +} + +table TactileDataRecord { + data: TactileData (id: 0); + timestamp: DeviceDataTimestamp (id: 1); +} + +root_type TactileDataRecord; diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs new file mode 100644 index 0000000000..49c637edd9 --- /dev/null +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// What a tactile device is and where its taxels sit, published once per session as the +// metadata of the tactile tracker. The rules a valid TactileMetadata follows are in +// docs/source/references/tactile_schema.rst. + +include "pose.fbs"; +include "skeleton_binding.fbs"; + +namespace core; + +// One rigid segment of the device, e.g. a finger link the taxels on it move with. +struct TactileSegment { + // Segment frame in the device frame at the reference configuration. + bind_pose: Pose; +} + +// One taxel at the reference configuration, in the device frame. +struct TactileTaxel { + // Taxel frame: +Z is the outward surface normal, +X the shear x axis, +Y = Z x X. + pose: Pose; + + // Linear blend skinning over four slots: segment segments[k] with weight weights[k]. + // Weights are finite, nonnegative and sum to 1; a zero-weight slot holds segment 0. + // p = sum_k weights[k] * T_current[segments[k]] * inverse(T_bind[segments[k]]) * p_bind + // moves the taxel position; its axes blend the same way and are renormalized. p comes out + // in the frame the binding's skeleton poses T_current in. + segments: [ushort:4]; + weights: [float:4]; +} + +// Segments and taxels of the device. Taxel i is entry i of every TactileData vector. +table TactileShape { + segments: [TactileSegment] (id: 0); + taxels: [TactileTaxel] (id: 1); + + // Sensing area of each taxel in square meters; empty when unknown. + taxel_areas: [float] (id: 2); +} + +// Identity of the device that produced the data. +table TactileDevice { + vendor: string (id: 0); + model: string (id: 1); + serial_number: string (id: 2); + firmware_version: string (id: 3); + side: BodySide (id: 4); + + // Rate the device samples at, in hertz; null when it is not fixed. + nominal_rate_hz: float = null (id: 5); + + // Identifies the calibration that turned raw readings into forces. + calibration_id: string (id: 6); +} + +// Which TactileData vectors the device fills. +table TactileChannels { + normal_force: bool (id: 0); + shear_force: bool (id: 1); + + // Raw readings per taxel in TactileData.raw; 0 when the device sends none. + raw_values_per_taxel: ubyte (id: 2); + + taxel_state: bool (id: 3); +} + +// Vendor data no core field covers, named by a registry id such as "com.example.glove". +table TactileExtension { + id: string (id: 0); + version: uint (id: 1); + data: [ubyte] (id: 2); +} + +// Published as the metadata of the tactile tracker (`metadata` in trackers.toml), not as +// samples: it carries no timestamp, and the tracker holds one for the whole session. +table TactileMetadata { + device: TactileDevice (id: 0); + shape: TactileShape (id: 1); + + // Names shape.segments in the skeleton that poses them. + binding: SkeletonBinding (id: 2); + + channels: TactileChannels (id: 3); + extensions: [TactileExtension] (id: 4); +} + +root_type TactileMetadata; diff --git a/src/core/schema/golden/tactile.bfbs b/src/core/schema/golden/tactile.bfbs new file mode 100644 index 0000000000000000000000000000000000000000..e8f8d2763a7a512f64c93c11101c65b1e96e0fd9 GIT binary patch literal 1632 zcmZ{kF;7!b6vt1YK#>ZS7{kz*nmBX<2L=ZRONxP{VW5RDkkm(cgvOR?`$XIr85|fK z_y7zJegFqQ0E2^r1A_w}!pXt<```OsDMT;%_uhN%x&QM&=blHw%$8m(RhjI*6)kNU z%UX{0oE0q3Z1x;1f+EO+A1TAb>={Ud{|(%18_4GqKieZ_XBe+W%~%q_pB?}1_s=;` zxK7z$;-kGdPTavPTBOd% z-Eh7Z9EGiF6hxtY?DzWx?8+7D!Kh(RxTdgftd%xPtCiAHWf^T!DTS(|ua{=!t8qGX zS2|bMHcOS&GQM*+?iDywT*XPox1qa}7^|h)#(Jr?TxOl+&m|eheT+?SmvbsuFjT#S zQ@mrqM>kVWdG^w(vn>Qm^A;$82OzFbBWkw7au5aUA(maU&wu)Tj#DWcAH3;z_^_$0 zj!$}{W;^UfLHjl9W8F!mT~B$Pxxu?hg>*7zq(x9}yToK7ubZ9VU)}d-qF;Tc{1%x% zFsmLHpaR5wI~PnAdCjQOEBcr#KDUfCsHb}2o44UXBV=jkvHz&u8RUK9Y}1nKbgMri zwN#zV>8Z>3&IoKtjFNidWrSZCc(=+0e zO*(q}VbHDbb-KvUJ8o5L91M>!&rcOpGtZsHOWmsBd&q2hN_xd`(CM~=R!q!(E6#C| zwK(J_4Vv$Qg@q_^O*Fr^(_^nbD+Bf4Juv(>RcXfdi+4!DkkstT$C%Q)2=BoKK8lxU zP845jwt@eV^UI?vOJ@47p)nWFmU_xgT+*%o57Mq?Mtjo5PQB`T0mLVN`8xgO@UYo% zpGhsbz>$4L_XzP4&V8T9q~a(K)u6{G(hBPeRHql}yN;^fXmuK|ICfu$bf{oU>F# zWctqZA_l3*h!mw?24o2DH>4`V7)7Rl319>m27V|ABt({h%fSCV5R-3#YrxT4A~n2> zUlpkih@A9`L~n~6T*7-%5O2_`Fo7IFt$Tr8F&rQ?lXY4b)HM%Y92S$$0`q_Fb}aTU;d(bDimRfoG{L_ z-?=teCYPL}ETVRDm1J={$xs)oa|vIZPTj1P5r)p6x~q*aNmLUWBHvq`KIG+etnV^? zFi*6Vti(waC+ku{eq1g{sVjeEeUJ;vGvBNcZm@1ID32_h?K3M&o0Mf-k=lw^A-w;l z)2U+axLbxH_6GUf0p@}C0H3p2mp|5zZEVbi+%o=};jBnsoI8G>|IFSgb>$8)$LD^R z@0Lo*s*Z{AJA2QJ+%?Lv593H=<~Qsiyt}xc!V!JF1nvRs?JCe)@7rD-mcWZ&w|nSO zsPxefL2mPKM>T^i4|65EkoRl0XXJIBl&7yW+zz(F=BCPm4V4C39n10!bp850{|{eT zLxpLu)=p6vo^57Flyr{Fjy0i=A^fqHe9S4RU%PRWlcb%t!lpHbd5)TlV|0A&pvbf_ z0h|F-NZ>w3CbNg}K1QA;e%)tm&pZM5_*5t%4#q>?@0&i*H5#N|*o;@wFh@ND*6Ymb zT$aJE&VkXnO76-)(r(2Gf`*z38V}_seCJ*q#W$08MaRtjM1F=a;-Pz8FWTWGGslpqkMYwY4IqTeB+ZR0Xd2u(oUU{GPt$>G7=0R2n1HWV5xRQp}%O9lyUf zJRR=ORy$Hnj7^@$UVk5Ya@?C+Dv8=KI`Zemz4;soa&PVeRbU>t4)`@%v}ZuwocRSm zhm*gU``}e_bl%%YKFnmt%KJUa^PguQdxm|&*mOhV6dOlu%)#%;5~{=6nCfsGEc3kN z9m4fYt+!N?>oai4@>Rn+>%f@k;}yUh4*=#z-QFCRxF@kb${0}kKB;x<%b2etL)v41 zZ>xB{0hjlzYylSO)ASSX9#)2#nLFlssi@DG-gxU)-<_$}cbVsr>${D1h7%RsSS}~= zJ_;}v=E3oOU~DWmmru~Lx6jSB6|h1(lzSJrP*cO3yy%JH*vz{z;HuYmMqmSJ{C%;1 zj&1*tyF;J~kO%E+@pIKwc{`c5mk!qVb-$3`{tKM3&gL3NU`x^W<}Ej7@nS`vlg$0@$-U z_64(f_)Fg0ebn-wD<%%mQ-Fl)kFWI^C1uutyzyLXr!kHhefL1e<>lMr`?)=GPX67! z+Rx8A3jI>1U4rY#BIc#WFE@8zxu9ZEZaZ@{{0?))g$u|GLw)gz=E?=G% +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace core +{ + +namespace tactile_detail +{ + +template +using CArray = py::array_t; + +//! Rows of `array`, which must be (rows, width), or (rows,) when width is 0. +template +size_t checked_rows(const CArray& array, const char* name, py::ssize_t width) +{ + const bool ok = width > 0 ? array.ndim() == 2 && array.shape(1) == width : array.ndim() == 1; + if (!ok) + { + throw py::value_error(std::string(name) + " must have shape (N" + + (width > 0 ? ", " + std::to_string(width) : std::string(",")) + ")"); + } + return static_cast(array.shape(0)); +} + +//! Integers of `array` narrowed to Out, refusing values Out cannot hold rather than wrapping them. +template +std::vector checked_integers(const CArray& array, const char* name, int64_t min, int64_t max) +{ + std::vector values; + values.reserve(static_cast(array.size())); + for (py::ssize_t i = 0; i < array.size(); ++i) + { + const int64_t value = array.data()[i]; + if (value < min || value > max) + { + throw py::value_error(std::string(name) + " holds " + std::to_string(value) + ", outside [" + + std::to_string(min) + ", " + std::to_string(max) + "]"); + } + values.push_back(static_cast(value)); + } + return values; +} + +//! (count, width) view of a vector's elements, `stride` bytes apart, or None when it is absent. +template +py::object vector_view(py::object owner, const uint8_t* first, size_t count, py::ssize_t stride, py::ssize_t width) +{ + if (first == nullptr || count == 0) + { + return py::none(); + } + return strided_field_view( + owner, reinterpret_cast(first), stride, static_cast(count), width); +} + +template +const uint8_t* vector_data(const V* vector) +{ + return vector != nullptr ? vector->Data() : nullptr; +} + +template +size_t vector_size(const V* vector) +{ + return vector != nullptr ? vector->size() : 0; +} + +template +std::shared_ptr unpack(const Serialized* handle) +{ + if (handle == nullptr || !*handle) + { + return nullptr; + } + auto native = std::make_shared(); + (*handle)->UnPackTo(native.get()); + return native; +} + +constexpr py::ssize_t SEGMENT_STRIDE = sizeof(TactileSegment); +constexpr py::ssize_t TAXEL_STRIDE = sizeof(TactileTaxel); + +} // namespace tactile_detail + +inline void bind_tactile(py::module& m) +{ + using namespace tactile_detail; + + py::enum_(m, "TaxelState") + .value("VALID", TaxelState_VALID) + .value("SATURATED", TaxelState_SATURATED) + .value("INVALID", TaxelState_INVALID) + .value("UNAVAILABLE", TaxelState_UNAVAILABLE); + + serialized_class(m, "TactileDevice", "Encoded identity of a tactile device.") + .def(py::init( + [](const std::string& vendor, const std::string& model, const std::string& serial_number, + const std::string& firmware_version, BodySide side, std::optional nominal_rate_hz, + const std::string& calibration_id) + { + TactileDeviceT native; + native.vendor = vendor; + native.model = model; + native.serial_number = serial_number; + native.firmware_version = firmware_version; + native.side = side; + if (nominal_rate_hz) + { + native.nominal_rate_hz = *nominal_rate_hz; + } + native.calibration_id = calibration_id; + return pack(native); + }), + py::arg("vendor") = std::string{}, py::arg("model") = std::string{}, + py::arg("serial_number") = std::string{}, py::arg("firmware_version") = std::string{}, + py::arg("side") = BodySide_UNSPECIFIED, py::arg("nominal_rate_hz") = py::none(), + py::arg("calibration_id") = std::string{}) + .def_property_readonly("vendor", string_field(&TactileDevice::vendor)) + .def_property_readonly("model", string_field(&TactileDevice::model)) + .def_property_readonly("serial_number", string_field(&TactileDevice::serial_number)) + .def_property_readonly("firmware_version", string_field(&TactileDevice::firmware_version)) + .def_property_readonly("side", field(&TactileDevice::side)) + .def_property_readonly("nominal_rate_hz", + [](const Serialized& self) -> std::optional + { + const auto rate = self->nominal_rate_hz(); + return rate ? std::optional(*rate) : std::nullopt; + }) + .def_property_readonly("calibration_id", string_field(&TactileDevice::calibration_id)) + .def("__repr__", + [](const Serialized& self) + { + const auto* vendor = self->vendor(); + const auto* model = self->model(); + return "TactileDevice(vendor=" + (vendor != nullptr ? vendor->str() : std::string{}) + + ", model=" + (model != nullptr ? model->str() : std::string{}) + + ", side=" + EnumNameBodySide(self->side()) + ")"; + }); + + serialized_class(m, "TactileChannels", "Encoded set of TactileData vectors a device fills.") + .def(py::init( + [](bool normal_force, bool shear_force, uint8_t raw_values_per_taxel, bool taxel_state) + { + TactileChannelsT native; + native.normal_force = normal_force; + native.shear_force = shear_force; + native.raw_values_per_taxel = raw_values_per_taxel; + native.taxel_state = taxel_state; + return pack(native); + }), + py::arg("normal_force") = false, py::arg("shear_force") = false, py::arg("raw_values_per_taxel") = 0, + py::arg("taxel_state") = false) + .def_property_readonly("normal_force", field(&TactileChannels::normal_force)) + .def_property_readonly("shear_force", field(&TactileChannels::shear_force)) + .def_property_readonly("raw_values_per_taxel", field(&TactileChannels::raw_values_per_taxel)) + .def_property_readonly("taxel_state", field(&TactileChannels::taxel_state)) + .def("__repr__", + [](const Serialized& self) + { + return std::string("TactileChannels(normal_force=") + (self->normal_force() ? "True" : "False") + + ", shear_force=" + (self->shear_force() ? "True" : "False") + + ", raw_values_per_taxel=" + std::to_string(self->raw_values_per_taxel()) + + ", taxel_state=" + (self->taxel_state() ? "True" : "False") + ")"; + }); + + serialized_class(m, "TactileExtension", "Encoded vendor data, named by a registry id.") + .def(py::init( + [](const std::string& id, uint32_t version, const py::bytes& data) + { + TactileExtensionT native; + native.id = id; + native.version = version; + const std::string bytes = data; + native.data.assign(bytes.begin(), bytes.end()); + return pack(native); + }), + py::arg("id"), py::arg("version"), py::arg("data") = py::bytes()) + .def_property_readonly("id", string_field(&TactileExtension::id)) + .def_property_readonly("version", field(&TactileExtension::version)) + .def_property_readonly("data", + [](const Serialized& self) + { + const auto* data = self->data(); + return data != nullptr ? + py::bytes(reinterpret_cast(data->data()), data->size()) : + py::bytes(); + }) + .def("__repr__", + [](const Serialized& self) + { + const auto* id = self->id(); + return "TactileExtension(id=" + (id != nullptr ? id->str() : std::string{}) + + ", version=" + std::to_string(self->version()) + ")"; + }); + + // Per-row views alias the buffer; see schema_array_views.h for why they are writable. + serialized_class(m, "TactileShape", "Encoded segments and taxels of a tactile device.") + .def(py::init( + [](const CArray& segment_bind_positions, const CArray& segment_bind_orientations, + const CArray& taxel_positions, const CArray& taxel_orientations, + const CArray& taxel_segments, const CArray& taxel_weights, + std::optional> taxel_areas) + { + const size_t segment_count = checked_rows(segment_bind_positions, "segment_bind_positions", 3); + const size_t taxel_count = checked_rows(taxel_positions, "taxel_positions", 3); + if (checked_rows(segment_bind_orientations, "segment_bind_orientations", 4) != segment_count) + { + throw py::value_error("segment_bind_orientations must have one row per segment"); + } + if (checked_rows(taxel_orientations, "taxel_orientations", 4) != taxel_count || + checked_rows(taxel_segments, "taxel_segments", 4) != taxel_count || + checked_rows(taxel_weights, "taxel_weights", 4) != taxel_count) + { + throw py::value_error( + "taxel_orientations, taxel_segments and taxel_weights must have one row per taxel"); + } + const auto segments = checked_integers( + taxel_segments, "taxel_segments", 0, std::numeric_limits::max()); + + const auto pose = [](const float* p, const float* q) + { return Pose(Point(p[0], p[1], p[2]), Quaternion(q[0], q[1], q[2], q[3])); }; + TactileShapeT native; + for (size_t s = 0; s < segment_count; ++s) + { + native.segments.emplace_back( + pose(segment_bind_positions.data() + 3 * s, segment_bind_orientations.data() + 4 * s)); + } + for (size_t i = 0; i < taxel_count; ++i) + { + native.taxels.emplace_back( + pose(taxel_positions.data() + 3 * i, taxel_orientations.data() + 4 * i), + flatbuffers::span(segments.data() + 4 * i, 4), + flatbuffers::span(taxel_weights.data() + 4 * i, 4)); + } + if (taxel_areas) + { + if (checked_rows(*taxel_areas, "taxel_areas", 0) != taxel_count) + { + throw py::value_error("taxel_areas must have one value per taxel"); + } + native.taxel_areas.assign(taxel_areas->data(), taxel_areas->data() + taxel_count); + } + return pack(native); + }), + py::arg("segment_bind_positions"), py::arg("segment_bind_orientations"), py::arg("taxel_positions"), + py::arg("taxel_orientations"), py::arg("taxel_segments"), py::arg("taxel_weights"), + py::arg("taxel_areas") = py::none(), + "Encode a shape from per-segment (S, 3) positions and (S, 4) XYZW orientations, and per-taxel " + "(N, 3) positions, (N, 4) orientations, (N, 4) segment indices, (N, 4) weights and (N,) areas.") + .def_property_readonly( + "segment_count", [](const Serialized& self) { return vector_size(self->segments()); }) + .def_property_readonly( + "taxel_count", [](const Serialized& self) { return vector_size(self->taxels()); }) + .def_property_readonly("segment_bind_positions", + [](py::object self) + { + const auto* segments = self.cast&>()->segments(); + return vector_view( + self, vector_data(segments), vector_size(segments), SEGMENT_STRIDE, 3); + }) + .def_property_readonly( + "segment_bind_orientations", + [](py::object self) + { + const auto* segments = self.cast&>()->segments(); + const auto* first = segments != nullptr && segments->size() > 0 ? + reinterpret_cast(&segments->Get(0)->bind_pose().orientation()) : + nullptr; + return vector_view(self, first, vector_size(segments), SEGMENT_STRIDE, 4); + }) + .def_property_readonly("taxel_positions", + [](py::object self) + { + const auto* taxels = self.cast&>()->taxels(); + return vector_view( + self, vector_data(taxels), vector_size(taxels), TAXEL_STRIDE, 3); + }) + .def_property_readonly("taxel_orientations", + [](py::object self) + { + const auto* taxels = self.cast&>()->taxels(); + const auto* first = + taxels != nullptr && taxels->size() > 0 ? + reinterpret_cast(&taxels->Get(0)->pose().orientation()) : + nullptr; + return vector_view(self, first, vector_size(taxels), TAXEL_STRIDE, 4); + }) + .def_property_readonly("taxel_segments", + [](py::object self) + { + const auto* taxels = self.cast&>()->taxels(); + const auto* first = taxels != nullptr && taxels->size() > 0 ? + taxels->Get(0)->segments()->Data() : + nullptr; + return vector_view(self, first, vector_size(taxels), TAXEL_STRIDE, 4); + }) + .def_property_readonly("taxel_weights", + [](py::object self) + { + const auto* taxels = self.cast&>()->taxels(); + const auto* first = taxels != nullptr && taxels->size() > 0 ? + taxels->Get(0)->weights()->Data() : + nullptr; + return vector_view(self, first, vector_size(taxels), TAXEL_STRIDE, 4); + }) + .def_property_readonly("taxel_areas", + [](py::object self) + { + const auto* areas = self.cast&>()->taxel_areas(); + return vector_view( + self, vector_data(areas), vector_size(areas), sizeof(float), 0); + }) + .def("__repr__", + [](const Serialized& self) + { + return "TactileShape(segments=" + std::to_string(vector_size(self->segments())) + + ", taxels=" + std::to_string(vector_size(self->taxels())) + ")"; + }); + + serialized_class(m, "TactileMetadata", "Encoded description of a tactile device.") + .def(py::init( + [](const Serialized& shape, const Serialized& channels, + const Serialized* device, const Serialized* binding, + const std::vector>& extensions) + { + TactileMetadataT native; + native.shape = unpack(&shape); + native.channels = unpack(&channels); + native.device = unpack(device); + native.binding = unpack(binding); + native.extensions = to_native_vector(extensions, "extensions"); + auto metadata = pack(native); + const std::string error = validate_tactile_metadata(*metadata); + if (!error.empty()) + { + throw py::value_error(error); + } + return metadata; + }), + py::arg("shape"), py::arg("channels"), py::arg("device") = nullptr, py::arg("binding") = nullptr, + py::arg("extensions") = std::vector>{}, + "Encode and validate a device description; raises ValueError naming the first rule it breaks.") + .def_property_readonly("device", + [](const Serialized& self) -> py::object + { return self->device() != nullptr ? py::cast(self.narrow(self->device())) : py::none(); }) + .def_property_readonly("shape", + [](const Serialized& self) -> py::object + { return self->shape() != nullptr ? py::cast(self.narrow(self->shape())) : py::none(); }) + .def_property_readonly( + "binding", + [](const Serialized& self) -> py::object + { return self->binding() != nullptr ? py::cast(self.narrow(self->binding())) : py::none(); }) + .def_property_readonly( + "channels", + [](const Serialized& self) -> py::object + { return self->channels() != nullptr ? py::cast(self.narrow(self->channels())) : py::none(); }) + .def_property_readonly("extensions", [](const Serialized& self) + { return narrow_vector(self, self->extensions()); }) + .def("__repr__", + [](const Serialized& self) + { + const auto* shape = self->shape(); + return "TactileMetadata(taxels=" + std::to_string(shape != nullptr ? vector_size(shape->taxels()) : 0) + + ")"; + }); + + serialized_class(m, "TactileData", "Encoded frame of per-taxel tactile readings.") + .def(py::init( + [](std::optional> normal_force, std::optional> shear_force, + std::optional> raw, std::optional> taxel_state, + std::optional sequence) + { + TactileDataT native; + if (normal_force) + { + const size_t n = checked_rows(*normal_force, "normal_force", 0); + native.normal_force.assign(normal_force->data(), normal_force->data() + n); + } + if (shear_force) + { + const size_t n = checked_rows(*shear_force, "shear_force", 2); + for (size_t i = 0; i < n; ++i) + { + native.shear_force.emplace_back(shear_force->data()[2 * i], shear_force->data()[2 * i + 1]); + } + } + if (raw) + { + checked_rows(*raw, "raw", 0); + native.raw = checked_integers( + *raw, "raw", std::numeric_limits::min(), std::numeric_limits::max()); + } + if (taxel_state) + { + checked_rows(*taxel_state, "taxel_state", 0); + native.taxel_state = + checked_integers(*taxel_state, "taxel_state", TaxelState_MIN, TaxelState_MAX); + } + if (sequence) + { + native.sequence = *sequence; + } + return pack(native); + }), + py::arg("normal_force") = py::none(), py::arg("shear_force") = py::none(), py::arg("raw") = py::none(), + py::arg("taxel_state") = py::none(), py::arg("sequence") = py::none(), + "Encode one frame from (N,) normal forces, (N, 2) shear forces, (N * k,) raw readings and (N,) " + "TaxelState values; leave out the channels the device does not declare.") + .def_property_readonly("normal_force", + [](py::object self) + { + const auto* values = self.cast&>()->normal_force(); + return vector_view( + self, vector_data(values), vector_size(values), sizeof(float), 0); + }) + .def_property_readonly("shear_force", + [](py::object self) + { + const auto* values = self.cast&>()->shear_force(); + return vector_view( + self, vector_data(values), vector_size(values), sizeof(TactileShear), 2); + }) + .def_property_readonly("raw", + [](py::object self) + { + const auto* values = self.cast&>()->raw(); + return vector_view( + self, vector_data(values), vector_size(values), sizeof(int32_t), 0); + }) + .def_property_readonly("taxel_state", + [](py::object self) + { + const auto* values = self.cast&>()->taxel_state(); + return vector_view( + self, vector_data(values), vector_size(values), sizeof(uint8_t), 0); + }) + .def_property_readonly("sequence", + [](const Serialized& self) -> std::optional + { + const auto sequence = self->sequence(); + return sequence ? std::optional(*sequence) : std::nullopt; + }) + .def("__repr__", + [](const Serialized& self) + { + return "TactileData(normal_force=" + std::to_string(vector_size(self->normal_force())) + + ", shear_force=" + std::to_string(vector_size(self->shear_force())) + + ", raw=" + std::to_string(vector_size(self->raw())) + + ", taxel_state=" + std::to_string(vector_size(self->taxel_state())) + ")"; + }); + + bind_record(m, "TactileDataRecord", "TactileData"); + + m.def( + "validate_tactile_metadata", + [](const Serialized& metadata) { return validate_tactile_metadata(*metadata); }, + py::arg("metadata"), "Check a device description. Returns \"\" when valid, otherwise the first rule it breaks."); + m.def( + "validate_tactile_data", + [](const Serialized& data, const Serialized& metadata) + { + const std::string error = validate_tactile_metadata(*metadata); + return error.empty() ? validate_tactile_data(*data, *metadata) : "metadata: " + error; + }, + py::arg("data"), py::arg("metadata"), + "Check that a frame fills exactly the channels `metadata` declares, one entry per taxel. Returns \"\" " + "when valid, otherwise what is wrong."); +} + +} // namespace core diff --git a/src/python/isaaccapture/schema/__init__.py b/src/python/isaaccapture/schema/__init__.py index 4823d57e72..31e5fe9d71 100644 --- a/src/python/isaaccapture/schema/__init__.py +++ b/src/python/isaaccapture/schema/__init__.py @@ -107,6 +107,17 @@ skeleton_profiles, find_skeleton_profile, validate_skeleton_binding, + # Tactile types (device description as tracker metadata, per-frame taxel readings). + TaxelState, + TactileDevice, + TactileChannels, + TactileExtension, + TactileShape, + TactileMetadata, + TactileData, + TactileDataRecord, + validate_tactile_metadata, + validate_tactile_data, # Shared SOMA types. SomaHandSide, # SOMA body joint-rotation types. @@ -258,6 +269,17 @@ def __getattr__(name: str): "validate_skeleton_binding", "asset_digest", "check_skeleton_asset", + # Tactile types. + "TaxelState", + "TactileDevice", + "TactileChannels", + "TactileExtension", + "TactileShape", + "TactileMetadata", + "TactileData", + "TactileDataRecord", + "validate_tactile_metadata", + "validate_tactile_data", # Shared SOMA types. "SomaHandSide", # SOMA body joint-rotation types. diff --git a/tests/cpp/core/schema/CMakeLists.txt b/tests/cpp/core/schema/CMakeLists.txt index 5bdee9fc7b..c6c67a3809 100644 --- a/tests/cpp/core/schema/CMakeLists.txt +++ b/tests/cpp/core/schema/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(schema_tests test_pedals.cpp test_skeleton_binding.cpp test_steering_wheel.cpp + test_tactile.cpp test_spacemouse.cpp test_gamepad.cpp test_plugin_device_status.cpp diff --git a/tests/cpp/core/schema/test_tactile.cpp b/tests/cpp/core/schema/test_tactile.cpp new file mode 100644 index 0000000000..30a5e917da --- /dev/null +++ b/tests/cpp/core/schema/test_tactile.cpp @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit tests for the TactileMetadata and TactileData schemas and their validation. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// Structs are baked into recorded layouts; see src/core/schema/README.md. +static_assert(sizeof(core::TactileSegment) == 28); +static_assert(sizeof(core::TactileTaxel) == 52); +static_assert(sizeof(core::TactileShear) == 8); + +namespace +{ + +core::Pose identity_at(float x, float y, float z) +{ + return core::Pose(core::Point(x, y, z), core::Quaternion(0, 0, 0, 1)); +} + +core::TactileTaxel taxel(const core::Pose& pose, std::array segments, std::array weights) +{ + return core::TactileTaxel(pose, flatbuffers::span(segments.data(), 4), + flatbuffers::span(weights.data(), 4)); +} + +// A left glove: wrist and index tip, a taxel on the tip and one skinned across both. +core::TactileMetadataT make_metadata() +{ + core::TactileMetadataT native; + native.device = std::make_shared(); + native.device->vendor = "Example"; + native.device->side = core::BodySide_LEFT; + native.device->nominal_rate_hz = 100.0f; + + native.shape = std::make_shared(); + native.shape->segments = { core::TactileSegment(identity_at(0, 0, 0)), + core::TactileSegment(identity_at(0, 0.15f, 0)) }; + native.shape->taxels = { taxel(identity_at(0, 0.16f, 0.01f), { 1, 0, 0, 0 }, { 1, 0, 0, 0 }), + taxel(identity_at(0, 0.08f, 0.01f), { 0, 1, 0, 0 }, { 0.5f, 0.5f, 0, 0 }) }; + native.shape->taxel_areas = { 1e-5f, 1e-5f }; + + native.binding = std::make_shared(); + native.binding->profile = std::make_shared(); + native.binding->profile->id = "openxr.hand"; + native.binding->profile->version = 1; + native.binding->segment_names = { "WRIST", "INDEX_TIP" }; + + native.channels = std::make_shared(); + native.channels->normal_force = true; + native.channels->shear_force = true; + native.channels->taxel_state = true; + return native; +} + +std::string validate(const core::TactileMetadataT& native) +{ + return core::validate_tactile_metadata(*core::pack(native)); +} + +// Error from validating make_metadata() after `edit`. +std::string validate_edited(const std::function& edit) +{ + auto native = make_metadata(); + edit(native); + return validate(native); +} + +core::TactileDataT make_frame() +{ + core::TactileDataT frame; + frame.normal_force = { 1.5f, 0.25f }; + frame.shear_force = { core::TactileShear(0.1f, 0), core::TactileShear(0, -0.1f) }; + frame.taxel_state = { core::TaxelState_VALID, core::TaxelState_SATURATED }; + frame.sequence = 7; + return frame; +} + +std::string validate_frame(const core::TactileDataT& frame, const core::TactileMetadataT& metadata = make_metadata()) +{ + return core::validate_tactile_data( + *core::pack(frame), *core::pack(metadata)); +} + +} // namespace + +TEST_CASE("TactileMetadata round-trips and verifies as its own root", "[unit][tactile]") +{ + const auto metadata = core::pack(make_metadata()); + const auto bytes = metadata.buffer(); + flatbuffers::Verifier verifier(bytes.data(), bytes.size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK(std::string(core::TactileMetadata::GetFullyQualifiedName()) == "core.TactileMetadata"); + + CHECK(metadata->device()->side() == core::BodySide_LEFT); + CHECK(*metadata->device()->nominal_rate_hz() == 100.0f); + const auto* second = metadata->shape()->taxels()->Get(1); + CHECK(second->segments()->Get(1) == 1); + CHECK(second->weights()->Get(0) == 0.5f); + CHECK(metadata->binding()->segment_names()->Get(1)->str() == "INDEX_TIP"); + CHECK(core::validate_tactile_metadata(*metadata).empty()); +} + +TEST_CASE("validate_tactile_metadata rejects malformed shapes", "[unit][tactile]") +{ + using M = core::TactileMetadataT; + CHECK_FALSE(validate_edited([](M& m) { m.shape.reset(); }).empty()); + CHECK_FALSE(validate_edited([](M& m) { m.shape->segments.clear(); }).empty()); + CHECK_FALSE(validate_edited([](M& m) { m.shape->taxels.clear(); }).empty()); + CHECK_FALSE(validate_edited( + [](M& m) { + m.shape->segments[0] = + core::TactileSegment(core::Pose(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 2))); + }) + .empty()); + // Weights that do not sum to 1, a negative weight, an index past the segments, a segment + // weighted twice, and a zero-weight slot naming a segment other than 0. + CHECK_FALSE(validate_edited( + [](M& m) { + m.shape->taxels[1] = taxel(identity_at(0, 0, 0), { 0, 1, 0, 0 }, { 0.5f, 0.4f, 0, 0 }); + }) + .empty()); + CHECK_FALSE(validate_edited( + [](M& m) { + m.shape->taxels[1] = taxel(identity_at(0, 0, 0), { 0, 1, 0, 0 }, { 1.5f, -0.5f, 0, 0 }); + }) + .empty()); + CHECK_FALSE(validate_edited( + [](M& m) { + m.shape->taxels[0] = taxel(identity_at(0, 0, 0), { 2, 0, 0, 0 }, { 1, 0, 0, 0 }); + }) + .empty()); + CHECK_FALSE(validate_edited( + [](M& m) { + m.shape->taxels[1] = taxel(identity_at(0, 0, 0), { 1, 1, 0, 0 }, { 0.5f, 0.5f, 0, 0 }); + }) + .empty()); + CHECK_FALSE(validate_edited( + [](M& m) { + m.shape->taxels[0] = taxel(identity_at(0, 0, 0), { 1, 1, 0, 0 }, { 1, 0, 0, 0 }); + }) + .empty()); + CHECK_FALSE(validate_edited([](M& m) { m.shape->taxel_areas = { 1e-5f }; }).empty()); + CHECK_FALSE(validate_edited([](M& m) { m.shape->taxel_areas = { 1e-5f, 0.0f }; }).empty()); +} + +TEST_CASE("validate_tactile_metadata checks channels, binding and extensions", "[unit][tactile]") +{ + using M = core::TactileMetadataT; + CHECK_FALSE(validate_edited([](M& m) { m.channels.reset(); }).empty()); + CHECK_FALSE(validate_edited([](M& m) { m.channels = std::make_shared(); }).empty()); + CHECK(validate_edited([](M& m) { m.binding.reset(); }).empty()); + CHECK(validate_edited([](M& m) { m.binding->segment_names = { "WRIST" }; }).find("binding:") == 0); + CHECK(validate_edited([](M& m) { m.device->side = core::BodySide_UNSPECIFIED; }).find("side") != std::string::npos); + + const auto extension = [](const std::string& id) + { + auto native = std::make_shared(); + native->id = id; + native->version = 1; + return native; + }; + CHECK(validate_edited([&](M& m) { m.extensions.push_back(extension("com.example.glove")); }).empty()); + CHECK_FALSE(validate_edited([&](M& m) { m.extensions.push_back(extension("glove")); }).empty()); + CHECK_FALSE(validate_edited( + [&](M& m) + { + m.extensions.push_back(extension("com.example.glove")); + m.extensions.push_back(extension("com.example.glove")); + }) + .empty()); +} + +TEST_CASE("validate_tactile_data matches the declared channels to the taxel count", "[unit][tactile]") +{ + CHECK(validate_frame(make_frame()).empty()); + + auto missing = make_frame(); + missing.normal_force.clear(); + CHECK(validate_frame(missing).find("normal_force") == 0); + + auto short_shear = make_frame(); + short_shear.shear_force.pop_back(); + CHECK(validate_frame(short_shear).find("shear_force") == 0); + + auto undeclared = make_frame(); + undeclared.raw = { 1, 2 }; + CHECK(validate_frame(undeclared).find("raw") == 0); + + auto with_raw = make_metadata(); + with_raw.channels->raw_values_per_taxel = 3; + auto raw = make_frame(); + raw.raw = { 1, 2, 3, 4, 5, 6 }; + CHECK(validate_frame(raw, with_raw).empty()); + raw.raw.pop_back(); + CHECK(validate_frame(raw, with_raw).find("raw") == 0); +} + +TEST_CASE("TactileData round-trips through its record", "[unit][tactile]") +{ + core::TactileDataRecordT native; + native.data = std::make_shared(make_frame()); + native.timestamp = std::make_shared(1, 2, 3); + const auto record = core::pack(native); + const auto* data = record->data(); + REQUIRE(data != nullptr); + CHECK(data->shear_force()->Get(1)->y() == -0.1f); + CHECK(data->taxel_state()->Get(1) == core::TaxelState_SATURATED); + CHECK(*data->sequence() == 7u); + CHECK(record->timestamp()->sample_time_local_common_clock() == 2); +} diff --git a/tests/python/core/schema/test_tactile.py b/tests/python/core/schema/test_tactile.py new file mode 100644 index 0000000000..b84e410500 --- /dev/null +++ b/tests/python/core/schema/test_tactile.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for TactileMetadata and TactileData in isaaccapture.schema.""" + +import numpy as np +import pytest + +from isaaccapture.schema import ( + BodySide, + DeviceDataTimestamp, + SkeletonBinding, + SkeletonProfileRef, + TactileChannels, + TactileData, + TactileDataRecord, + TactileDevice, + TactileExtension, + TactileMetadata, + TactileShape, + TaxelState, + validate_tactile_data, + validate_tactile_metadata, +) + +IDENTITY = [0.0, 0.0, 0.0, 1.0] + + +def make_shape(taxel_segments=((1, 0, 0, 0), (0, 1, 0, 0)), **overrides): + """Wrist and index tip; one taxel on the tip, one skinned across both.""" + arguments = dict( + segment_bind_positions=[[0, 0, 0], [0, 0.15, 0]], + segment_bind_orientations=[IDENTITY, IDENTITY], + taxel_positions=[[0, 0.16, 0.01], [0, 0.08, 0.01]], + taxel_orientations=[IDENTITY, IDENTITY], + taxel_segments=taxel_segments, + taxel_weights=[[1, 0, 0, 0], [0.5, 0.5, 0, 0]], + taxel_areas=[1e-5, 2e-5], + ) + arguments.update(overrides) + return TactileShape(**arguments) + + +def make_metadata(**overrides): + arguments = dict( + shape=make_shape(), + channels=TactileChannels(normal_force=True, shear_force=True), + device=TactileDevice( + vendor="Example", side=BodySide.LEFT, nominal_rate_hz=100.0 + ), + binding=SkeletonBinding( + ["WRIST", "INDEX_TIP"], profile=SkeletonProfileRef("openxr.hand", 1) + ), + extensions=[TactileExtension("com.example.glove", 1, b"\x01\x02")], + ) + arguments.update(overrides) + return TactileMetadata(**arguments) + + +def test_shape_reads_back_as_views(): + shape = make_shape() + + assert shape.segment_count == 2 + assert shape.taxel_count == 2 + np.testing.assert_allclose(shape.segment_bind_positions[1], [0, 0.15, 0]) + np.testing.assert_allclose(shape.taxel_orientations[0], IDENTITY) + assert shape.taxel_segments.dtype == np.uint16 + assert shape.taxel_segments.tolist() == [[1, 0, 0, 0], [0, 1, 0, 0]] + np.testing.assert_allclose(shape.taxel_weights[1], [0.5, 0.5, 0, 0]) + np.testing.assert_allclose(shape.taxel_areas, [1e-5, 2e-5]) + + +def test_shape_rejects_mismatched_or_out_of_range_input(): + with pytest.raises(ValueError, match="one row per taxel"): + make_shape(taxel_weights=[[1, 0, 0, 0]]) + with pytest.raises(ValueError, match="taxel_segments"): + make_shape(taxel_segments=[[70000, 0, 0, 0], [0, 1, 0, 0]]) + with pytest.raises(ValueError, match="shape"): + make_shape(taxel_positions=[0, 0.16, 0.01]) + + +def test_metadata_round_trips(): + metadata = make_metadata() + + assert metadata.device.vendor == "Example" + assert metadata.device.side == BodySide.LEFT + assert metadata.device.nominal_rate_hz == pytest.approx(100.0) + assert TactileDevice().nominal_rate_hz is None + assert metadata.shape.taxel_count == 2 + assert metadata.binding.segment_names == ["WRIST", "INDEX_TIP"] + assert metadata.channels.shear_force + assert metadata.extensions[0].data == b"\x01\x02" + assert validate_tactile_metadata(metadata) == "" + + +def test_metadata_constructor_validates(): + with pytest.raises(ValueError, match="sum to"): + make_metadata(shape=make_shape(taxel_weights=[[1, 0, 0, 0], [0.5, 0.4, 0, 0]])) + with pytest.raises(ValueError, match="past the 2 segments"): + make_metadata(shape=make_shape(taxel_segments=[[2, 0, 0, 0], [0, 1, 0, 0]])) + with pytest.raises(ValueError, match=r"binding: .*side"): + make_metadata(device=TactileDevice(side=BodySide.UNSPECIFIED)) + with pytest.raises(ValueError, match="no force"): + make_metadata(channels=TactileChannels(taxel_state=True)) + + +def test_data_reads_back_as_views(): + data = TactileData( + normal_force=[1.5, 0.25], + shear_force=[[0.1, 0.0], [0.0, -0.1]], + taxel_state=[TaxelState.VALID, int(TaxelState.SATURATED)], + sequence=7, + ) + + np.testing.assert_allclose(data.normal_force, [1.5, 0.25]) + assert data.shear_force.shape == (2, 2) + np.testing.assert_allclose(data.shear_force[1], [0.0, -0.1]) + assert data.taxel_state.tolist() == [0, 1] + assert data.raw is None + assert data.sequence == 7 + assert TactileData().sequence is None + + +def test_data_matches_declared_channels(): + metadata = make_metadata() + + full = TactileData(normal_force=[1.0, 2.0], shear_force=np.zeros((2, 2))) + assert validate_tactile_data(full, metadata) == "" + assert "shear_force" in validate_tactile_data( + TactileData(normal_force=[1.0, 2.0]), metadata + ) + assert "normal_force has 1" in validate_tactile_data( + TactileData(normal_force=[1.0], shear_force=np.zeros((2, 2))), metadata + ) + with pytest.raises(ValueError, match="taxel_state"): + TactileData(taxel_state=[4]) + + +def test_record_wraps_data(): + record = TactileDataRecord( + TactileData(normal_force=[1.0, 2.0]), DeviceDataTimestamp(1, 2, 3) + ) + + np.testing.assert_allclose(record.data.normal_force, [1.0, 2.0]) + assert record.timestamp.sample_time_local_common_clock == 2 From 5457700ad2aae7a6eb443dd7085efc4649f70fe8 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 20 Oct 2026 15:58:58 -0700 Subject: [PATCH 3/3] Add a tactile feeder and printer example The feeder publishes a mock left index fingertip pad: a TactileMetadata with two segments bound to the OpenXR hand joints INDEX_DISTAL and INDEX_TIP, and a TactileData frame at --rate. The printer reads them through a TactileTracker on a DeviceIOSession, prints the metadata once and the first and last taxel of the latest frame once per second. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lotus Li --- examples/CMakeLists.txt | 1 + examples/tactile/CMakeLists.txt | 7 + examples/tactile/README.md | 30 +++ examples/tactile/pyproject.toml | 23 +++ .../isaaccapture_examples/tactile/__init__.py | 2 + .../isaaccapture_examples/tactile/common.py | 9 + .../isaaccapture_examples/tactile/feeder.py | 180 ++++++++++++++++++ .../isaaccapture_examples/tactile/printer.py | 117 ++++++++++++ 8 files changed, 369 insertions(+) create mode 100644 examples/tactile/CMakeLists.txt create mode 100644 examples/tactile/README.md create mode 100644 examples/tactile/pyproject.toml create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/__init__.py create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/common.py create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/feeder.py create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/printer.py diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9b96fea7f9..b1855c9a1d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -19,6 +19,7 @@ if(BUILD_EXAMPLES) add_subdirectory(deviceio_live_view) add_subdirectory(soma_body_publisher) add_subdirectory(soma_hand_publisher) + add_subdirectory(tactile) add_subdirectory(haptic_feedback) add_subdirectory(sharpa_avatar) add_subdirectory(vehicle_teleop) diff --git a/examples/tactile/CMakeLists.txt b/examples/tactile/CMakeLists.txt new file mode 100644 index 0000000000..e3caba6501 --- /dev/null +++ b/examples/tactile/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +include(${CMAKE_SOURCE_DIR}/cmake/InstallPythonExample.cmake) +install_python_example(DESTINATION examples/tactile) diff --git a/examples/tactile/README.md b/examples/tactile/README.md new file mode 100644 index 0000000000..27c62a7d41 --- /dev/null +++ b/examples/tactile/README.md @@ -0,0 +1,30 @@ + + +# Tactile feeder and printer + +1. In the Python environment that has Isaac Capture installed, install the example from the + repository root: + + ```bash + uv pip install -e ./examples/tactile + ``` + +2. Start the printer. It starts the CloudXR runtime if none is running: + + ```bash + python -m isaaccapture_examples.tactile.printer --accept-eula + ``` + +3. In a second terminal with the same environment, start the feeder: + + ```bash + python -m isaaccapture_examples.tactile.feeder + ``` + +4. In the printer terminal, check that a `TactileMetadata` block is printed once, followed by a + `TactileData` block with the values of taxel 0 and taxel 15 every second. + +5. Press `Ctrl+C` in the feeder terminal, then in the printer terminal. diff --git a/examples/tactile/pyproject.toml b/examples/tactile/pyproject.toml new file mode 100644 index 0000000000..ef0c445997 --- /dev/null +++ b/examples/tactile/pyproject.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "isaaccapture-examples-tactile" +version = "0.0.0" +description = "Isaac Capture tactile feeder and printer" +requires-python = ">=3.11,<3.14" +dependencies = [ + "isaaccapture[cloudxr]", + "numpy>=1.23.0", +] + +[tool.hatch.build.targets.wheel] +only-include = ["python/isaaccapture_examples/tactile"] +sources = ["python"] + +[tool.uv.sources] +isaaccapture = { path = "../..", editable = true } diff --git a/examples/tactile/python/isaaccapture_examples/tactile/__init__.py b/examples/tactile/python/isaaccapture_examples/tactile/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/tactile/python/isaaccapture_examples/tactile/common.py b/examples/tactile/python/isaaccapture_examples/tactile/common.py new file mode 100644 index 0000000000..ece22730f5 --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/common.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Values the feeder and the printer must agree on.""" + +DEFAULT_COLLECTION_ID = "tactile" + +# TactileTracker's default max_flatbuffer_size (trackers.toml); a larger collection is rejected. +TACTILE_MAX_PAYLOAD_SIZE = 32768 diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py new file mode 100644 index 0000000000..f8d62643dc --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Publish mock tactile frames and their TactileMetadata, as a tactile device would. + +The device is a left index fingertip pad: two segments bound to the OpenXR hand joints +INDEX_DISTAL and INDEX_TIP, with taxels in a line along the pad. Each frame pushes one +TactileData on the data collection; the TactileMetadata goes on "/metadata" +when the feeder starts and every ``--metadata-interval`` seconds after that. +""" + +import argparse +import logging +import math +import sys +import time +from contextlib import ExitStack + +import numpy as np + +from isaaccapture.cloudxr import CloudXRLauncher +from isaaccapture.deviceio_session import DeviceIOSession +from isaaccapture.deviceio_trackers import TensorPushTracker +from isaaccapture.oxr import OpenXRSession +from isaaccapture.schema import ( + TRACKER_METADATA_MAX_SIZE, + TRACKER_METADATA_TENSOR_IDENTIFIER, + BodySide, + SkeletonBinding, + SkeletonProfileRef, + TactileChannels, + TactileData, + TactileDevice, + TactileMetadata, + TactileShape, + TaxelState, + tracker_metadata_collection_id, +) + +from .common import DEFAULT_COLLECTION_ID, TACTILE_MAX_PAYLOAD_SIZE + +logger = logging.getLogger("isaaccapture.examples.tactile.feeder") + +PAD_LENGTH_M = 0.025 +DISTAL_TO_TIP_M = 0.022 +# 180 degrees about X: the taxel +Z normal faces the palm side (-Z) of the finger. +PAD_ORIENTATION = (1.0, 0.0, 0.0, 0.0) + + +def mock_metadata(num_taxels: int, rate_hz: float) -> TactileMetadata: + """Taxel i sits a fraction f = i / (N - 1) along the pad, skinned (1 - f, f) to the two segments.""" + along = np.linspace(0.0, 1.0, num_taxels, dtype=np.float32) + taxel_positions = np.stack( + [np.zeros_like(along), along * PAD_LENGTH_M, np.full_like(along, -0.008)], + axis=1, + ) + weights = np.zeros((num_taxels, 4), np.float32) + weights[:, 0] = 1.0 - along + weights[:, 1] = along + segments = np.where(weights > 0.0, [0, 1, 0, 0], 0) + + shape = TactileShape( + segment_bind_positions=[[0.0, 0.0, 0.0], [0.0, DISTAL_TO_TIP_M, 0.0]], + segment_bind_orientations=[[0.0, 0.0, 0.0, 1.0]] * 2, + taxel_positions=taxel_positions, + taxel_orientations=np.tile(PAD_ORIENTATION, (num_taxels, 1)), + taxel_segments=segments, + taxel_weights=weights, + taxel_areas=np.full(num_taxels, 4e-6, np.float32), + ) + return TactileMetadata( + shape=shape, + channels=TactileChannels(normal_force=True, shear_force=True, taxel_state=True), + device=TactileDevice( + vendor="Example", + model="Mock fingertip pad", + serial_number="0001", + side=BodySide.LEFT, + nominal_rate_hz=rate_hz, + ), + binding=SkeletonBinding( + ["INDEX_DISTAL", "INDEX_TIP"], profile=SkeletonProfileRef("openxr.hand", 1) + ), + ) + + +def mock_frame(num_taxels: int, t: float, sequence: int) -> TactileData: + """A 0.5 Hz press that travels along the pad, with a small shear along the pad.""" + along = np.linspace(0.0, 1.0, num_taxels) + normal = 1.0 + np.sin(2.0 * math.pi * (0.5 * t - 0.25 * along)) + shear = np.stack([np.zeros(num_taxels), 0.1 * normal], axis=1) + state = np.where(normal > 1.9, TaxelState.SATURATED.value, TaxelState.VALID.value) + return TactileData( + normal_force=normal, + shear_force=shear, + taxel_state=state, + sequence=sequence, + ) + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--collection-id", default=DEFAULT_COLLECTION_ID) + parser.add_argument("--taxels", type=int, default=16) + parser.add_argument("--rate", type=float, default=60.0, help="Frames per second") + parser.add_argument( + "--metadata-interval", + type=float, + default=1.0, + help="Seconds between pushes of the TactileMetadata", + ) + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args(argv[1:]) + if args.taxels < 2: + parser.error("--taxels must be at least 2") + if not math.isfinite(args.rate) or args.rate <= 0: + parser.error("--rate must be finite and positive") + if not math.isfinite(args.metadata_interval) or args.metadata_interval <= 0: + parser.error("--metadata-interval must be finite and positive") + + metadata = mock_metadata(args.taxels, args.rate).to_bytes() + frame_size = len(mock_frame(args.taxels, 0.0, 0).to_bytes()) + if ( + len(metadata) > TRACKER_METADATA_MAX_SIZE + or frame_size > TACTILE_MAX_PAYLOAD_SIZE + ): + parser.error(f"--taxels {args.taxels} does not fit the tactile tracker") + logger.info( + "Publishing %d taxels on '%s' at %.0f Hz", + args.taxels, + args.collection_id, + args.rate, + ) + + frames = 0 + with ExitStack() as stack: + stack.enter_context(CloudXRLauncher.launch_context(args)) + data_tracker = TensorPushTracker( + args.collection_id, "tactile", TACTILE_MAX_PAYLOAD_SIZE + ) + metadata_tracker = TensorPushTracker( + tracker_metadata_collection_id(args.collection_id), + TRACKER_METADATA_TENSOR_IDENTIFIER, + len(metadata), + ) + trackers = [data_tracker, metadata_tracker] + oxr_session = stack.enter_context( + OpenXRSession( + "TactileFeeder", DeviceIOSession.get_required_extensions(trackers) + ) + ) + session = stack.enter_context( + DeviceIOSession.run(trackers, oxr_session.get_handles()) + ) + + start = time.monotonic() + last_metadata_push = -math.inf + try: + while True: + remaining = start + frames / args.rate - time.monotonic() + if remaining > 0: + time.sleep(remaining) + session.update() + t = time.monotonic() - start + if t - last_metadata_push >= args.metadata_interval: + metadata_tracker.push(session, metadata) + last_metadata_push = t + data_tracker.push( + session, mock_frame(args.taxels, t, frames).to_bytes() + ) + frames += 1 + except KeyboardInterrupt: + pass + + logger.info("Published %d frames", frames) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/printer.py b/examples/tactile/python/isaaccapture_examples/tactile/printer.py new file mode 100644 index 0000000000..a9100121ef --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/printer.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Print what a TactileTracker receives from ``python -m isaaccapture_examples.tactile.feeder``. + +Prints the TactileMetadata once it arrives, then the first and last taxel of the latest +TactileData once per second. +""" + +import argparse +import sys +import time + +from isaaccapture.cloudxr import CloudXRLauncher +from isaaccapture.deviceio_session import DeviceIOSession +from isaaccapture.deviceio_trackers import TactileTracker +from isaaccapture.oxr import OpenXRSession +from isaaccapture.schema import ( + TactileData, + TactileMetadata, + TaxelState, + validate_tactile_data, + validate_tactile_metadata, +) + +from .common import DEFAULT_COLLECTION_ID + +PRINT_INTERVAL_S = 1.0 + + +def print_metadata(metadata: TactileMetadata) -> None: + device, shape, binding, channels = ( + metadata.device, + metadata.shape, + metadata.binding, + metadata.channels, + ) + print("TactileMetadata") + if device is not None: + print( + f" device: {device.vendor} {device.model} (serial {device.serial_number}), " + f"side {device.side.name}, {device.nominal_rate_hz} Hz" + ) + print(f" shape: {shape.segment_count} segments, {shape.taxel_count} taxels") + if binding is not None: + profile = binding.profile + print( + f" binding: {profile.id if profile else 'generic'} " + f"v{profile.version if profile else '-'}: {', '.join(binding.segment_names)}" + ) + print( + f" channels: normal_force={channels.normal_force}, shear_force={channels.shear_force}, " + f"raw_values_per_taxel={channels.raw_values_per_taxel}, taxel_state={channels.taxel_state}" + ) + print(f" valid: {validate_tactile_metadata(metadata) or 'yes'}") + + +def print_taxel(data: TactileData, index: int, raw_per_taxel: int) -> None: + values = [] + if data.normal_force is not None: + values.append(f"normal {data.normal_force[index]:6.3f} N") + if data.shear_force is not None: + x, y = data.shear_force[index] + values.append(f"shear ({x:6.3f}, {y:6.3f}) N") + if data.raw is not None: + raw = data.raw[index * raw_per_taxel : (index + 1) * raw_per_taxel] + values.append(f"raw {raw.tolist()}") + if data.taxel_state is not None: + values.append(TaxelState(int(data.taxel_state[index])).name) + print(f" taxel {index}: {', '.join(values)}") + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--collection-id", default=DEFAULT_COLLECTION_ID) + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args(argv[1:]) + + tracker = TactileTracker(args.collection_id) + trackers = [tracker] + try: + with ( + CloudXRLauncher.launch_context(args), + OpenXRSession( + "TactilePrinter", DeviceIOSession.get_required_extensions(trackers) + ) as oxr_session, + DeviceIOSession.run(trackers, oxr_session.get_handles()) as session, + ): + print(f"Waiting for tactile data on '{args.collection_id}'. Ctrl+C stops.") + metadata = None + last_print = -PRINT_INTERVAL_S + while True: + session.update() + if metadata is None: + metadata = tracker.get_metadata(session) + if metadata is not None: + print_metadata(metadata) + data = tracker.get_data(session) + now = time.monotonic() + # The tracker withholds data until the producer's metadata has arrived. + if data is not None and now - last_print >= PRINT_INTERVAL_S: + raw_per_taxel = metadata.channels.raw_values_per_taxel + print(f"TactileData sequence {data.sequence}") + print_taxel(data, 0, raw_per_taxel) + print_taxel(data, metadata.shape.taxel_count - 1, raw_per_taxel) + error = validate_tactile_data(data, metadata) + if error: + print(f" invalid: {error}") + last_print = now + time.sleep(0.005) + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))