From 3e57a955a70b6d9b0e433b61844c42528ae9e5c0 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 15 Oct 2026 16:58:33 -0700 Subject: [PATCH 01/45] Add tactile schemas, trackers, bindings and sources Signed-off-by: lotusl Co-Authored-By: Claude Opus 5.5 --- src/core/AGENTS.md | 1 + src/core/deviceio_trackers/trackers.toml | 8 + src/core/schema/fbs/tactile.fbs | 23 ++ src/core/schema/fbs/tactile_shape.fbs | 46 ++++ src/core/schema/golden/tactile.bfbs | Bin 0 -> 1112 bytes src/core/schema/golden/tactile_shape.bfbs | Bin 0 -> 1664 bytes src/core/schema/python/CMakeLists.txt | 2 + src/core/schema/python/schema_array_views.h | 36 ++- src/core/schema/python/schema_module.cpp | 6 + src/core/schema/python/tactile_bindings.h | 59 +++++ .../schema/python/tactile_shape_bindings.h | 218 ++++++++++++++++++ .../deviceio_source_nodes/__init__.py | 16 ++ .../deviceio_tensor_types.py | 30 +++ .../deviceio_source_nodes/tactile_source.py | 177 ++++++++++++++ src/python/isaaccapture/schema/__init__.py | 8 + .../test_replay_session.cpp | 95 ++++++++ tests/cpp/core/schema/CMakeLists.txt | 2 + tests/cpp/core/schema/test_tactile.cpp | 70 ++++++ tests/cpp/core/schema/test_tactile_shape.cpp | 77 +++++++ .../test_tactile_sources.py | 146 ++++++++++++ tests/python/core/schema/test_tactile.py | 51 ++++ .../python/core/schema/test_tactile_shape.py | 80 +++++++ 22 files changed, 1150 insertions(+), 1 deletion(-) create mode 100644 src/core/schema/fbs/tactile.fbs create mode 100644 src/core/schema/fbs/tactile_shape.fbs create mode 100644 src/core/schema/golden/tactile.bfbs create mode 100644 src/core/schema/golden/tactile_shape.bfbs create mode 100644 src/core/schema/python/tactile_bindings.h create mode 100644 src/core/schema/python/tactile_shape_bindings.h create mode 100644 src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py create mode 100644 tests/cpp/core/schema/test_tactile.cpp create mode 100644 tests/cpp/core/schema/test_tactile_shape.cpp create mode 100644 tests/python/core/retargeting_engine/test_tactile_sources.py create mode 100644 tests/python/core/schema/test_tactile.py create mode 100644 tests/python/core/schema/test_tactile_shape.py diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index b758b2bcfc..a139c46a8d 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -18,6 +18,7 @@ If work under **`src/core/`** went wrong—**user** correction, **pre-commit/CI* - When changing `TeleopSession` retargeting execution defaults, update config, docs, and default-behavior tests together so opt-in vs. default semantics stay aligned. - Preserve existing `TeleopSession` lifecycle flag semantics unless changing the public/context-manager contract intentionally; use tests to lock down cleanup details before altering them. - After Python test or session-manager edits, let `ruff format`/pre-commit own wrapping and rerun the hook when it modifies files. +- Schema NumPy properties are stubbed as `numpy.ndarray[numpy.float32]`, so mypy (`retargeting_mypy_type_check`) rejects `.shape[i]` on them; use `len()` or `np.asarray()` first. - **`IDeviceIOSource` leaves are only discovered when reachable from a declared graph root** — the `pipeline` combiner's outputs, the `teleop_control_pipeline`, or a subgraph registered in `TeleopSessionConfig(sinks=[...])`. `TeleopSession._discover_sources` unions the leaves of all three. An input *source* whose only purpose is a side effect (e.g. message-channel send) must therefore expose at least one output (a heartbeat boolean is the established pattern) **and** be reachable from one of those roots; silent no-discovery is the recurring footgun. **Output `IDeviceIOSink` nodes are different:** they are registered via `TeleopSessionConfig(sinks=[...])` and the session runs and flushes them explicitly each frame, so a sink needs no heartbeat output and no `OutputCombiner` reachability. - **A plugin a wheel install must be able to start is bundled into `isaaccapture/plugins//`** (binary + `plugin.yaml`, via a POST_BUILD copy and `add_dependencies(python_package ...)`), and its binary must not need shared libraries the wheel does not ship. Apps start it explicitly with `PluginConfig(search_paths=[BUNDLED_PLUGINS_DIR])`; sources never start plugins. See `src/plugins/spacemouse`. - **All C++ diagnostics go through `isaaccapture::Logger`** (`log_bridge/cpp/inc/log_bridge/logger.hpp`), never `std::cout`/`std::cerr`/`printf`, and the logger name is always `isaaccapture..`. The env-var contract and wire format are shared with the Python half, so a change to either must land in both — see [`log_bridge/AGENTS.md`](log_bridge/AGENTS.md) before touching the logging machinery, and the repo root [`AGENTS.md`](../../AGENTS.md) "Logging" section for the rules that apply to every call site. diff --git a/src/core/deviceio_trackers/trackers.toml b/src/core/deviceio_trackers/trackers.toml index d27518d43c..7f9ed90439 100644 --- a/src/core/deviceio_trackers/trackers.toml +++ b/src/core/deviceio_trackers/trackers.toml @@ -111,3 +111,11 @@ python_accessor = "get_spacemouse_data" name = "gamepad" table = "GamepadSnapshot" python_accessor = "get_gamepad_data" + +# Defaults fit about 1300 taxels (12 B each); larger devices pass max_flatbuffer_size. The +# TactileShape metadata must fit TRACKER_METADATA_MAX_SIZE, which holds 26 joints and 1000 taxels. +[[tracker]] +name = "tactile" +table = "TactileData" +metadata = "TactileShape" +max_flatbuffer_size = 16384 diff --git a/src/core/schema/fbs/tactile.fbs b/src/core/schema/fbs/tactile.fbs new file mode 100644 index 0000000000..c73745c585 --- /dev/null +++ b/src/core/schema/fbs/tactile.fbs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "point.fbs"; +include "timestamp.fbs"; + +namespace core; + +// Per-frame tactile reading. The taxel layout it indexes is TactileShape (tactile_shape.fbs), +// the tactile tracker's metadata. +table TactileData { + // Force vector per taxel, in TactileShape taxel order. + taxel_force: [Point] (id: 0); +} + +// MCAP recording wrapper. Record types are the root types written to MCAP +// channels; trackers serialize into Record types in serialize_all(). +table TactileDataRecord { + data: TactileData (id: 0); + timestamp: DeviceDataTimestamp (id: 1); +} + +root_type TactileDataRecord; diff --git a/src/core/schema/fbs/tactile_shape.fbs b/src/core/schema/fbs/tactile_shape.fbs new file mode 100644 index 0000000000..d4f5fb79d6 --- /dev/null +++ b/src/core/schema/fbs/tactile_shape.fbs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "pose.fbs"; + +namespace core; + +// One joint of the hand skeleton at the reference bind pose. +struct TactileJoint { + // Pose in hand-root space. Position in meters, unit quaternion. + bind_pose: Pose; + + // Parent joint ordinal; the root holds 0x7FFFFFFF (XR_HAND_JOINT_MAX_ENUM_EXT). + // The hierarchy is acyclic. + parent: uint32; +} + +// One taxel at the reference bind pose, in hand-root space. +struct TactileTaxel { + // Position in meters; unit normal; unit tangent perpendicular to the normal. + position: Point; + normal: Point; + tangent: Point; + + // Linear blend skinning over four slots: joint_indices[k], an XrHandJointEXT ordinal (HandJoint + // in hand.fbs), weighted by joint_weights[k]. Every slot names a valid joint, zero-weight slots + // included. Weights are finite and nonnegative, 1-4 of them nonzero, summing to 1. A taxel moves as + // p_current = sum_k w_k * T_current[j_k] * inverse(T_bind[j_k]) * p_bind + // with T_bind from the joint's bind_pose and T_current the current joint-to-hand-root + // transform, both rigid. It blends transformed points, not poses. + joint_indices: [uint32:4]; + joint_weights: [float:4]; +} + +// Hand skeleton and taxel layout at the reference bind pose, for the device that publishes +// TactileData. `joints` is indexed by XrHandJointEXT ordinal; `taxels` is in +// TactileData.taxel_force order. +// +// Published as the metadata of the tactile tracker (`metadata` in trackers.toml), not as +// samples: it carries no timestamp, and the tracker holds one shape for the whole session. +table TactileShape { + joints: [TactileJoint] (id: 0); + taxels: [TactileTaxel] (id: 1); +} + +root_type TactileShape; diff --git a/src/core/schema/golden/tactile.bfbs b/src/core/schema/golden/tactile.bfbs new file mode 100644 index 0000000000000000000000000000000000000000..e476e46c4b90679853988754a53d9e9289546cc7 GIT binary patch literal 1112 zcmZ{jy>1gh6ou~^V>`rP7Z#$hkdacPKruy%6jV`!C@2u5pj%DW8)d~h8`-N65Jet^ zl9GZ4px^;grl6qUAyOdBcV;ujtYB8hGxzV@pBWF$Z1>6T%YluoX&viY-v)LM?U3DU z8$1LfAo(10z$5U#)x>ZanQh!4mi-wrN6#f@*j6o4DN!Yn5A3$E-AA--E*aTk)oMY9jU*L-NI{iwdvNcD6*FC zUimJ_#nzKWxz<3LP1AYlrYD?_WRIG*$yu}M^R?MEA63}*yY*C`;>9_k9;pX>ns?QS zG@ARLSqJ~y=*Hu+YG&*3V2&)?4iJC6S5)+(9yOts#Fcfk>a$jJi>m*w`A$zc#IO!Z zcb4YyVKGZm`})hzBvSqjptx|S8WgpL3HfSu@1H>q8P`82#M1Ng$zdM1~9*JY-;q>JAw)bcm`7MW_R)9oSMgHLX#wL*l9dF)()M z*oWu?FfcIi0DS-k1|Fd!LVz@<&NH4i3-#NdQ~t_i~BFvZRRW=-8i^v~AUpo~j3PP5gNo|-O_;nXrM+Y|YSc84wC6b|zV}O3|fSYML7LVW$t~oX$U2&wG2}C&4sAPj%*+)5g1N;1t*e7(mtF z#fmQXsAmC{Nc;3f*SC_^{}Oi^h`#}~Dd$Iy^Bx@b(Sov+V{6STKkvZlz?Qym=ibj5 z2bJf)a0%IwCzyD>iq*x*1(^2D8D0SHjmq!2Iiq> mg5Pve#mVOMEyrh0)XV=__KfvrTn9GxS3mkc0sB=gsPQk*KJ=mh literal 0 HcmV?d00001 diff --git a/src/core/schema/python/CMakeLists.txt b/src/core/schema/python/CMakeLists.txt index b34cac9c90..c37e632705 100644 --- a/src/core/schema/python/CMakeLists.txt +++ b/src/core/schema/python/CMakeLists.txt @@ -28,6 +28,8 @@ pybind11_add_module(schema_py schema_module.cpp spacemouse_bindings.h steering_wheel_bindings.h + tactile_bindings.h + tactile_shape_bindings.h tracker_metadata_bindings.h ) diff --git a/src/core/schema/python/schema_array_views.h b/src/core/schema/python/schema_array_views.h index 07e376ba1c..2d5ec5c130 100644 --- a/src/core/schema/python/schema_array_views.h +++ b/src/core/schema/python/schema_array_views.h @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Helpers for exposing fixed-size FlatBuffers struct arrays to Python as NumPy views. +// Helpers for exposing FlatBuffers struct arrays and vectors to Python as NumPy views. // // A FlatBuffers struct array is interleaved (array of structs), so a field is exposed as a // strided view rather than a packed buffer; NumPy and DLPack both carry strides natively, @@ -27,6 +27,7 @@ #include #include #include +#include namespace py = pybind11; @@ -115,4 +116,37 @@ inline py::array_t strided_field_view( return py::array_t({ count }, { stride }, first, owner); } +// (count, Width) view over a FlatBuffers vector of structs that are exactly Width packed T +// (Point as 3 floats, say). FlatBuffers omits an empty vector, so absent reads as (0, Width). +template +inline py::array_t struct_vector_view(py::object owner, const ::flatbuffers::Vector* values) +{ + static_assert(sizeof(StructT) == sizeof(T) * Width, "struct must be exactly Width packed elements"); + if (values == nullptr || values->size() == 0) + { + return py::array_t(std::vector{ 0, Width }); + } + return strided_field_view(owner, reinterpret_cast(values->Data()), + static_cast(sizeof(StructT)), values->size(), Width); +} + +// Rows of a (count, Width) array as structs that are exactly Width packed T, for a table +// constructor. Raises ValueError naming `field` on any other shape. +template +inline std::vector structs_from_rows(const py::array_t& rows, + const char* field) +{ + static_assert(sizeof(StructT) == sizeof(T) * Width, "struct must be exactly Width packed elements"); + if (rows.ndim() != 2 || rows.shape(1) != Width) + { + throw py::value_error(std::string(field) + ": expected an (N, " + std::to_string(Width) + ") array"); + } + std::vector out(static_cast(rows.shape(0))); + if (!out.empty()) + { + std::memcpy(static_cast(out.data()), rows.data(), out.size() * sizeof(StructT)); + } + return out; +} + } // namespace core diff --git a/src/core/schema/python/schema_module.cpp b/src/core/schema/python/schema_module.cpp index 7d7a950be7..d904d55833 100644 --- a/src/core/schema/python/schema_module.cpp +++ b/src/core/schema/python/schema_module.cpp @@ -29,6 +29,8 @@ #include "soma_hand_joint_rotations_bindings.h" #include "spacemouse_bindings.h" #include "steering_wheel_bindings.h" +#include "tactile_bindings.h" +#include "tactile_shape_bindings.h" #include "timestamp_bindings.h" #include "tracker_metadata_bindings.h" @@ -62,6 +64,10 @@ PYBIND11_MODULE(_schema, m) // Bind OGLO tactile glove types (OgloGloveSample table). core::bind_oglo_tactile(m); + // Bind tactile types (TactileData, TactileShape tables). + core::bind_tactile(m); + core::bind_tactile_shape(m); + // Bind the helpers for producers of tracker metadata. core::bind_tracker_metadata(m); diff --git a/src/core/schema/python/tactile_bindings.h b/src/core/schema/python/tactile_bindings.h new file mode 100644 index 0000000000..1d83b19ee8 --- /dev/null +++ b/src/core/schema/python/tactile_bindings.h @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Python bindings for the TactileData FlatBuffer schema (see tactile.fbs). +// Types: TactileData (table) and TactileDataRecord, exposed as encoded views. + +#pragma once + +#include "schema_array_views.h" +#include "schema_serialized.h" + +#include +#include +#include +#include + +#include +#include + +namespace py = pybind11; + +namespace core +{ + +inline void bind_tactile(py::module& m) +{ + using FloatRows = py::array_t; + + serialized_class(m, "TactileData", "Encoded per-frame tactile reading.") + .def(py::init( + [](const std::optional& taxel_force) + { + TactileDataT native; + if (taxel_force.has_value()) + { + native.taxel_force = structs_from_rows(*taxel_force, "taxel_force"); + } + return pack(native); + }), + py::arg("taxel_force") = py::none(), "Encode a tactile frame. taxel_force is an (N, 3) array.") + .def_property_readonly( + "taxel_force", + [](py::object self) + { + const auto& view = self.cast&>(); + return struct_vector_view(self, view->taxel_force()); + }, + "Per-taxel force as an (N, 3) float32 view aliasing this object's buffer; call .copy() to own it.") + .def("__repr__", + [](const Serialized& self) + { + const auto* forces = self->taxel_force(); + return "TactileData(taxels=" + std::to_string(forces != nullptr ? forces->size() : 0) + ")"; + }); + + bind_record(m, "TactileDataRecord", "TactileData"); +} + +} // namespace core diff --git a/src/core/schema/python/tactile_shape_bindings.h b/src/core/schema/python/tactile_shape_bindings.h new file mode 100644 index 0000000000..2a2661840a --- /dev/null +++ b/src/core/schema/python/tactile_shape_bindings.h @@ -0,0 +1,218 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Python bindings for the TactileShape FlatBuffer schema (see tactile_shape.fbs), exposed as an +// encoded view. Each joint and taxel attribute reads as a zero-copy NumPy view strided over the +// TactileJoint / TactileTaxel structs; see schema_array_views.h for the aliasing caveat. + +#pragma once + +#include "schema_array_views.h" +#include "schema_serialized.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace core +{ + +// TactileJoint.parent of a root joint: XR_HAND_JOINT_MAX_ENUM_EXT. +constexpr uint32_t TACTILE_ROOT_PARENT = 0x7FFFFFFF; + +inline void bind_tactile_shape(py::module& m) +{ + using FloatRows = py::array_t; + using IndexRows = py::array_t; + + constexpr auto joint_stride = static_cast(sizeof(TactileJoint)); + constexpr auto taxel_stride = static_cast(sizeof(TactileTaxel)); + + // (count, width) view of one field across a vector of structs; absent reads as (0, width). + auto struct_field_view = [](py::object self, const auto* first, py::ssize_t stride, size_t count, py::ssize_t width) + { + using T = std::remove_cv_t>; + if (count == 0) + { + return width > 0 ? py::array_t(std::vector{ 0, width }) : py::array_t(py::ssize_t{ 0 }); + } + return strided_field_view(self, first, stride, static_cast(count), width); + }; + auto joints_of = [](py::object self) { return self.cast&>()->joints(); }; + auto taxels_of = [](py::object self) { return self.cast&>()->taxels(); }; + + auto shape = serialized_class(m, "TactileShape", "Encoded tactile skeleton and taxel layout."); + shape + .def(py::init( + [](const FloatRows& joint_bind_positions, const FloatRows& joint_bind_orientations, + const IndexRows& joint_parents, const FloatRows& taxel_positions, const FloatRows& taxel_normals, + const FloatRows& taxel_tangents, const IndexRows& taxel_joint_indices, + const FloatRows& taxel_joint_weights) + { + const auto positions = + structs_from_rows(joint_bind_positions, "joint_bind_positions"); + const auto orientations = + structs_from_rows(joint_bind_orientations, "joint_bind_orientations"); + if (joint_parents.ndim() != 1) + { + throw py::value_error("joint_parents: expected an (J,) array"); + } + const size_t joints = positions.size(); + if (orientations.size() != joints || static_cast(joint_parents.shape(0)) != joints) + { + throw py::value_error( + "joint_bind_positions, joint_bind_orientations and joint_parents must have the same length"); + } + + TactileShapeT native; + native.joints.reserve(joints); + for (size_t j = 0; j < joints; ++j) + { + const uint32_t parent = joint_parents.data()[j]; + if (parent != TACTILE_ROOT_PARENT && parent >= joints) + { + throw py::value_error("joint_parents: " + std::to_string(parent) + + " is neither a joint index nor ROOT_PARENT"); + } + native.joints.emplace_back(Pose(positions[j], orientations[j]), parent); + } + + const auto points = structs_from_rows(taxel_positions, "taxel_positions"); + const auto normals = structs_from_rows(taxel_normals, "taxel_normals"); + const auto tangents = structs_from_rows(taxel_tangents, "taxel_tangents"); + const auto indices = structs_from_rows, uint32_t, 4>( + taxel_joint_indices, "taxel_joint_indices"); + const auto weights = + structs_from_rows, float, 4>(taxel_joint_weights, "taxel_joint_weights"); + const size_t taxels = points.size(); + if (normals.size() != taxels || tangents.size() != taxels || indices.size() != taxels || + weights.size() != taxels) + { + throw py::value_error("every taxel_* array must have the same length"); + } + native.taxels.reserve(taxels); + for (size_t t = 0; t < taxels; ++t) + { + for (uint32_t joint : indices[t]) + { + if (joint >= joints) + { + throw py::value_error("taxel_joint_indices: " + std::to_string(joint) + + " is not a joint index"); + } + } + native.taxels.emplace_back(points[t], normals[t], tangents[t], + ::flatbuffers::span(indices[t].data(), 4), + ::flatbuffers::span(weights[t].data(), 4)); + } + return pack(native); + }), + py::arg("joint_bind_positions"), py::arg("joint_bind_orientations"), py::arg("joint_parents"), + py::arg("taxel_positions"), py::arg("taxel_normals"), py::arg("taxel_tangents"), + py::arg("taxel_joint_indices"), py::arg("taxel_joint_weights"), + "Encode a tactile shape. Joint arrays are (J, 3), (J, 4) and (J,); taxel arrays are (N, 3) or " + "(N, 4). Mismatched lengths and out-of-range joint indices raise ValueError; weights are not " + "checked.") + .def_property_readonly( + "joint_bind_positions", + [=](py::object self) + { + const auto* joints = joints_of(self); + const size_t count = joints != nullptr ? joints->size() : 0; + const float* first = + count > 0 ? reinterpret_cast(&joints->Get(0)->bind_pose().position()) : nullptr; + return struct_field_view(self, first, joint_stride, count, 3); + }, + "Joint bind positions as a (J, 3) float32 strided view.") + .def_property_readonly( + "joint_bind_orientations", + [=](py::object self) + { + const auto* joints = joints_of(self); + const size_t count = joints != nullptr ? joints->size() : 0; + const float* first = + count > 0 ? reinterpret_cast(&joints->Get(0)->bind_pose().orientation()) : nullptr; + return struct_field_view(self, first, joint_stride, count, 4); + }, + "Joint bind orientations (XYZW) as a (J, 4) float32 strided view.") + .def_property_readonly( + "joint_parents", + [=, offset = FBS_FIELD_OFFSET(TactileJoint, parent)](py::object self) + { + const auto* joints = joints_of(self); + const size_t count = joints != nullptr ? joints->size() : 0; + const uint32_t* first = count > 0 ? fbs_field_address(*joints->Get(0), offset) : nullptr; + return struct_field_view(self, first, joint_stride, count, 0); + }, + "Parent joint per joint as a (J,) uint32 strided view; ROOT_PARENT marks a root.") + .def_property_readonly( + "taxel_positions", + [=](py::object self) + { + const auto* taxels = taxels_of(self); + const size_t count = taxels != nullptr ? taxels->size() : 0; + const float* first = count > 0 ? reinterpret_cast(&taxels->Get(0)->position()) : nullptr; + return struct_field_view(self, first, taxel_stride, count, 3); + }, + "Taxel bind positions as an (N, 3) float32 strided view.") + .def_property_readonly( + "taxel_normals", + [=](py::object self) + { + const auto* taxels = taxels_of(self); + const size_t count = taxels != nullptr ? taxels->size() : 0; + const float* first = count > 0 ? reinterpret_cast(&taxels->Get(0)->normal()) : nullptr; + return struct_field_view(self, first, taxel_stride, count, 3); + }, + "Taxel unit normals as an (N, 3) float32 strided view.") + .def_property_readonly( + "taxel_tangents", + [=](py::object self) + { + const auto* taxels = taxels_of(self); + const size_t count = taxels != nullptr ? taxels->size() : 0; + const float* first = count > 0 ? reinterpret_cast(&taxels->Get(0)->tangent()) : nullptr; + return struct_field_view(self, first, taxel_stride, count, 3); + }, + "Taxel unit tangents as an (N, 3) float32 strided view.") + .def_property_readonly( + "taxel_joint_indices", + [=](py::object self) + { + const auto* taxels = taxels_of(self); + const size_t count = taxels != nullptr ? taxels->size() : 0; + const uint32_t* first = count > 0 ? taxels->Get(0)->joint_indices()->data() : nullptr; + return struct_field_view(self, first, taxel_stride, count, 4); + }, + "Skinning joint per slot as an (N, 4) uint32 strided view.") + .def_property_readonly( + "taxel_joint_weights", + [=](py::object self) + { + const auto* taxels = taxels_of(self); + const size_t count = taxels != nullptr ? taxels->size() : 0; + const float* first = count > 0 ? taxels->Get(0)->joint_weights()->data() : nullptr; + return struct_field_view(self, first, taxel_stride, count, 4); + }, + "Skinning weight per slot as an (N, 4) float32 strided view.") + .def("__repr__", + [](const Serialized& self) + { + const auto* joints = self->joints(); + const auto* taxels = self->taxels(); + return "TactileShape(joints=" + std::to_string(joints != nullptr ? joints->size() : 0) + + ", taxels=" + std::to_string(taxels != nullptr ? taxels->size() : 0) + ")"; + }); + shape.attr("ROOT_PARENT") = TACTILE_ROOT_PARENT; +} + +} // namespace core diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index 338327a763..c6233c7941 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -42,6 +42,12 @@ messageChannelConfig, ) from .haptic_sink import HapticSink +from .tactile_source import ( + TactileForces, + TactileShapeIndex, + TactileShapeLayout, + TactileSource, +) from .deviceio_tensor_types import ( HeadPoseTrackedType, HandPoseTrackedType, @@ -56,6 +62,8 @@ SomaBodyJointPosesTrackedType, SomaHandJointRotationsTrackedType, SomaHandJointPosesTrackedType, + TactileDataTrackedType, + TactileShapeTrackedType, DeviceIOHeadPoseTracked, DeviceIOHandPoseTracked, DeviceIOControllerSnapshotTracked, @@ -69,6 +77,7 @@ DeviceIOSomaBodyJointPosesTracked, DeviceIOSomaHandJointRotationsTracked, DeviceIOSomaHandJointPosesTracked, + DeviceIOTactileTracked, MessageChannelMessagesTrackedType, MessageChannelConnectionStatus, MessageChannelStatusType, @@ -110,6 +119,10 @@ "message_channel_config", "messageChannelConfig", "HapticSink", + "TactileSource", + "TactileShapeIndex", + "TactileForces", + "TactileShapeLayout", "HeadPoseTrackedType", "HandPoseTrackedType", "ControllerSnapshotTrackedType", @@ -123,6 +136,8 @@ "SomaBodyJointPosesTrackedType", "SomaHandJointRotationsTrackedType", "SomaHandJointPosesTrackedType", + "TactileDataTrackedType", + "TactileShapeTrackedType", "MessageChannelMessagesTrackedType", "MessageChannelConnectionStatus", "MessageChannelStatusType", @@ -139,6 +154,7 @@ "DeviceIOSomaBodyJointPosesTracked", "DeviceIOSomaHandJointRotationsTracked", "DeviceIOSomaHandJointPosesTracked", + "DeviceIOTactileTracked", "DeviceIOMessageChannelMessagesTracked", "MessageChannelMessagesTrackedGroup", "MessageChannelStatusGroup", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py index c062178318..ff5153723e 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py @@ -29,6 +29,8 @@ SomaHandJointPoses, SomaHandJointRotations, MessageChannelMessagesTracked, + TactileData, + TactileShape, ) @@ -157,6 +159,18 @@ class SomaHandJointPosesTrackedType(_PayloadTensorType): _payload_cls = SomaHandJointPoses +class TactileDataTrackedType(_PayloadTensorType): + """TactileData payload from DeviceIO TactileTracker.""" + + _payload_cls = TactileData + + +class TactileShapeTrackedType(_PayloadTensorType): + """TactileShape metadata from DeviceIO TactileTracker.""" + + _payload_cls = TactileShape + + class MessageChannelMessagesTrackedType(_RequiredPayloadTensorType): """MessageChannelMessagesTracked batch from DeviceIO MessageChannelTracker.""" @@ -313,6 +327,22 @@ def DeviceIOSomaHandJointPosesTracked() -> TensorGroupType: ) +def DeviceIOTactileTracked() -> TensorGroupType: + """Tracked tactile frame and its shape from DeviceIO TactileTracker. + + Contains: + tactile_tracked: TactileData handle, or None when inactive + tactile_shape: TactileShape metadata handle, or None until it arrives + """ + return TensorGroupType( + "deviceio_tactile", + [ + TactileDataTrackedType("tactile_tracked"), + TactileShapeTrackedType("tactile_shape"), + ], + ) + + def DeviceIOMessageChannelMessagesTracked() -> TensorGroupType: """Message batch from DeviceIO MessageChannelTracker.""" return TensorGroupType( diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py new file mode 100644 index 0000000000..b82955c33e --- /dev/null +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tactile Source Node - DeviceIO to Retargeting Engine converter. + +``TactileSource`` turns per-frame ``TactileData`` into a force array, and the tracker's +``TactileShape`` metadata into the arrays that lay those forces out. Taxel and joint counts are +fixed at construction so the retargeting graph has a static spec. +""" + +from __future__ import annotations + +from enum import IntEnum +from typing import Any, Optional, TYPE_CHECKING + +import numpy as np + +from .interface import IDeviceIOSource +from ..interface.retargeter_core_types import RetargeterIO, RetargeterIOType +from ..interface.tensor_group import TensorGroup +from ..interface.tensor_group_type import OptionalType, TensorGroupType +from ..tensor_types import DLDataType, NDArrayType +from .deviceio_tensor_types import DeviceIOTactileTracked + +if TYPE_CHECKING: + from isaaccapture.deviceio_trackers import ITracker + from isaaccapture.schema import TactileData, TactileShape + + +def _float_array(name: str, rows: int, width: int) -> NDArrayType: + return NDArrayType(name, shape=(rows, width), dtype=DLDataType.FLOAT, dtype_bits=32) + + +def _int_array(name: str, shape: tuple[int, ...]) -> NDArrayType: + return NDArrayType(name, shape=shape, dtype=DLDataType.INT, dtype_bits=32) + + +def TactileForces(num_taxels: int) -> TensorGroupType: + """Per-taxel force vectors: one ``(num_taxels, 3)`` float32 array.""" + return TensorGroupType( + f"tactile_forces_{num_taxels}", + [_float_array("taxel_force", num_taxels, 3)], + ) + + +class TactileShapeIndex(IntEnum): + """Positions of the arrays in a :func:`TactileShapeLayout` group.""" + + JOINT_BIND_POSITIONS = 0 + JOINT_BIND_ORIENTATIONS = 1 + JOINT_PARENTS = 2 + TAXEL_POSITIONS = 3 + TAXEL_NORMALS = 4 + TAXEL_TANGENTS = 5 + TAXEL_JOINT_INDICES = 6 + TAXEL_JOINT_WEIGHTS = 7 + + +def TactileShapeLayout(num_joints: int, num_taxels: int) -> TensorGroupType: + """``TactileShape`` as arrays, ordered by :class:`TactileShapeIndex`. + + Each array is named after the ``TactileShape`` property it is read from. Joint parents and + joint indices are int32; the root parent stays ``TactileShape.ROOT_PARENT``. + """ + return TensorGroupType( + f"tactile_shape_{num_joints}x{num_taxels}", + [ + _float_array("joint_bind_positions", num_joints, 3), + _float_array("joint_bind_orientations", num_joints, 4), + _int_array("joint_parents", (num_joints,)), + _float_array("taxel_positions", num_taxels, 3), + _float_array("taxel_normals", num_taxels, 3), + _float_array("taxel_tangents", num_taxels, 3), + _int_array("taxel_joint_indices", (num_taxels, 4)), + _float_array("taxel_joint_weights", num_taxels, 4), + ], + ) + + +class TactileSource(IDeviceIOSource): + """Converter: DeviceIO ``TactileTracker`` -> per-taxel forces and the shape that lays them out. + + Outputs (Optional): + - :data:`FORCES`: :func:`TactileForces` of ``num_taxels``, in ``TactileShape`` taxel + order; absent when the device is inactive. + - :data:`SHAPE`: :func:`TactileShapeLayout` of ``num_joints`` x ``num_taxels``; absent + until the tracker holds the device's shape. The tracker publishes no forces before then. + + A frame or shape whose counts differ from the constructor's raises ``ValueError``. + """ + + FORCES = "taxel_forces" + SHAPE = "tactile_shape" + + def __init__( + self, + name: str, + collection_id: str, + num_joints: int, + num_taxels: int, + max_flatbuffer_size: Optional[int] = None, + ) -> None: + """Initialize the tactile source. + + Args: + name: Unique name for this source node. + collection_id: Tensor collection ID of the producer's ``TactileData`` stream. + num_joints: Joint count of the device skeleton; fixes the shape output spec. + num_taxels: Taxel count of the device; fixes the output specs. + max_flatbuffer_size: Override for ``TactileTracker``'s default collection size. + """ + from isaaccapture.deviceio_trackers import TactileTracker + + self._tracker = ( + TactileTracker(collection_id) + if max_flatbuffer_size is None + else TactileTracker(collection_id, max_flatbuffer_size) + ) + self._num_joints = int(num_joints) + self._num_taxels = int(num_taxels) + super().__init__(name) + + def get_tracker(self) -> "ITracker": + """Return the ``TactileTracker`` instance for ``TeleopSession`` to initialize.""" + return self._tracker + + def poll_tracker(self, deviceio_session: Any) -> RetargeterIO: + """Poll the tracker's frame and metadata for the compute step.""" + tg = TensorGroup(DeviceIOTactileTracked()) + tg[0] = self._tracker.get_data(deviceio_session) + tg[1] = self._tracker.get_metadata(deviceio_session) + return {"deviceio_tactile": tg} + + def input_spec(self) -> RetargeterIOType: + """Declare the raw DeviceIO tactile input.""" + return {"deviceio_tactile": DeviceIOTactileTracked()} + + def output_spec(self) -> RetargeterIOType: + """Declare the force and shape outputs (Optional -- may be absent).""" + return { + self.FORCES: OptionalType(TactileForces(self._num_taxels)), + self.SHAPE: OptionalType( + TactileShapeLayout(self._num_joints, self._num_taxels) + ), + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + """Copy the forces and shape out, or call ``set_none()`` on whichever is absent.""" + data: "TactileData | None" = inputs["deviceio_tactile"][0] + shape: "TactileShape | None" = inputs["deviceio_tactile"][1] + + forces_out = outputs[self.FORCES] + if data is None: + forces_out.set_none() + else: + forces = data.taxel_force + if len(forces) != self._num_taxels: + raise ValueError( + f"{self.name}: frame has {len(forces)} taxels, source expects {self._num_taxels}" + ) + forces_out[0] = forces + + shape_out = outputs[self.SHAPE] + if shape is None: + shape_out.set_none() + return + joints, taxels = len(shape.joint_parents), len(shape.taxel_positions) + if (joints, taxels) != (self._num_joints, self._num_taxels): + raise ValueError( + f"{self.name}: shape has {joints} joints and {taxels} taxels, " + f"source expects {self._num_joints} and {self._num_taxels}" + ) + for index, tensor_type in enumerate(shape_out.group_type.types): + value = getattr(shape, tensor_type.name) + if value.dtype == np.uint32: + value = value.astype(np.int32) + shape_out[index] = value diff --git a/src/python/isaaccapture/schema/__init__.py b/src/python/isaaccapture/schema/__init__.py index 9b484fafb1..a64904dffa 100644 --- a/src/python/isaaccapture/schema/__init__.py +++ b/src/python/isaaccapture/schema/__init__.py @@ -56,6 +56,10 @@ # OGLO tactile glove types. OgloGloveSample, OgloGloveSampleRecord, + # Tactile types. + TactileData, + TactileDataRecord, + TactileShape, # Tracker metadata, for producers of a tracker's metadata. TRACKER_METADATA_MAX_SIZE, TRACKER_METADATA_TENSOR_IDENTIFIER, @@ -192,6 +196,10 @@ def __getattr__(name: str): # OGLO tactile glove types. "OgloGloveSample", "OgloGloveSampleRecord", + # Tactile types. + "TactileData", + "TactileDataRecord", + "TactileShape", # Tracker metadata. "TRACKER_METADATA_MAX_SIZE", "TRACKER_METADATA_TENSOR_IDENTIFIER", diff --git a/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp b/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp index 1b43f72a98..557dab36a3 100644 --- a/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp +++ b/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,8 @@ #include #include #include +#include +#include #include #include @@ -824,3 +827,95 @@ TEST_CASE("ReplaySession: keyboard tracker round-trip ignores live provider inpu session->update(); CHECK_FALSE(keyboard_tracker.get_data(*session)); } + +// ============================================================================= +// Tracker with metadata — TactileTracker (TactileShape on its own channel) +// ============================================================================= + +namespace +{ + +// Live writes the metadata once, before any data it lets through; mirror that order. +void write_tactile_recording(const std::string& path, const std::string& base_name, bool with_metadata) +{ + auto writer = open_writer(path); + core::McapTrackerChannels data( + *writer, base_name, to_string_vec(core::TactileRecordingTraits::recording_channels)); + core::McapTrackerChannels metadata( + *writer, base_name, to_string_vec(core::TactileRecordingTraits::metadata_channels)); + + if (with_metadata) + { + core::TactileShapeT shape; + shape.joints = { core::TactileJoint(make_pose(0, 0, 0), 0x7FFFFFFF) }; + shape.taxels = { core::TactileTaxel(), core::TactileTaxel() }; + metadata.write(0, core::pack(shape), 500); + } + + for (int i = 0; i < 2; ++i) + { + core::TactileDataT frame; + frame.taxel_force = { core::Point(static_cast(i), 0, 0), core::Point(0, 0, 1) }; + const int64_t time_ns = (i + 1) * 1000; + const auto record = + core::pack_record(&frame, core::DeviceDataTimestamp(time_ns, time_ns, time_ns)); + data.write(0, record); + data.write(1, record); + } + writer->close(); +} + +} // namespace + +TEST_CASE("ReplaySession: tracker metadata is replayed before its data", "[replay][session][metadata]") +{ + // Frozen wire strings, as for the record channels above. + CHECK(std::string_view(core::TactileShape::GetFullyQualifiedName()) == "core.TactileShape"); + REQUIRE(core::TactileRecordingTraits::metadata_channels.size() == 1); + CHECK(std::string_view(core::TactileRecordingTraits::metadata_channels[0]) == "tactile_metadata"); + + auto path = get_temp_mcap_path(); + TempFileCleanup cleanup(path); + write_tactile_recording(path, "tactile", true); + + core::TactileTracker tracker("tactile/right"); + core::McapReplayConfig config; + config.filename = path; + config.tracker_names = { { &tracker, "tactile" } }; + auto session = core::ReplaySession::run(config); + + // Read when the session opens, so it is there before the first frame. + const auto& shape = tracker.get_metadata(*session); + REQUIRE(shape); + CHECK(shape->taxels()->size() == 2); + + for (int i = 0; i < 2; ++i) + { + session->update(); + const auto& frame = tracker.get_data(*session); + REQUIRE(frame); + CHECK(frame->taxel_force()->Get(0)->x() == static_cast(i)); + CHECK(tracker.get_metadata(*session).get() == shape.get()); + } + + session->update(); + CHECK_FALSE(tracker.get_data(*session)); + CHECK(tracker.get_metadata(*session)); +} + +TEST_CASE("ReplaySession: data recorded without its metadata is withheld", "[replay][session][metadata]") +{ + auto path = get_temp_mcap_path(); + TempFileCleanup cleanup(path); + write_tactile_recording(path, "tactile", false); + + core::TactileTracker tracker("tactile/right"); + core::McapReplayConfig config; + config.filename = path; + config.tracker_names = { { &tracker, "tactile" } }; + auto session = core::ReplaySession::run(config); + + session->update(); + CHECK_FALSE(tracker.get_metadata(*session)); + CHECK_FALSE(tracker.get_data(*session)); +} diff --git a/tests/cpp/core/schema/CMakeLists.txt b/tests/cpp/core/schema/CMakeLists.txt index 3fa2548b78..d2f839e50f 100644 --- a/tests/cpp/core/schema/CMakeLists.txt +++ b/tests/cpp/core/schema/CMakeLists.txt @@ -22,6 +22,8 @@ add_executable(schema_tests test_soma_body_joint_poses.cpp test_soma_hand_joint_rotations.cpp test_soma_hand_joint_poses.cpp + test_tactile.cpp + test_tactile_shape.cpp test_schema_conform.cpp ) target_link_libraries(schema_tests PRIVATE diff --git a/tests/cpp/core/schema/test_tactile.cpp b/tests/cpp/core/schema/test_tactile.cpp new file mode 100644 index 0000000000..3f15e08326 --- /dev/null +++ b/tests/cpp/core/schema/test_tactile.cpp @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit tests for the generated TactileData FlatBuffer message. + +#include +#include +#include +#include +#include +#include + +#include +#include + +// VT_ == (id + 2) * 2. +#define VT(field) (field + 2) * 2 +static_assert(core::TactileData::VT_TAXEL_FORCE == VT(0)); +static_assert(core::TactileDataRecord::VT_DATA == VT(0)); +static_assert(core::TactileDataRecord::VT_TIMESTAMP == VT(1)); + +// Producers copy XrVector3f arrays straight into taxel_force. +static_assert(sizeof(core::Point) == sizeof(XrVector3f)); + +namespace +{ + +core::TactileDataT make_frame() +{ + core::TactileDataT native; + native.taxel_force = { core::Point(0.0f, 0.0f, 1.0f), core::Point(0.5f, -0.5f, 2.0f) }; + return native; +} + +} // namespace + +TEST_CASE("TactileData round-trips every field", "[unit][tactile]") +{ + const auto data = core::pack(make_frame()); + + REQUIRE(data); + REQUIRE(data->taxel_force() != nullptr); + REQUIRE(data->taxel_force()->size() == 2); + CHECK(data->taxel_force()->Get(1)->x() == 0.5f); + CHECK(data->taxel_force()->Get(1)->z() == 2.0f); +} + +TEST_CASE("TactileData leaves an empty frame's vector absent", "[unit][tactile]") +{ + const auto data = core::pack(core::TactileDataT{}); + + REQUIRE(data); + CHECK(data->taxel_force() == nullptr); +} + +TEST_CASE("TactileDataRecord verifies and carries its timestamp", "[unit][tactile]") +{ + core::TactileDataRecordT record; + record.data = std::make_shared(make_frame()); + record.timestamp = std::make_shared(1, 2, 3); + + flatbuffers::FlatBufferBuilder builder; + builder.Finish(core::TactileDataRecord::Pack(builder, &record)); + + flatbuffers::Verifier verifier(builder.GetBufferPointer(), builder.GetSize()); + REQUIRE(core::VerifyTactileDataRecordBuffer(verifier)); + const auto* read = core::GetTactileDataRecord(builder.GetBufferPointer()); + CHECK(read->timestamp()->sample_time_local_common_clock() == 2); + CHECK(read->data()->taxel_force()->size() == 2); +} diff --git a/tests/cpp/core/schema/test_tactile_shape.cpp b/tests/cpp/core/schema/test_tactile_shape.cpp new file mode 100644 index 0000000000..787f5ab7e4 --- /dev/null +++ b/tests/cpp/core/schema/test_tactile_shape.cpp @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit tests for the generated TactileShape FlatBuffer message. + +#include +#include +#include +#include +#include + +#include +#include +#include + +// VT_ == (id + 2) * 2. +#define VT(field) (field + 2) * 2 +static_assert(core::TactileShape::VT_JOINTS == VT(0)); +static_assert(core::TactileShape::VT_TAXELS == VT(1)); + +// Producers copy OpenXR values straight into the structs. +static_assert(sizeof(core::Point) == sizeof(XrVector3f)); +static_assert(sizeof(core::TactileJoint) == sizeof(XrPosef) + sizeof(XrHandJointEXT)); +static_assert(sizeof(core::TactileTaxel) == 3 * sizeof(XrVector3f) + 4 * sizeof(XrHandJointEXT) + sizeof(XrVector4f)); + +namespace +{ + +constexpr uint32_t kRootParent = 0x7FFFFFFF; + +// Two joints (wrist -> index tip) and one taxel skinned to both. +core::TactileShapeT make_shape() +{ + core::TactileShapeT native; + native.joints = { core::TactileJoint(core::Pose(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)), kRootParent), + core::TactileJoint(core::Pose(core::Point(0, 0.1f, 0), core::Quaternion(0, 0, 0, 1)), 0) }; + + const std::array indices = { 1, 0, 0, 0 }; + const std::array weights = { 0.75f, 0.25f, 0.0f, 0.0f }; + native.taxels = { core::TactileTaxel(core::Point(0, 0.09f, 0.01f), core::Point(0, 0, 1), core::Point(1, 0, 0), + ::flatbuffers::span(indices.data(), 4), + ::flatbuffers::span(weights.data(), 4)) }; + return native; +} + +} // namespace + +TEST_CASE("TactileShape round-trips skeleton, taxels and skinning", "[unit][tactile_shape]") +{ + const auto shape = core::pack(make_shape()); + + REQUIRE(shape); + REQUIRE(shape->joints()->size() == 2); + CHECK(shape->joints()->Get(0)->parent() == kRootParent); + CHECK(shape->joints()->Get(1)->parent() == 0); + CHECK(shape->joints()->Get(1)->bind_pose().position().y() == 0.1f); + REQUIRE(shape->taxels()->size() == 1); + const auto* taxel = shape->taxels()->Get(0); + CHECK(taxel->normal().z() == 1.0f); + CHECK(taxel->joint_indices()->Get(0) == 1); + CHECK(taxel->joint_weights()->Get(1) == 0.25f); +} + +TEST_CASE("TactileShape is its own root and verifies", "[unit][tactile_shape]") +{ + const auto shape = core::pack(make_shape()); + const auto bytes = shape.buffer(); + + flatbuffers::Verifier verifier(bytes.data(), bytes.size()); + REQUIRE(core::VerifyTactileShapeBuffer(verifier)); + CHECK(std::string(core::TactileShape::GetFullyQualifiedName()) == "core.TactileShape"); + + core::TactileShapeT unpacked; + core::GetTactileShape(bytes.data())->UnPackTo(&unpacked); + REQUIRE(unpacked.taxels.size() == 1); + CHECK(unpacked.taxels[0].joint_weights()->Get(0) == 0.75f); +} diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py new file mode 100644 index 0000000000..068bec1401 --- /dev/null +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the TactileSource DeviceIO converter. + +Payloads are built through the real schema bindings; no OpenXR device is involved. +""" + +import numpy as np +import pytest + +from isaaccapture.retargeting_engine.deviceio_source_nodes import ( + TactileShapeIndex, + TactileShapeLayout, + TactileSource, +) +from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group +from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup +from isaaccapture.schema import TactileData, TactileShape + +NUM_JOINTS = 2 +NUM_TAXELS = 3 + + +def _source(**kwargs) -> TactileSource: + args = dict( + collection_id="tactile/right", num_joints=NUM_JOINTS, num_taxels=NUM_TAXELS + ) + args.update(kwargs) + return TactileSource("tactile", **args) + + +def _inputs(source, data, shape) -> dict: + tg = TensorGroup(source.input_spec()["deviceio_tactile"]) + tg[0] = data + tg[1] = shape + return {"deviceio_tactile": tg} + + +def _outputs(source) -> dict: + return {name: _make_output_group(gt) for name, gt in source.output_spec().items()} + + +def _shape(num_joints: int = NUM_JOINTS, num_taxels: int = NUM_TAXELS) -> TactileShape: + indices = np.zeros((num_taxels, 4), np.uint32) + weights = np.zeros((num_taxels, 4), np.float32) + weights[:, 0] = 1.0 + return TactileShape( + joint_bind_positions=np.zeros((num_joints, 3), np.float32), + joint_bind_orientations=np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1)), + joint_parents=[TactileShape.ROOT_PARENT] + list(range(num_joints - 1)), + taxel_positions=np.full((num_taxels, 3), 0.01, np.float32), + taxel_normals=np.tile([0.0, 0.0, 1.0], (num_taxels, 1)), + taxel_tangents=np.tile([1.0, 0.0, 0.0], (num_taxels, 1)), + taxel_joint_indices=indices, + taxel_joint_weights=weights, + ) + + +class _FakeTracker: + """Stands in for a TactileTracker: fixed data and metadata.""" + + def __init__(self, data, metadata): + self._data = data + self._metadata = metadata + + def get_data(self, _session): + return self._data + + def get_metadata(self, _session): + return self._metadata + + +class TestTactileSource: + def test_creation_and_tracker(self): + assert _source().get_tracker().get_name() == "TactileTracker" + + def test_specs(self): + src = _source() + assert list(src.input_spec()) == ["deviceio_tactile"] + forces = src.output_spec()[TactileSource.FORCES] + assert forces.is_optional + assert forces.inner_type.types[0].shape == (NUM_TAXELS, 3) + assert src.output_spec()[TactileSource.SHAPE].is_optional + + def test_index_matches_layout_order(self): + names = [t.name for t in TactileShapeLayout(NUM_JOINTS, NUM_TAXELS).types] + assert names == [member.name.lower() for member in TactileShapeIndex] + + def test_poll_reads_data_and_metadata_from_one_tracker(self): + src = _source() + data = TactileData(np.zeros((NUM_TAXELS, 3), np.float32)) + shape = _shape() + src._tracker = _FakeTracker(data, shape) + + polled = src.poll_tracker(None)["deviceio_tactile"] + + assert polled[0] is data + assert polled[1] is shape + + def test_active_conversion(self): + src = _source() + forces = np.arange(9, dtype=np.float32).reshape(3, 3) + outputs = _outputs(src) + src.compute(_inputs(src, TactileData(forces), _shape()), outputs) + + np.testing.assert_array_equal( + np.asarray(outputs[TactileSource.FORCES][0]), forces + ) + group = outputs[TactileSource.SHAPE] + assert not group.is_none + parents = np.asarray(group[TactileShapeIndex.JOINT_PARENTS]) + assert parents.dtype == np.int32 + assert parents.tolist() == [TactileShape.ROOT_PARENT, 0] + assert ( + np.asarray(group[TactileShapeIndex.TAXEL_JOINT_INDICES]).dtype == np.int32 + ) + np.testing.assert_array_equal( + np.asarray(group[TactileShapeIndex.TAXEL_NORMALS]), + np.tile([0.0, 0.0, 1.0], (NUM_TAXELS, 1)), + ) + + def test_shape_without_data(self): + src = _source() + outputs = _outputs(src) + src.compute(_inputs(src, None, _shape()), outputs) + assert outputs[TactileSource.FORCES].is_none + assert not outputs[TactileSource.SHAPE].is_none + + def test_inactive_sets_none(self): + src = _source() + outputs = _outputs(src) + src.compute(_inputs(src, None, None), outputs) + assert outputs[TactileSource.FORCES].is_none + assert outputs[TactileSource.SHAPE].is_none + + def test_taxel_count_mismatch_raises(self): + src = _source(num_taxels=4) + frame = TactileData(np.zeros((3, 3), np.float32)) + with pytest.raises(ValueError, match="3 taxels"): + src.compute(_inputs(src, frame, None), _outputs(src)) + + def test_shape_count_mismatch_raises(self): + src = _source(num_joints=NUM_JOINTS + 1) + with pytest.raises(ValueError, match="2 joints"): + src.compute(_inputs(src, None, _shape()), _outputs(src)) diff --git a/tests/python/core/schema/test_tactile.py b/tests/python/core/schema/test_tactile.py new file mode 100644 index 0000000000..f6d91cb243 --- /dev/null +++ b/tests/python/core/schema/test_tactile.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for TactileData / TactileDataRecord in isaaccapture.schema.""" + +import gc + +import numpy as np +import pytest + +from isaaccapture.schema import DeviceDataTimestamp, TactileData, TactileDataRecord + +FORCES = np.array([[0.0, 0.0, 1.0], [0.5, -0.5, 2.0], [0.0, 0.1, 0.0]], np.float32) + + +class TestTactileData: + def test_default_construction(self): + data = TactileData() + assert data.taxel_force.shape == (0, 3) + + def test_round_trip(self): + data = TactileData(FORCES) + assert data.taxel_force.dtype == np.float32 + np.testing.assert_array_equal(data.taxel_force, FORCES) + + def test_accepts_nested_lists(self): + data = TactileData([[1, 2, 3]]) + np.testing.assert_array_equal(data.taxel_force, [[1.0, 2.0, 3.0]]) + + def test_view_outlives_its_table(self): + forces = TactileData(FORCES).taxel_force + gc.collect() + np.testing.assert_array_equal(forces, FORCES) + + def test_rejects_wrong_width(self): + with pytest.raises(ValueError, match="taxel_force"): + TactileData(np.zeros((2, 4), np.float32)) + + def test_repr(self): + assert "taxels=3" in repr(TactileData(FORCES)) + + +class TestTactileDataRecord: + def test_record_carries_payload_and_timestamp(self): + record = TactileDataRecord(TactileData(FORCES), DeviceDataTimestamp(1, 2, 3)) + np.testing.assert_array_equal(record.data.taxel_force, FORCES) + assert record.timestamp.sample_time_raw_device_clock == 3 + + def test_payload_less_record(self): + record = TactileDataRecord(None, DeviceDataTimestamp(1, 2, 3)) + assert record.data is None diff --git a/tests/python/core/schema/test_tactile_shape.py b/tests/python/core/schema/test_tactile_shape.py new file mode 100644 index 0000000000..733d5a8534 --- /dev/null +++ b/tests/python/core/schema/test_tactile_shape.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for TactileShape in isaaccapture.schema.""" + +import numpy as np +import pytest + +from isaaccapture.schema import TactileShape + +ROOT = TactileShape.ROOT_PARENT + + +def shape_arrays(num_joints: int = 2, num_taxels: int = 3) -> dict: + """A wrist-rooted chain with every taxel skinned to its last two joints.""" + indices = np.zeros((num_taxels, 4), np.uint32) + indices[:, 0] = num_joints - 1 + weights = np.zeros((num_taxels, 4), np.float32) + weights[:, 0] = 1.0 + return dict( + joint_bind_positions=np.arange(num_joints * 3, dtype=np.float32).reshape(-1, 3), + joint_bind_orientations=np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1)).astype( + np.float32 + ), + joint_parents=np.array([ROOT] + list(range(num_joints - 1)), np.uint32), + taxel_positions=np.full((num_taxels, 3), 0.01, np.float32), + taxel_normals=np.tile([0.0, 0.0, 1.0], (num_taxels, 1)).astype(np.float32), + taxel_tangents=np.tile([1.0, 0.0, 0.0], (num_taxels, 1)).astype(np.float32), + taxel_joint_indices=indices, + taxel_joint_weights=weights, + ) + + +class TestTactileShape: + def test_root_parent_is_the_openxr_max_enum(self): + assert ROOT == 0x7FFFFFFF + + def test_round_trip(self): + arrays = shape_arrays() + shape = TactileShape(**arrays) + + for name, expected in arrays.items(): + value = getattr(shape, name) + assert value.dtype == expected.dtype, name + np.testing.assert_array_equal(value, expected, err_msg=name) + + def test_rejects_mismatched_joint_lengths(self): + arrays = shape_arrays() + arrays["joint_parents"] = arrays["joint_parents"][:1] + with pytest.raises(ValueError, match="same length"): + TactileShape(**arrays) + + def test_rejects_mismatched_taxel_lengths(self): + arrays = shape_arrays() + arrays["taxel_normals"] = arrays["taxel_normals"][:2] + with pytest.raises(ValueError, match="same length"): + TactileShape(**arrays) + + def test_rejects_parent_out_of_range(self): + arrays = shape_arrays() + arrays["joint_parents"][1] = 5 + with pytest.raises(ValueError, match="joint_parents"): + TactileShape(**arrays) + + def test_rejects_skinning_joint_out_of_range(self): + arrays = shape_arrays() + arrays["taxel_joint_indices"][0, 3] = 2 + with pytest.raises(ValueError, match="taxel_joint_indices"): + TactileShape(**arrays) + + def test_rejects_wrong_width(self): + arrays = shape_arrays() + arrays["taxel_joint_weights"] = np.zeros((3, 3), np.float32) + with pytest.raises(ValueError, match="taxel_joint_weights"): + TactileShape(**arrays) + + def test_repr(self): + assert ( + repr(TactileShape(**shape_arrays())) == "TactileShape(joints=2, taxels=3)" + ) From 7ff1680c35ce5c0ba94ac2a6a362a1a5c6024f03 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 15 Oct 2026 16:58:44 -0700 Subject: [PATCH 02/45] Add tactile pusher and feeder example Signed-off-by: lotusl Co-Authored-By: Claude Opus 5.5 --- examples/CMakeLists.txt | 1 + examples/tactile/CMakeLists.txt | 7 + examples/tactile/README.md | 50 +++++ examples/tactile/pyproject.toml | 23 ++ .../isaaccapture_examples/tactile/__init__.py | 2 + .../isaaccapture_examples/tactile/common.py | 14 ++ .../isaaccapture_examples/tactile/feeder.py | 203 ++++++++++++++++++ .../isaaccapture_examples/tactile/printer.py | 177 +++++++++++++++ 8 files changed, 477 insertions(+) create mode 100644 examples/tactile/CMakeLists.txt create mode 100644 examples/tactile/README.md create mode 100644 examples/tactile/pyproject.toml create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/__init__.py create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/common.py create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/feeder.py create mode 100644 examples/tactile/python/isaaccapture_examples/tactile/printer.py diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9b96fea7f9..b1855c9a1d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -19,6 +19,7 @@ if(BUILD_EXAMPLES) add_subdirectory(deviceio_live_view) add_subdirectory(soma_body_publisher) add_subdirectory(soma_hand_publisher) + add_subdirectory(tactile) add_subdirectory(haptic_feedback) add_subdirectory(sharpa_avatar) add_subdirectory(vehicle_teleop) diff --git a/examples/tactile/CMakeLists.txt b/examples/tactile/CMakeLists.txt new file mode 100644 index 0000000000..e3caba6501 --- /dev/null +++ b/examples/tactile/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +include(${CMAKE_SOURCE_DIR}/cmake/InstallPythonExample.cmake) +install_python_example(DESTINATION examples/tactile) diff --git a/examples/tactile/README.md b/examples/tactile/README.md new file mode 100644 index 0000000000..79d66e10d6 --- /dev/null +++ b/examples/tactile/README.md @@ -0,0 +1,50 @@ + + +# Tactile + +A mock tactile device and a consumer for exercising the tactile tracker without hardware. + +- `feeder` publishes a 26-joint hand with 200 taxels by default. Each frame carries one + `TactileData`; the `TactileShape` metadata goes whole when the feeder starts and every second + after that. It publishes through `TensorPushTracker` on a `DeviceIOSession`, as a tactile + plugin would through `SchemaPusher`. +- `printer` reads one `TactileTracker` directly. It discovers the joint and taxel counts from + `TactileShape`, prints the first and last taxel forces, and records the session to MCAP. + +```text +feeder -> "tactile" (TactileData, every frame) + -> "tactile/metadata" (TactileShape, at start and every --metadata-interval s) + -> TactileTracker -> printer, and tactile/* MCAP channels +``` + +Install the example: + +```bash +uv pip install -e ./examples/tactile +``` + +Start the printer first; it starts the CloudXR runtime, and no headset needs to connect: + +```bash +python -m isaaccapture_examples.tactile.printer --accept-eula --duration 0 +``` + +Then, in another terminal, start the feeder. It attaches to the same runtime: + +```bash +python -m isaaccapture_examples.tactile.feeder +``` + +Stop the printer with `Ctrl+C`; it writes `./recordings/tactile_.mcap`. Print a +recording with no runtime at all: + +```bash +python -m isaaccapture_examples.tactile.printer --replay recordings/tactile_.mcap +``` + +Use feeder options `--joints` and `--taxels` to change the shape; the printer discovers both +counts from metadata. The `TactileShape` must fit `TRACKER_METADATA_MAX_SIZE`, which holds 26 +joints and 1000 taxels. The two processes need only agree on `--collection-id`. diff --git a/examples/tactile/pyproject.toml b/examples/tactile/pyproject.toml new file mode 100644 index 0000000000..d9cdeedcab --- /dev/null +++ b/examples/tactile/pyproject.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "isaaccapture-examples-tactile" +version = "0.0.0" +description = "Isaac Teleop tactile feeder and printer" +requires-python = ">=3.11,<3.14" +dependencies = [ + "isaaccapture[cloudxr]", + "numpy>=1.23.0", +] + +[tool.hatch.build.targets.wheel] +only-include = ["python/isaaccapture_examples/tactile"] +sources = ["python"] + +[tool.uv.sources] +isaaccapture = { path = "../..", editable = true } diff --git a/examples/tactile/python/isaaccapture_examples/tactile/__init__.py b/examples/tactile/python/isaaccapture_examples/tactile/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/tactile/python/isaaccapture_examples/tactile/common.py b/examples/tactile/python/isaaccapture_examples/tactile/common.py new file mode 100644 index 0000000000..1535dd0152 --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/common.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Values the feeder and the printer must agree on.""" + +DEFAULT_COLLECTION_ID = "tactile" + +MIN_NUM_JOINTS = 2 +MAX_NUM_JOINTS = 26 +DEFAULT_NUM_JOINTS = MAX_NUM_JOINTS +DEFAULT_NUM_TAXELS = 200 + +# TactileTracker's default max_flatbuffer_size: the reader rejects a larger collection. +TACTILE_SAMPLE_SIZE = 16384 diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py new file mode 100644 index 0000000000..b8415e62d1 --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Publish mock tactile frames and their TactileShape metadata, as a tactile device would. + +Each frame pushes one TactileData sample on the data collection. The encoded TactileShape goes +on "/metadata" when the feeder starts and every ``--metadata-interval`` seconds +after that. Pair it with ``python -m isaaccapture_examples.tactile.printer``. +""" + +import argparse +import logging +import math +import sys +import time +from contextlib import ExitStack + +import numpy as np + +from isaaccapture.cloudxr import CloudXRLauncher +from isaaccapture.deviceio_session import DeviceIOSession +from isaaccapture.deviceio_trackers import TensorPushTracker +from isaaccapture.oxr import OpenXRSession +from isaaccapture.schema import ( + TRACKER_METADATA_MAX_SIZE, + TRACKER_METADATA_TENSOR_IDENTIFIER, + TactileData, + TactileShape, + tracker_metadata_collection_id, +) + +from .common import ( + DEFAULT_COLLECTION_ID, + DEFAULT_NUM_JOINTS, + DEFAULT_NUM_TAXELS, + MAX_NUM_JOINTS, + MIN_NUM_JOINTS, + TACTILE_SAMPLE_SIZE, +) + +logger = logging.getLogger("isaaccapture.examples.tactile.feeder") + +WRIST = 1 +# First and distal joint of each finger, thumb to little, as XrHandJointEXT ordinals. +FINGER_FIRST_JOINTS = (2, 6, 11, 16, 21) +FINGER_DISTAL_JOINTS = (4, 9, 14, 19, 24) + + +def mock_shape(num_joints: int, num_taxels: int) -> TactileShape: + """Build a flat hand in hand-root space with taxels on its distal segments.""" + positions = np.zeros((num_joints, 3), np.float32) + parents = np.full(num_joints, WRIST, np.uint32) + parents[WRIST] = TactileShape.ROOT_PARENT + positions[0] = (0.0, 0.05, 0.0) + + taxel_joints = [] + for finger, first in enumerate(FINGER_FIRST_JOINTS): + if first >= num_joints: + break + finger_end = ( + FINGER_FIRST_JOINTS[finger + 1] - 1 if finger < 4 else MAX_NUM_JOINTS - 1 + ) + last = min(finger_end, num_joints - 1) + for joint in range(first, last + 1): + positions[joint] = ( + -0.04 + 0.02 * finger, + 0.02 + 0.025 * (joint - first), + 0.0, + ) + if joint > first: + parents[joint] = joint - 1 + taxel_joints.append(min(FINGER_DISTAL_JOINTS[finger], last)) + + if not taxel_joints: + taxel_joints.append(WRIST) + + taxel_positions = np.zeros((num_taxels, 3), np.float32) + joint_indices = np.zeros((num_taxels, 4), np.uint32) + joint_weights = np.zeros((num_taxels, 4), np.float32) + per_segment = math.ceil(num_taxels / len(taxel_joints)) + for taxel in range(num_taxels): + segment, slot = taxel % len(taxel_joints), taxel // len(taxel_joints) + joint = taxel_joints[segment] + taxel_positions[taxel] = positions[joint] + ( + 0.0, + 0.02 * slot / per_segment, + -0.005, + ) + joint_indices[taxel] = joint + joint_weights[taxel, 0] = 1.0 + + return TactileShape( + joint_bind_positions=positions, + joint_bind_orientations=np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1)), + joint_parents=parents, + taxel_positions=taxel_positions, + taxel_normals=np.tile([0.0, 0.0, -1.0], (num_taxels, 1)), + taxel_tangents=np.tile([1.0, 0.0, 0.0], (num_taxels, 1)), + taxel_joint_indices=joint_indices, + taxel_joint_weights=joint_weights, + ) + + +def mock_forces(num_taxels: int, t: float) -> np.ndarray: + """Generate a positive 0.5 Hz Z-force wave, phase-shifted per finger.""" + finger = np.arange(num_taxels) % 5 + phase = 2.0 * math.pi * 0.5 * t - 1.2 * finger + forces = np.zeros((num_taxels, 3), np.float32) + forces[:, 2] = 1.25 + 0.75 * np.sin(phase) + return forces + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--collection-id", default=DEFAULT_COLLECTION_ID) + parser.add_argument("--joints", type=int, default=DEFAULT_NUM_JOINTS) + parser.add_argument("--taxels", type=int, default=DEFAULT_NUM_TAXELS) + parser.add_argument("--rate", type=float, default=60.0, help="Frames per second") + parser.add_argument( + "--metadata-interval", + type=float, + default=1.0, + help="Seconds between pushes of the TactileShape", + ) + parser.add_argument( + "--duration", + type=float, + default=0.0, + help="Seconds to run; 0 runs until Ctrl+C", + ) + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args(argv[1:]) + if not MIN_NUM_JOINTS <= args.joints <= MAX_NUM_JOINTS: + parser.error(f"--joints must be between {MIN_NUM_JOINTS} and {MAX_NUM_JOINTS}") + if args.taxels <= 0: + parser.error("--taxels must be positive") + if not math.isfinite(args.rate) or args.rate <= 0: + parser.error("--rate must be finite and positive") + if not math.isfinite(args.metadata_interval) or args.metadata_interval <= 0: + parser.error("--metadata-interval must be finite and positive") + + shape = mock_shape(args.joints, args.taxels).to_bytes() + if len(shape) > TRACKER_METADATA_MAX_SIZE: + parser.error( + f"{args.joints} joints and {args.taxels} taxels make a {len(shape)} B TactileShape; " + f"a tracker accepts at most {TRACKER_METADATA_MAX_SIZE} B" + ) + logger.info( + "Publishing %d joints and %d taxels on '%s'; TactileShape is %d B", + args.joints, + args.taxels, + args.collection_id, + len(shape), + ) + + frames = 0 + last_metadata_push = -math.inf + with ExitStack() as stack: + stack.enter_context(CloudXRLauncher.launch_context(args)) + data_tracker = TensorPushTracker( + args.collection_id, "tactile", TACTILE_SAMPLE_SIZE + ) + metadata_tracker = TensorPushTracker( + tracker_metadata_collection_id(args.collection_id), + TRACKER_METADATA_TENSOR_IDENTIFIER, + len(shape), + ) + trackers = [data_tracker, metadata_tracker] + oxr_session = stack.enter_context( + OpenXRSession( + "TactileFeeder", DeviceIOSession.get_required_extensions(trackers) + ) + ) + session = stack.enter_context( + DeviceIOSession.run(trackers, oxr_session.get_handles()) + ) + + start = time.monotonic() + try: + while args.duration <= 0 or time.monotonic() - start < args.duration: + remaining = start + frames / args.rate - time.monotonic() + if remaining > 0: + time.sleep(remaining) + session.update() + t = time.monotonic() - start + data_tracker.push( + session, TactileData(mock_forces(args.taxels, t)).to_bytes() + ) + if t - last_metadata_push >= args.metadata_interval: + metadata_tracker.push(session, shape) + last_metadata_push = t + frames += 1 + if frames % max(1, int(args.rate * 5)) == 0: + logger.info("Published %d frames", frames) + except KeyboardInterrupt: + pass + + logger.info("Published %d frames in total", frames) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/printer.py b/examples/tactile/python/isaaccapture_examples/tactile/printer.py new file mode 100644 index 0000000000..a4eeae7f68 --- /dev/null +++ b/examples/tactile/python/isaaccapture_examples/tactile/printer.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Print tactile frames from a TactileTracker and record them to MCAP. + +The tracker discovers the TactileShape published by +``python -m isaaccapture_examples.tactile.feeder`` and derives the joint and taxel counts from +it. The session records the "tactile" channels to ``--output``. ``--replay FILE`` prints a +recording instead, with no OpenXR runtime. +""" + +import argparse +import math +import sys +import time +from datetime import datetime +from pathlib import Path + +import numpy as np + +from isaaccapture.cloudxr import CloudXRLauncher +from isaaccapture.deviceio_session import ( + DeviceIOSession, + McapRecordingConfig, + McapReplayConfig, + ReplaySession, +) +from isaaccapture.deviceio_trackers import TactileTracker +from isaaccapture.oxr import OpenXRSession + +from .common import DEFAULT_COLLECTION_ID + +GREEN = "\033[32m" +RESET = "\033[0m" + +# Replay is unpaced, so use records rather than wall time to limit terminal output. +REPLAY_PRINT_EVERY_RECORDS = 30 +REPLAY_EOF_STEPS = 30 + + +def print_frame(elapsed: float | None, index: int, forces: np.ndarray) -> None: + first, last = forces[0], forces[-1] + prefix = ( + f"[replay] record {index:6d}" + if elapsed is None + else f"[{elapsed:6.2f}s] step {index:6d}" + ) + print( + f"{prefix} forces: " + f"({first[0]:.2f}, {first[1]:.2f}, {first[2]:.2f}) ... " + f"({last[0]:.2f}, {last[1]:.2f}, {last[2]:.2f})" + ) + + +def run(session, tracker: TactileTracker, args: argparse.Namespace) -> int: + start = time.monotonic() + shape_seen = False + expected_taxels = None + steps = empty = 0 + last_print = -1.0 + last_printed_step = 0 + last_replay_forces = None + while args.duration <= 0 or time.monotonic() - start < args.duration: + session.update() + elapsed = time.monotonic() - start + + shape = tracker.get_metadata(session) + if not shape_seen and shape is not None: + expected_taxels = len(shape.taxel_positions) + print( + f"{GREEN}[{elapsed:6.2f}s] (shape received) " + f"{len(shape.joint_parents)} joints, " + f"{expected_taxels} taxels{RESET}" + ) + shape_seen = True + + data = tracker.get_data(session) + if data is None: + empty += 1 + if args.replay and empty >= REPLAY_EOF_STEPS: + break + else: + empty = 0 + steps += 1 + force_array = np.asarray(data.taxel_force) + if expected_taxels is not None and len(force_array) != expected_taxels: + raise ValueError( + f"frame has {len(force_array)} taxels, " + f"TactileShape declares {expected_taxels}" + ) + if args.replay: + last_replay_forces = force_array.copy() + if steps == 1 or steps % REPLAY_PRINT_EVERY_RECORDS == 0: + print_frame(None, steps, force_array) + last_printed_step = steps + elif elapsed - last_print >= args.print_interval: + print_frame(elapsed, steps, force_array) + last_print = elapsed + if not args.replay: + time.sleep(0.005) + + if args.replay and last_replay_forces is not None and last_printed_step != steps: + print_frame(None, steps, last_replay_forces) + + if args.replay: + print(f"Replayed {steps} tactile records.") + else: + print(f"Processed {steps} session steps containing tactile data.") + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--collection-id", default=DEFAULT_COLLECTION_ID) + parser.add_argument( + "--duration", + type=float, + default=30.0, + help="Seconds to run; 0 runs until Ctrl+C", + ) + parser.add_argument( + "--print-interval", + type=float, + default=0.5, + help="Seconds between live prints; replay prints every 30 records", + ) + parser.add_argument( + "--output", + type=Path, + help="MCAP to record to (default: ./recordings/tactile_.mcap)", + ) + parser.add_argument( + "--replay", type=Path, help="Print this recording instead of live data" + ) + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args(argv[1:]) + if not math.isfinite(args.print_interval) or args.print_interval <= 0: + parser.error("--print-interval must be finite and positive") + + # Shape sizes belong to producer metadata, not receiver configuration. + tracker = TactileTracker(args.collection_id) + + if args.replay: + config = McapReplayConfig( + str(args.replay), tracker_names=[(tracker, "tactile")] + ) + with ReplaySession.run(config) as session: + return run(session, tracker, args) + + output = ( + args.output + or Path.cwd() / "recordings" / f"tactile_{datetime.now():%Y%m%d_%H%M%S}.mcap" + ) + output.parent.mkdir(parents=True, exist_ok=True) + trackers = [tracker] + recording = McapRecordingConfig(str(output), tracker_names=[(tracker, "tactile")]) + print(f"Recording to {output}. Start the feeder; Ctrl+C stops.") + try: + with ( + CloudXRLauncher.launch_context(args), + OpenXRSession( + "TactilePrinter", + DeviceIOSession.get_required_extensions(trackers), + ) as oxr_session, + DeviceIOSession.run( + trackers, oxr_session.get_handles(), recording + ) as session, + ): + run(session, tracker, args) + except KeyboardInterrupt: + pass + print(f"Recorded {output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 24b547df93007ebc145121b46caee8b3fa768eb1 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 19 Oct 2026 14:56:14 -0700 Subject: [PATCH 03/45] Copy the tactile shape out of TactileSource TactileSource published the TactileShape arrays as zero-copy views of the metadata buffer the tracker adopted for the session. A consumer that modified one in place changed the adopted bytes, so the producer's next identical metadata push no longer matched and the session update threw. The docstring also claimed the arrays were copies. Publish copies of the shape arrays. Forces stay per-frame views, as in the other DeviceIO sources. Signed-off-by: Lotus Li Co-Authored-By: Claude Opus 5.5 --- .../deviceio_source_nodes/tactile_source.py | 9 +++++---- .../core/retargeting_engine/test_tactile_sources.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index b82955c33e..0bbddb02b6 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -145,7 +145,7 @@ def output_spec(self) -> RetargeterIOType: } def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: - """Copy the forces and shape out, or call ``set_none()`` on whichever is absent.""" + """Publish the frame's forces and a copy of the shape; ``set_none()`` whichever is absent.""" data: "TactileData | None" = inputs["deviceio_tactile"][0] shape: "TactileShape | None" = inputs["deviceio_tactile"][1] @@ -172,6 +172,7 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N ) for index, tensor_type in enumerate(shape_out.group_type.types): value = getattr(shape, tensor_type.name) - if value.dtype == np.uint32: - value = value.astype(np.int32) - shape_out[index] = value + # Copied: the views alias the metadata the tracker holds and compares later pushes to. + shape_out[index] = value.astype( + np.int32 if value.dtype == np.uint32 else value.dtype + ) diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 068bec1401..5b4e37652d 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -120,6 +120,19 @@ def test_active_conversion(self): np.tile([0.0, 0.0, 1.0], (NUM_TAXELS, 1)), ) + def test_shape_outputs_do_not_alias_the_metadata(self): + src = _source() + shape = _shape() + encoded = shape.to_bytes() + outputs = _outputs(src) + src.compute(_inputs(src, None, shape), outputs) + + group = outputs[TactileSource.SHAPE] + for index in TactileShapeIndex: + array = np.asarray(group[index]) + array[...] = 7 + assert shape.to_bytes() == encoded + def test_shape_without_data(self): src = _source() outputs = _outputs(src) From dfa2891f715a74d347982c6b7eeeae0d93a3bb55 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 19 Oct 2026 14:56:52 -0700 Subject: [PATCH 04/45] Describe TactileDataRecord without a nonexistent serialize_all() The comment said trackers serialize into Record types in serialize_all(), which nothing in the tree defines. Describe the record as the other recent schemas do. Signed-off-by: Lotus Li Co-Authored-By: Claude Opus 5.5 --- src/core/schema/fbs/tactile.fbs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/schema/fbs/tactile.fbs b/src/core/schema/fbs/tactile.fbs index c73745c585..780226a13b 100644 --- a/src/core/schema/fbs/tactile.fbs +++ b/src/core/schema/fbs/tactile.fbs @@ -13,8 +13,7 @@ table TactileData { taxel_force: [Point] (id: 0); } -// MCAP recording wrapper. Record types are the root types written to MCAP -// channels; trackers serialize into Record types in serialize_all(). +// MCAP recording wrapper for TactileData. table TactileDataRecord { data: TactileData (id: 0); timestamp: DeviceDataTimestamp (id: 1); From e9d43b16571cab9dbbd58607e2a23fceb3c64c93 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 19 Oct 2026 14:57:20 -0700 Subject: [PATCH 05/45] Describe the TactileShape test fixture as it is built shape_arrays() skins every taxel to the last joint alone, with weight 1 in the first slot; its docstring said the last two joints. Signed-off-by: Lotus Li Co-Authored-By: Claude Opus 5.5 --- tests/python/core/schema/test_tactile_shape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/core/schema/test_tactile_shape.py b/tests/python/core/schema/test_tactile_shape.py index 733d5a8534..9bac9efee9 100644 --- a/tests/python/core/schema/test_tactile_shape.py +++ b/tests/python/core/schema/test_tactile_shape.py @@ -12,7 +12,7 @@ def shape_arrays(num_joints: int = 2, num_taxels: int = 3) -> dict: - """A wrist-rooted chain with every taxel skinned to its last two joints.""" + """A joint chain rooted at joint 0, with every taxel skinned to its last joint.""" indices = np.zeros((num_taxels, 4), np.uint32) indices[:, 0] = num_joints - 1 weights = np.zeros((num_taxels, 4), np.float32) From aee5aa89d2c895bbb173346a2446ff7c8e7dc507 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 21 Oct 2026 14:13:11 -0700 Subject: [PATCH 06/45] Improve schema to be extensible Signed-off-by: Lotus Li --- docs/source/references/generated_trackers.rst | 7 +- examples/tactile/README.md | 19 +- .../isaaccapture_examples/tactile/feeder.py | 43 ++-- .../isaaccapture_examples/tactile/printer.py | 13 +- src/core/codegen/templates.py | 2 +- src/core/codegen/test_manifest.py | 19 +- src/core/deviceio_trackers/trackers.toml | 7 +- src/core/schema/AGENTS.md | 3 + src/core/schema/fbs/tactile.fbs | 22 -- src/core/schema/fbs/tactile_array.fbs | 23 ++ src/core/schema/fbs/tactile_metadata.fbs | 62 +++++ src/core/schema/fbs/tactile_shape.fbs | 46 ---- src/core/schema/golden/tactile.bfbs | Bin 1112 -> 0 bytes src/core/schema/golden/tactile_array.bfbs | Bin 0 -> 1124 bytes src/core/schema/golden/tactile_metadata.bfbs | Bin 0 -> 4968 bytes src/core/schema/golden/tactile_shape.bfbs | Bin 1664 -> 0 bytes src/core/schema/python/CMakeLists.txt | 4 +- src/core/schema/python/schema_module.cpp | 10 +- ...le_bindings.h => tactile_array_bindings.h} | 22 +- .../schema/python/tactile_metadata_bindings.h | 189 +++++++++++++++ .../schema/python/tactile_shape_bindings.h | 218 ------------------ .../deviceio_tensor_types.py | 16 +- .../deviceio_source_nodes/tactile_source.py | 64 ++--- src/python/isaaccapture/schema/__init__.py | 20 +- .../test_replay_session.cpp | 38 +-- tests/cpp/core/schema/CMakeLists.txt | 4 +- ...est_tactile.cpp => test_tactile_array.cpp} | 34 +-- .../cpp/core/schema/test_tactile_metadata.cpp | 78 +++++++ tests/cpp/core/schema/test_tactile_shape.cpp | 77 ------- .../test_tactile_sources.py | 47 ++-- ...{test_tactile.py => test_tactile_array.py} | 30 ++- .../core/schema/test_tactile_metadata.py | 123 ++++++++++ .../python/core/schema/test_tactile_shape.py | 80 ------- 33 files changed, 705 insertions(+), 615 deletions(-) delete mode 100644 src/core/schema/fbs/tactile.fbs create mode 100644 src/core/schema/fbs/tactile_array.fbs create mode 100644 src/core/schema/fbs/tactile_metadata.fbs delete mode 100644 src/core/schema/fbs/tactile_shape.fbs delete mode 100644 src/core/schema/golden/tactile.bfbs create mode 100644 src/core/schema/golden/tactile_array.bfbs create mode 100644 src/core/schema/golden/tactile_metadata.bfbs delete mode 100644 src/core/schema/golden/tactile_shape.bfbs rename src/core/schema/python/{tactile_bindings.h => tactile_array_bindings.h} (64%) create mode 100644 src/core/schema/python/tactile_metadata_bindings.h delete mode 100644 src/core/schema/python/tactile_shape_bindings.h rename tests/cpp/core/schema/{test_tactile.cpp => test_tactile_array.cpp} (53%) create mode 100644 tests/cpp/core/schema/test_tactile_metadata.cpp delete mode 100644 tests/cpp/core/schema/test_tactile_shape.cpp rename tests/python/core/schema/{test_tactile.py => test_tactile_array.py} (60%) create mode 100644 tests/python/core/schema/test_tactile_metadata.py delete mode 100644 tests/python/core/schema/test_tactile_shape.py diff --git a/docs/source/references/generated_trackers.rst b/docs/source/references/generated_trackers.rst index 1147ed9ce6..0e9d55533f 100644 --- a/docs/source/references/generated_trackers.rst +++ b/docs/source/references/generated_trackers.rst @@ -85,11 +85,12 @@ example) declares that description as the tracker's ``metadata`` table: [[tracker]] name = "tactile" - table = "TactileData" - metadata = "TactileShape" + table = "TactileDataArray" + schema = "tactile_array" + metadata = "TactileMetadata" The metadata table is the ``root_type`` of a schema of its own, named after the table in snake case -(``tactile_shape.fbs``), with no Record wrapper: MCAP records the table itself. Like the data table, +(``tactile_metadata.fbs``), with no Record wrapper: MCAP records the table itself. Like the data table, it needs a hand-written ``schema/python/*_bindings.h`` before ``get_metadata()`` can return it to Python. Only ``direction = "pull"`` supports ``metadata``. diff --git a/examples/tactile/README.md b/examples/tactile/README.md index 79d66e10d6..a974d067b3 100644 --- a/examples/tactile/README.md +++ b/examples/tactile/README.md @@ -8,15 +8,15 @@ SPDX-License-Identifier: Apache-2.0 A mock tactile device and a consumer for exercising the tactile tracker without hardware. - `feeder` publishes a 26-joint hand with 200 taxels by default. Each frame carries one - `TactileData`; the `TactileShape` metadata goes whole when the feeder starts and every second - after that. It publishes through `TensorPushTracker` on a `DeviceIOSession`, as a tactile - plugin would through `SchemaPusher`. + `TactileDataArray`; it publishes `TactileMetadata` when it starts and every second after that. + It publishes through `TensorPushTracker` on a `DeviceIOSession`, as a tactile plugin would + through `SchemaPusher`. - `printer` reads one `TactileTracker` directly. It discovers the joint and taxel counts from - `TactileShape`, prints the first and last taxel forces, and records the session to MCAP. + `TactileMetadata`, prints the first and last taxel forces, and records the session to MCAP. ```text -feeder -> "tactile" (TactileData, every frame) - -> "tactile/metadata" (TactileShape, at start and every --metadata-interval s) +feeder -> "tactile" (TactileDataArray, every frame) + -> "tactile/metadata" (TactileMetadata, at start and every --metadata-interval s) -> TactileTracker -> printer, and tactile/* MCAP channels ``` @@ -46,5 +46,10 @@ python -m isaaccapture_examples.tactile.printer --replay recordings/tactile_/metadata" when the feeder starts and every ``--metadata-interval`` seconds after that. Pair it with ``python -m isaaccapture_examples.tactile.printer``. """ @@ -24,8 +24,11 @@ from isaaccapture.schema import ( TRACKER_METADATA_MAX_SIZE, TRACKER_METADATA_TENSOR_IDENTIFIER, - TactileData, - TactileShape, + TactileDataArray, + TactileMetadata, + HandJoint, + HandSkeletonRef, + LBS, tracker_metadata_collection_id, ) @@ -46,11 +49,9 @@ FINGER_DISTAL_JOINTS = (4, 9, 14, 19, 24) -def mock_shape(num_joints: int, num_taxels: int) -> TactileShape: +def mock_shape(num_joints: int, num_taxels: int) -> TactileMetadata: """Build a flat hand in hand-root space with taxels on its distal segments.""" positions = np.zeros((num_joints, 3), np.float32) - parents = np.full(num_joints, WRIST, np.uint32) - parents[WRIST] = TactileShape.ROOT_PARENT positions[0] = (0.0, 0.05, 0.0) taxel_joints = [] @@ -67,8 +68,6 @@ def mock_shape(num_joints: int, num_taxels: int) -> TactileShape: 0.02 + 0.025 * (joint - first), 0.0, ) - if joint > first: - parents[joint] = joint - 1 taxel_joints.append(min(FINGER_DISTAL_JOINTS[finger], last)) if not taxel_joints: @@ -89,15 +88,15 @@ def mock_shape(num_joints: int, num_taxels: int) -> TactileShape: joint_indices[taxel] = joint joint_weights[taxel, 0] = 1.0 - return TactileShape( - joint_bind_positions=positions, - joint_bind_orientations=np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1)), - joint_parents=parents, - taxel_positions=taxel_positions, - taxel_normals=np.tile([0.0, 0.0, -1.0], (num_taxels, 1)), - taxel_tangents=np.tile([1.0, 0.0, 0.0], (num_taxels, 1)), - taxel_joint_indices=joint_indices, - taxel_joint_weights=joint_weights, + segment_poses = np.column_stack( + (positions, np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1))) + ) + taxel_poses = np.column_stack( + (taxel_positions, np.tile([1.0, 0.0, 0.0, 0.0], (num_taxels, 1))) + ) + return TactileMetadata( + algorithm=LBS(segment_poses, taxel_poses, joint_indices, joint_weights), + segments=HandSkeletonRef([HandJoint(i) for i in range(num_joints)]), ) @@ -120,7 +119,7 @@ def main(argv: list[str]) -> int: "--metadata-interval", type=float, default=1.0, - help="Seconds between pushes of the TactileShape", + help="Seconds between pushes of the TactileMetadata", ) parser.add_argument( "--duration", @@ -142,11 +141,11 @@ def main(argv: list[str]) -> int: shape = mock_shape(args.joints, args.taxels).to_bytes() if len(shape) > TRACKER_METADATA_MAX_SIZE: parser.error( - f"{args.joints} joints and {args.taxels} taxels make a {len(shape)} B TactileShape; " + f"{args.joints} joints and {args.taxels} taxels make a {len(shape)} B TactileMetadata; " f"a tracker accepts at most {TRACKER_METADATA_MAX_SIZE} B" ) logger.info( - "Publishing %d joints and %d taxels on '%s'; TactileShape is %d B", + "Publishing %d joints and %d taxels on '%s'; TactileMetadata is %d B", args.joints, args.taxels, args.collection_id, @@ -184,7 +183,7 @@ def main(argv: list[str]) -> int: session.update() t = time.monotonic() - start data_tracker.push( - session, TactileData(mock_forces(args.taxels, t)).to_bytes() + session, TactileDataArray(mock_forces(args.taxels, t)).to_bytes() ) if t - last_metadata_push >= args.metadata_interval: metadata_tracker.push(session, shape) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/printer.py b/examples/tactile/python/isaaccapture_examples/tactile/printer.py index a4eeae7f68..eea404db9b 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/printer.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/printer.py @@ -3,7 +3,7 @@ """Print tactile frames from a TactileTracker and record them to MCAP. -The tracker discovers the TactileShape published by +The tracker discovers the TactileMetadata published by ``python -m isaaccapture_examples.tactile.feeder`` and derives the joint and taxel counts from it. The session records the "tactile" channels to ``--output``. ``--replay FILE`` prints a recording instead, with no OpenXR runtime. @@ -66,10 +66,15 @@ def run(session, tracker: TactileTracker, args: argparse.Namespace) -> int: shape = tracker.get_metadata(session) if not shape_seen and shape is not None: - expected_taxels = len(shape.taxel_positions) + algorithm, segments = shape.algorithm, shape.segments + if algorithm is None or segments is None: + raise ValueError( + "TactileMetadata must specify an algorithm and segments" + ) + expected_taxels = len(algorithm.taxel_bind_pose) print( f"{GREEN}[{elapsed:6.2f}s] (shape received) " - f"{len(shape.joint_parents)} joints, " + f"{len(segments.joints)} joints, " f"{expected_taxels} taxels{RESET}" ) shape_seen = True @@ -86,7 +91,7 @@ def run(session, tracker: TactileTracker, args: argparse.Namespace) -> int: if expected_taxels is not None and len(force_array) != expected_taxels: raise ValueError( f"frame has {len(force_array)} taxels, " - f"TactileShape declares {expected_taxels}" + f"TactileMetadata declares {expected_taxels}" ) if args.replay: last_replay_forces = force_array.copy() diff --git a/src/core/codegen/templates.py b/src/core/codegen/templates.py index e755929b98..026ced75a9 100644 --- a/src/core/codegen/templates.py +++ b/src/core/codegen/templates.py @@ -21,7 +21,7 @@ def _header() -> str: def camel_to_snake(value: str) -> str: - """Convert CamelCase to snake_case (TactileShape -> tactile_shape).""" + """Convert CamelCase to snake_case (TactileMetadata -> tactile_metadata).""" return re.sub( r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", "_", value ).lower() diff --git a/src/core/codegen/test_manifest.py b/src/core/codegen/test_manifest.py index 5fdc5aaf51..df705836d5 100644 --- a/src/core/codegen/test_manifest.py +++ b/src/core/codegen/test_manifest.py @@ -213,7 +213,12 @@ class MetadataTest(unittest.TestCase): def _context(self, **overrides): from templates import enrich_context - raw = {"name": "tactile", "table": "TactileData", **overrides} + raw = { + "name": "tactile", + "table": "TactileDataArray", + "schema": "tactile_array", + **overrides, + } return enrich_context(resolve_tracker_entry(raw, load_defaults(DEFAULTS_PATH))) def test_unset_by_default(self) -> None: @@ -223,11 +228,13 @@ def test_names_derive_from_the_table_and_channel(self) -> None: from templates import MetadataGenContext self.assertEqual( - self._context(metadata="TactileShape").metadata, - MetadataGenContext("TactileShape", "tactile_shape", "tactile_metadata"), + self._context(metadata="TactileMetadata").metadata, + MetadataGenContext( + "TactileMetadata", "tactile_metadata", "tactile_metadata" + ), ) self.assertEqual( - self._context(metadata="TactileShape", channel="glove").metadata.channel, + self._context(metadata="TactileMetadata", channel="glove").metadata.channel, "glove_metadata", ) @@ -241,7 +248,7 @@ def test_invalid_values_are_rejected(self) -> None: for overrides in ( {"metadata": ""}, {"metadata": 1}, - {"metadata": "TactileShape", "direction": "push"}, + {"metadata": "TactileMetadata", "direction": "push"}, ): with self.subTest(**overrides), self.assertRaises(ValueError) as ctx: self._context(**overrides) @@ -255,7 +262,7 @@ def test_every_placeholder_is_filled(self) -> None: render_for_entry, ) - for metadata in (None, "TactileShape"): + for metadata in (None, "TactileMetadata"): ctx = self._context(**({"metadata": metadata} if metadata else {})) files = render_for_entry(ctx) for fragment in ( diff --git a/src/core/deviceio_trackers/trackers.toml b/src/core/deviceio_trackers/trackers.toml index 7f9ed90439..5220ecbde3 100644 --- a/src/core/deviceio_trackers/trackers.toml +++ b/src/core/deviceio_trackers/trackers.toml @@ -113,9 +113,10 @@ table = "GamepadSnapshot" python_accessor = "get_gamepad_data" # Defaults fit about 1300 taxels (12 B each); larger devices pass max_flatbuffer_size. The -# TactileShape metadata must fit TRACKER_METADATA_MAX_SIZE, which holds 26 joints and 1000 taxels. +# TactileMetadata must also fit the separate TRACKER_METADATA_MAX_SIZE limit. [[tracker]] name = "tactile" -table = "TactileData" -metadata = "TactileShape" +table = "TactileDataArray" +schema = "tactile_array" +metadata = "TactileMetadata" max_flatbuffer_size = 16384 diff --git a/src/core/schema/AGENTS.md b/src/core/schema/AGENTS.md index 2977b70417..e44c412fcb 100644 --- a/src/core/schema/AGENTS.md +++ b/src/core/schema/AGENTS.md @@ -29,3 +29,6 @@ fails. The short version: existing offset survives; `check_schema_compat()` catches it on the struct's size. - **Never refresh `golden/` to clear a failing test.** The failure is the signal. Refresh only once breaking existing recordings is a decision you have made and stated. +- For tactile metadata union changes, validate C++ and Python against the same freshly + generated schemas. Run validators and hooks through the documented uv-managed Python + environment; the macOS system Python may be too old for the hook dependencies. diff --git a/src/core/schema/fbs/tactile.fbs b/src/core/schema/fbs/tactile.fbs deleted file mode 100644 index 780226a13b..0000000000 --- a/src/core/schema/fbs/tactile.fbs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -include "point.fbs"; -include "timestamp.fbs"; - -namespace core; - -// Per-frame tactile reading. The taxel layout it indexes is TactileShape (tactile_shape.fbs), -// the tactile tracker's metadata. -table TactileData { - // Force vector per taxel, in TactileShape taxel order. - taxel_force: [Point] (id: 0); -} - -// MCAP recording wrapper for TactileData. -table TactileDataRecord { - data: TactileData (id: 0); - timestamp: DeviceDataTimestamp (id: 1); -} - -root_type TactileDataRecord; diff --git a/src/core/schema/fbs/tactile_array.fbs b/src/core/schema/fbs/tactile_array.fbs new file mode 100644 index 0000000000..97b6d39776 --- /dev/null +++ b/src/core/schema/fbs/tactile_array.fbs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "point.fbs"; +include "timestamp.fbs"; + +namespace core; + +// Per-frame tactile forces. TactileMetadata (tactile_metadata.fbs) defines the +// taxel order and reference geometry for the same tracker. +table TactileDataArray { + // Force vectors in newtons, expressed in taxel-local axes. For LBS, taxel_bind_pose + // defines these axes at bind; the vector length equals the metadata taxel count. + taxel_force: [Point] (id: 0); +} + +// MCAP recording wrapper for TactileDataArray. +table TactileDataArrayRecord { + data: TactileDataArray (id: 0); + timestamp: DeviceDataTimestamp (id: 1); +} + +root_type TactileDataArrayRecord; diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs new file mode 100644 index 0000000000..432e518975 --- /dev/null +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "hand.fbs"; +include "pose.fbs"; + +namespace core; + +table HandSkeletonRef { + // Ordered, unique HandJoint identifiers; NUM_JOINTS is not a joint. + // Segment indices address this vector, not HandJoint ordinals. Bind and current + // segment poses map joint-local coordinates into the same hand-root frame. + joints: [HandJoint] (id: 0); +} + +union SegmentSource { + HandSkeletonRef = 1, +} + +struct UInt4 { + values: [uint32:4]; +} + +struct Float4 { + values: [float:4]; +} + +table LBS { + // Segment-to-source-frame bind poses, in SegmentSource order. Positions are in + // meters and orientations are unit quaternions (x, y, z, w). + segment_bind_pose: [Pose] (id: 0); + + // Taxel-local-to-source-frame bind poses, in TactileDataArray.taxel_force order. + // Uses the same units and quaternion convention as segment_bind_pose. + taxel_bind_pose: [Pose] (id: 1); + + // One four-slot row per taxel; all taxel_* vectors have the same length. + // Every slot indexes segment_bind_pose, including slots whose weight is zero. + taxel_segment_indices: [UInt4] (id: 2); + + // Finite, nonnegative weights, summing to one per taxel. Bind positions move as + // p(t) = sum_k w_k * T_current[j_k] * inverse(T_bind[j_k]) * p_bind. + // LBS blends transformed points; it does not define a rigid animated taxel frame. + taxel_segment_weights: [Float4] (id: 3); +} + +union AlgorithmSource { + LBS = 1, +} + +// Static layout published as tracker metadata, without a timestamp or Record wrapper. +// A usable layout sets both unions; lengths, indices and weights require semantic +// validation by producers/consumers, beyond FlatBuffers buffer verification. +table TactileMetadata { + // The implicit algorithm_type discriminator occupies id 0. + algorithm: AlgorithmSource (id: 1); + + // The implicit segments_type discriminator occupies id 2. + segments: SegmentSource (id: 3); +} + +root_type TactileMetadata; diff --git a/src/core/schema/fbs/tactile_shape.fbs b/src/core/schema/fbs/tactile_shape.fbs deleted file mode 100644 index d4f5fb79d6..0000000000 --- a/src/core/schema/fbs/tactile_shape.fbs +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -include "pose.fbs"; - -namespace core; - -// One joint of the hand skeleton at the reference bind pose. -struct TactileJoint { - // Pose in hand-root space. Position in meters, unit quaternion. - bind_pose: Pose; - - // Parent joint ordinal; the root holds 0x7FFFFFFF (XR_HAND_JOINT_MAX_ENUM_EXT). - // The hierarchy is acyclic. - parent: uint32; -} - -// One taxel at the reference bind pose, in hand-root space. -struct TactileTaxel { - // Position in meters; unit normal; unit tangent perpendicular to the normal. - position: Point; - normal: Point; - tangent: Point; - - // Linear blend skinning over four slots: joint_indices[k], an XrHandJointEXT ordinal (HandJoint - // in hand.fbs), weighted by joint_weights[k]. Every slot names a valid joint, zero-weight slots - // included. Weights are finite and nonnegative, 1-4 of them nonzero, summing to 1. A taxel moves as - // p_current = sum_k w_k * T_current[j_k] * inverse(T_bind[j_k]) * p_bind - // with T_bind from the joint's bind_pose and T_current the current joint-to-hand-root - // transform, both rigid. It blends transformed points, not poses. - joint_indices: [uint32:4]; - joint_weights: [float:4]; -} - -// Hand skeleton and taxel layout at the reference bind pose, for the device that publishes -// TactileData. `joints` is indexed by XrHandJointEXT ordinal; `taxels` is in -// TactileData.taxel_force order. -// -// Published as the metadata of the tactile tracker (`metadata` in trackers.toml), not as -// samples: it carries no timestamp, and the tracker holds one shape for the whole session. -table TactileShape { - joints: [TactileJoint] (id: 0); - taxels: [TactileTaxel] (id: 1); -} - -root_type TactileShape; diff --git a/src/core/schema/golden/tactile.bfbs b/src/core/schema/golden/tactile.bfbs deleted file mode 100644 index e476e46c4b90679853988754a53d9e9289546cc7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1112 zcmZ{jy>1gh6ou~^V>`rP7Z#$hkdacPKruy%6jV`!C@2u5pj%DW8)d~h8`-N65Jet^ zl9GZ4px^;grl6qUAyOdBcV;ujtYB8hGxzV@pBWF$Z1>6T%YluoX&viY-v)LM?U3DU z8$1LfAo(10z$5U#)x>ZanQh!4mi-wrN6#f@*j6o4DN!Yn5A3$E-AA--E*aTk)oMY9jU*L-NI{iwdvNcD6*FC zUimJ_#nzKWxz<3LP1AYlrYD?_WRIG*$yu}M^R?MEA63}*yY*C`;>9_k9;pX>ns?QS zG@ARLSqJ~y=*Hu+YG&*3V2&)?4iJC6S5)+(9yOts#Fcfk>a$jJi>m*w`A$zc#IO!Z zcb4YyVKGZm`})hzBvSqjptx|S8WgpL3HfSu@1H>qwEMz2#6ey-}K|ujggeX#=kP_+GoG+5aITxKziGwJ3 z6iOa|l7fN-@li$5-Y6rdv|AM|NUoXuV-f4ueRTIt#5T}S=&PE+6LAh zv)K?l0eztTXV3yq!T&}b$2=hB7GU0G&(d>=S^nL>i^VxUtu@}U-?-e@*oPeW&c482 zXI;e&7mGy?e;eSoIZ*rzyZ{eCoaM>ZzKe@-n(Vm3z07lWwwGWY`K**<%kB_)bu{t$ z5d_xo70{i(xR0qLkM3Io#dwlT3pY7t{a*f{Zkw!CpO9aZSv8W%(9hP9Mx7Vu2u(6~ z`Kjh*HnOPpWo;ibSMm2)4+ceCo z>)P*jq(cnrpm4J!jSjOsPVDDDUz^|=cInl*s=q22nyX5Ay?O)RKnfJESg(60IT^>k zRr}QeT=oV28u}~FPh~IGbdNNtUsKwUCXp_Ra(5byJXI8@S^NRF&EW`8R&o4T;dr>h z!8w;kaWf9G1qB6#3JMAe3KtX>l4O-vFcm+xo)yxTiF%lx$O z?f$;MKi~cBzCFsum`fjBT2*Jp%mq_4B{OVB^*nCMWTa7(^Qwl#fW?05gBwxwTw8l?s zcvsSp%u9%6TvC+W{d>3j1ex{opUP&?`|4BB2-kvZE^7qGXw=4675A>%jQK+UQzcW9$alKk@;Dp;)=L3m%f zyfg=q<9032*tNp5sHOGATstejK4#(!c6eTsu-@Qo%&Y0O&9t3`h#&m($ML@O@v>&v z5?y9KX*X6sPg`l$X|JT88E_%?yLt{j{BBFfFumBsoem#a%P?YyH%|CcDM zJsQwosMcx=bM;zPh4ksa9q%ev8-oSt&-W2VC2KsUQeCah67Pa=A+C2lXjEe%uD5#_ z1va;`I9I98*5=H2k-v95jt+YP%Q}01%PW`Hs*CdYB=YwQkGmCNw9Nu$ajrJ|yV;dx z8|QI^qfThwMmW}grMmR7W5bxqlU`orKzkACt5lr6XQCGA`#oXCH5PIqE^EJ!1=Wyr z%9?|BbV9>kz{0!5<}9cV{?o&O4Q(&N!JoxNOq^&wtQYgN{Rl%HEmkX)P#0O`hq?^y zIKmL0$K-X<5H9#(o_0ZFAtyNXm_A01B7bh5TByNZz!JB|=8POf{#YB@{RjiM9uu)4 z9{<~mliX>~A{?KO>QZHH%@@XP>6bMI=gfsx)Od)GxE&^^!(@>kYK=A%VW=60>7&CO zNB%gkL&IM1$GSLdPKP;){Bds59z_`9cbGmp%!Ix{Vcm4Z+4elbk#lYS%Hk!@$35wn zHHJMMfq5I#c!&=^9Hz&|?Z}UgO55fm3^_SWFQ;LhOhtPFa-VSLHhy)i=4~&s!JMQ%hCCt)EH;UsEtMsYJ6ev8*)cqC(t5u-{|}R zL+s37RN~CvmmEl#!w)v?x3QM2XU$f+m}W^M$r25l%zd8|aF&H=dnnyE{CZ)R*x-o% zsf2vth_lgh^iivmWYZ?|F*1k#L)FIlb4B~ixp53}&&8na4e12ioS)YAm1iw2q_tc5<`c zu%_zkteSkLsPB`vW%MD7GP7(yFg_Ha9!%U}Zu!;@SvzAZnJ{P|IWbjb4%) zNwbw)4PpJ1Ni*qr{y_R=3A~aIy{j8oc6Gx(r@n4USO?3QD-~f}1(YmGlX*L1?der} zpIeKw{n+I{f)6<4EY|OJ{rXL^W~VJo*o&IzYdw2|T(BO!^UI8c^`Tc5O>0=sC zw|d3~xx!!Qr>6dLJ;~DTwgPdPySYi}>`5Au4<-2?{(Y`)UWw0B$}{YlCyrmcXUOxW zg!>@h(V)YfQp35|E4Xw-{IfOXPn2Qff&ex$|Q!O&IbI%IUNq@latde Vsdsx5zkDl0t)t;8P`82#M1Ng$zdM1~9*JY-;q>JAw)bcm`7MW_R)9oSMgHLX#wL*l9dF)()M z*oWu?FfcIi0DS-k1|Fd!LVz@<&NH4i3-#NdQ~t_i~BFvZRRW=-8i^v~AUpo~j3PP5gNo|-O_;nXrM+Y|YSc84wC6b|zV}O3|fSYML7LVW$t~oX$U2&wG2}C&4sAPj%*+)5g1N;1t*e7(mtF z#fmQXsAmC{Nc;3f*SC_^{}Oi^h`#}~Dd$Iy^Bx@b(Sov+V{6STKkvZlz?Qym=ibj5 z2bJf)a0%IwCzyD>iq*x*1(^2D8D0SHjmq!2Iiq> mg5Pve#mVOMEyrh0)XV=__KfvrTn9GxS3mkc0sB=gsPQk*KJ=mh diff --git a/src/core/schema/python/CMakeLists.txt b/src/core/schema/python/CMakeLists.txt index c37e632705..183f172d22 100644 --- a/src/core/schema/python/CMakeLists.txt +++ b/src/core/schema/python/CMakeLists.txt @@ -28,8 +28,8 @@ pybind11_add_module(schema_py schema_module.cpp spacemouse_bindings.h steering_wheel_bindings.h - tactile_bindings.h - tactile_shape_bindings.h + tactile_array_bindings.h + tactile_metadata_bindings.h tracker_metadata_bindings.h ) diff --git a/src/core/schema/python/schema_module.cpp b/src/core/schema/python/schema_module.cpp index d904d55833..d9889f851f 100644 --- a/src/core/schema/python/schema_module.cpp +++ b/src/core/schema/python/schema_module.cpp @@ -29,8 +29,8 @@ #include "soma_hand_joint_rotations_bindings.h" #include "spacemouse_bindings.h" #include "steering_wheel_bindings.h" -#include "tactile_bindings.h" -#include "tactile_shape_bindings.h" +#include "tactile_array_bindings.h" +#include "tactile_metadata_bindings.h" #include "timestamp_bindings.h" #include "tracker_metadata_bindings.h" @@ -64,9 +64,9 @@ PYBIND11_MODULE(_schema, m) // Bind OGLO tactile glove types (OgloGloveSample table). core::bind_oglo_tactile(m); - // Bind tactile types (TactileData, TactileShape tables). - core::bind_tactile(m); - core::bind_tactile_shape(m); + // Bind tactile types (TactileDataArray, TactileMetadata tables). + core::bind_tactile_array(m); + core::bind_tactile_metadata(m); // Bind the helpers for producers of tracker metadata. core::bind_tracker_metadata(m); diff --git a/src/core/schema/python/tactile_bindings.h b/src/core/schema/python/tactile_array_bindings.h similarity index 64% rename from src/core/schema/python/tactile_bindings.h rename to src/core/schema/python/tactile_array_bindings.h index 1d83b19ee8..0a140ac284 100644 --- a/src/core/schema/python/tactile_bindings.h +++ b/src/core/schema/python/tactile_array_bindings.h @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Python bindings for the TactileData FlatBuffer schema (see tactile.fbs). -// Types: TactileData (table) and TactileDataRecord, exposed as encoded views. +// Python bindings for the TactileDataArray FlatBuffer schema (see tactile_array.fbs). +// Types: TactileDataArray (table) and TactileDataArrayRecord, exposed as encoded views. #pragma once @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include @@ -22,38 +22,38 @@ namespace py = pybind11; namespace core { -inline void bind_tactile(py::module& m) +inline void bind_tactile_array(py::module& m) { using FloatRows = py::array_t; - serialized_class(m, "TactileData", "Encoded per-frame tactile reading.") + serialized_class(m, "TactileDataArray", "Encoded per-frame tactile reading.") .def(py::init( [](const std::optional& taxel_force) { - TactileDataT native; + TactileDataArrayT native; if (taxel_force.has_value()) { native.taxel_force = structs_from_rows(*taxel_force, "taxel_force"); } - return pack(native); + return pack(native); }), py::arg("taxel_force") = py::none(), "Encode a tactile frame. taxel_force is an (N, 3) array.") .def_property_readonly( "taxel_force", [](py::object self) { - const auto& view = self.cast&>(); + const auto& view = self.cast&>(); return struct_vector_view(self, view->taxel_force()); }, "Per-taxel force as an (N, 3) float32 view aliasing this object's buffer; call .copy() to own it.") .def("__repr__", - [](const Serialized& self) + [](const Serialized& self) { const auto* forces = self->taxel_force(); - return "TactileData(taxels=" + std::to_string(forces != nullptr ? forces->size() : 0) + ")"; + return "TactileDataArray(taxels=" + std::to_string(forces != nullptr ? forces->size() : 0) + ")"; }); - bind_record(m, "TactileDataRecord", "TactileData"); + bind_record(m, "TactileDataArrayRecord", "TactileDataArray"); } } // namespace core diff --git a/src/core/schema/python/tactile_metadata_bindings.h b/src/core/schema/python/tactile_metadata_bindings.h new file mode 100644 index 0000000000..8163057860 --- /dev/null +++ b/src/core/schema/python/tactile_metadata_bindings.h @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Encoded tactile metadata views. Pose arrays use rows (x, y, z, qx, qy, qz, qw). + +#pragma once + +#include "schema_array_views.h" +#include "schema_serialized.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace core +{ + +inline void bind_tactile_metadata(py::module& m) +{ + using FloatRows = py::array_t; + using IndexRows = py::array_t; + + py::enum_(m, "SegmentSource") + .value("NONE", SegmentSource_NONE) + .value("HandSkeletonRef", SegmentSource_HandSkeletonRef); + py::enum_(m, "AlgorithmSource").value("NONE", AlgorithmSource_NONE).value("LBS", AlgorithmSource_LBS); + + serialized_class(m, "HandSkeletonRef", "Ordered hand joints referenced by tactile segments.") + .def(py::init( + [](const std::vector& joints) + { + std::array seen{}; + for (const auto joint : joints) + { + if (joint >= HandJoint_NUM_JOINTS || seen[joint]) + { + throw py::value_error("joints: expected unique HandJoint identifiers, excluding NUM_JOINTS"); + } + seen[joint] = true; + } + HandSkeletonRefT native; + native.joints = joints; + return pack(native); + }), + py::arg("joints")) + .def_property_readonly("joints", vector_field(&HandSkeletonRef::joints)); + + auto poses_from_rows = [](const FloatRows& rows, const char* name) + { + auto poses = structs_from_rows(rows, name); + for (py::ssize_t i = 0; i < rows.shape(0); ++i) + { + const float* row = rows.data(i); + for (int c = 0; c < 7; ++c) + { + if (!std::isfinite(row[c])) + { + throw py::value_error(std::string(name) + ": poses must be finite"); + } + } + const float norm2 = row[3] * row[3] + row[4] * row[4] + row[5] * row[5] + row[6] * row[6]; + if (std::abs(norm2 - 1.0f) > 1e-4f) + { + throw py::value_error(std::string(name) + ": orientations must be unit quaternions (x, y, z, w)"); + } + } + return poses; + }; + + serialized_class(m, "LBS", "Linear blend skinning bind poses and four-slot influences.") + .def(py::init( + [poses_from_rows](const FloatRows& segment_bind_pose, const FloatRows& taxel_bind_pose, + const IndexRows& taxel_segment_indices, const FloatRows& taxel_segment_weights) + { + LBST native; + native.segment_bind_pose = poses_from_rows(segment_bind_pose, "segment_bind_pose"); + native.taxel_bind_pose = poses_from_rows(taxel_bind_pose, "taxel_bind_pose"); + native.taxel_segment_indices = + structs_from_rows(taxel_segment_indices, "taxel_segment_indices"); + native.taxel_segment_weights = + structs_from_rows(taxel_segment_weights, "taxel_segment_weights"); + const size_t taxels = native.taxel_bind_pose.size(); + if (native.taxel_segment_indices.size() != taxels || native.taxel_segment_weights.size() != taxels) + { + throw py::value_error("all taxel_* arrays must have the same length"); + } + for (size_t i = 0; i < taxels; ++i) + { + float total = 0; + for (int k = 0; k < 4; ++k) + { + if (native.taxel_segment_indices[i].values()->Get(k) >= native.segment_bind_pose.size()) + { + throw py::value_error("taxel_segment_indices: slot is not a segment index"); + } + const float weight = native.taxel_segment_weights[i].values()->Get(k); + if (!std::isfinite(weight) || weight < 0) + { + throw py::value_error("taxel_segment_weights: weights must be finite and nonnegative"); + } + total += weight; + } + if (std::abs(total - 1.0f) > 1e-5f) + { + throw py::value_error("taxel_segment_weights: each row must sum to one"); + } + } + return pack(native); + }), + py::arg("segment_bind_pose"), py::arg("taxel_bind_pose"), py::arg("taxel_segment_indices"), + py::arg("taxel_segment_weights"), + "Encode bind poses as (J, 7)/(N, 7) XYZ/XYZW rows and influences as (N, 4) arrays.") + .def_property_readonly( + "segment_bind_pose", [](py::object self) + { return struct_vector_view(self, self.cast&>()->segment_bind_pose()); }) + .def_property_readonly( + "taxel_bind_pose", [](py::object self) + { return struct_vector_view(self, self.cast&>()->taxel_bind_pose()); }) + .def_property_readonly("taxel_segment_indices", + [](py::object self) { + return struct_vector_view( + self, self.cast&>()->taxel_segment_indices()); + }) + .def_property_readonly( + "taxel_segment_weights", [](py::object self) + { return struct_vector_view(self, self.cast&>()->taxel_segment_weights()); }); + + serialized_class(m, "TactileMetadata", "Static tactile layout and its segment source.") + .def(py::init( + [](const Serialized& algorithm, const Serialized& segments) + { + const auto* poses = algorithm->segment_bind_pose(); + const auto* joints = segments->joints(); + if ((poses != nullptr ? poses->size() : 0) != (joints != nullptr ? joints->size() : 0)) + { + throw py::value_error("segment_bind_pose and segments.joints must have the same length"); + } + TactileMetadataT native; + LBST lbs; + algorithm->UnPackTo(&lbs); + native.algorithm.Set(std::move(lbs)); + HandSkeletonRefT hand; + segments->UnPackTo(&hand); + native.segments.Set(std::move(hand)); + return pack(native); + }), + py::arg("algorithm"), py::arg("segments")) + .def_property_readonly("algorithm_type", field(&TactileMetadata::algorithm_type)) + .def_property_readonly("segments_type", field(&TactileMetadata::segments_type)) + .def_property_readonly("algorithm", + [](const Serialized& self) -> std::optional> + { + if (self->algorithm_type() == AlgorithmSource_NONE) + { + return std::nullopt; + } + if (const auto* algorithm = self->algorithm_as_LBS()) + { + return self.narrow(algorithm); + } + throw py::value_error("unsupported tactile algorithm"); + }) + .def_property_readonly("segments", + [](const Serialized& self) -> std::optional> + { + if (self->segments_type() == SegmentSource_NONE) + { + return std::nullopt; + } + if (const auto* segments = self->segments_as_HandSkeletonRef()) + { + return self.narrow(segments); + } + throw py::value_error("unsupported tactile segment source"); + }); +} + +} // namespace core diff --git a/src/core/schema/python/tactile_shape_bindings.h b/src/core/schema/python/tactile_shape_bindings.h deleted file mode 100644 index 2a2661840a..0000000000 --- a/src/core/schema/python/tactile_shape_bindings.h +++ /dev/null @@ -1,218 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Python bindings for the TactileShape FlatBuffer schema (see tactile_shape.fbs), exposed as an -// encoded view. Each joint and taxel attribute reads as a zero-copy NumPy view strided over the -// TactileJoint / TactileTaxel structs; see schema_array_views.h for the aliasing caveat. - -#pragma once - -#include "schema_array_views.h" -#include "schema_serialized.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace py = pybind11; - -namespace core -{ - -// TactileJoint.parent of a root joint: XR_HAND_JOINT_MAX_ENUM_EXT. -constexpr uint32_t TACTILE_ROOT_PARENT = 0x7FFFFFFF; - -inline void bind_tactile_shape(py::module& m) -{ - using FloatRows = py::array_t; - using IndexRows = py::array_t; - - constexpr auto joint_stride = static_cast(sizeof(TactileJoint)); - constexpr auto taxel_stride = static_cast(sizeof(TactileTaxel)); - - // (count, width) view of one field across a vector of structs; absent reads as (0, width). - auto struct_field_view = [](py::object self, const auto* first, py::ssize_t stride, size_t count, py::ssize_t width) - { - using T = std::remove_cv_t>; - if (count == 0) - { - return width > 0 ? py::array_t(std::vector{ 0, width }) : py::array_t(py::ssize_t{ 0 }); - } - return strided_field_view(self, first, stride, static_cast(count), width); - }; - auto joints_of = [](py::object self) { return self.cast&>()->joints(); }; - auto taxels_of = [](py::object self) { return self.cast&>()->taxels(); }; - - auto shape = serialized_class(m, "TactileShape", "Encoded tactile skeleton and taxel layout."); - shape - .def(py::init( - [](const FloatRows& joint_bind_positions, const FloatRows& joint_bind_orientations, - const IndexRows& joint_parents, const FloatRows& taxel_positions, const FloatRows& taxel_normals, - const FloatRows& taxel_tangents, const IndexRows& taxel_joint_indices, - const FloatRows& taxel_joint_weights) - { - const auto positions = - structs_from_rows(joint_bind_positions, "joint_bind_positions"); - const auto orientations = - structs_from_rows(joint_bind_orientations, "joint_bind_orientations"); - if (joint_parents.ndim() != 1) - { - throw py::value_error("joint_parents: expected an (J,) array"); - } - const size_t joints = positions.size(); - if (orientations.size() != joints || static_cast(joint_parents.shape(0)) != joints) - { - throw py::value_error( - "joint_bind_positions, joint_bind_orientations and joint_parents must have the same length"); - } - - TactileShapeT native; - native.joints.reserve(joints); - for (size_t j = 0; j < joints; ++j) - { - const uint32_t parent = joint_parents.data()[j]; - if (parent != TACTILE_ROOT_PARENT && parent >= joints) - { - throw py::value_error("joint_parents: " + std::to_string(parent) + - " is neither a joint index nor ROOT_PARENT"); - } - native.joints.emplace_back(Pose(positions[j], orientations[j]), parent); - } - - const auto points = structs_from_rows(taxel_positions, "taxel_positions"); - const auto normals = structs_from_rows(taxel_normals, "taxel_normals"); - const auto tangents = structs_from_rows(taxel_tangents, "taxel_tangents"); - const auto indices = structs_from_rows, uint32_t, 4>( - taxel_joint_indices, "taxel_joint_indices"); - const auto weights = - structs_from_rows, float, 4>(taxel_joint_weights, "taxel_joint_weights"); - const size_t taxels = points.size(); - if (normals.size() != taxels || tangents.size() != taxels || indices.size() != taxels || - weights.size() != taxels) - { - throw py::value_error("every taxel_* array must have the same length"); - } - native.taxels.reserve(taxels); - for (size_t t = 0; t < taxels; ++t) - { - for (uint32_t joint : indices[t]) - { - if (joint >= joints) - { - throw py::value_error("taxel_joint_indices: " + std::to_string(joint) + - " is not a joint index"); - } - } - native.taxels.emplace_back(points[t], normals[t], tangents[t], - ::flatbuffers::span(indices[t].data(), 4), - ::flatbuffers::span(weights[t].data(), 4)); - } - return pack(native); - }), - py::arg("joint_bind_positions"), py::arg("joint_bind_orientations"), py::arg("joint_parents"), - py::arg("taxel_positions"), py::arg("taxel_normals"), py::arg("taxel_tangents"), - py::arg("taxel_joint_indices"), py::arg("taxel_joint_weights"), - "Encode a tactile shape. Joint arrays are (J, 3), (J, 4) and (J,); taxel arrays are (N, 3) or " - "(N, 4). Mismatched lengths and out-of-range joint indices raise ValueError; weights are not " - "checked.") - .def_property_readonly( - "joint_bind_positions", - [=](py::object self) - { - const auto* joints = joints_of(self); - const size_t count = joints != nullptr ? joints->size() : 0; - const float* first = - count > 0 ? reinterpret_cast(&joints->Get(0)->bind_pose().position()) : nullptr; - return struct_field_view(self, first, joint_stride, count, 3); - }, - "Joint bind positions as a (J, 3) float32 strided view.") - .def_property_readonly( - "joint_bind_orientations", - [=](py::object self) - { - const auto* joints = joints_of(self); - const size_t count = joints != nullptr ? joints->size() : 0; - const float* first = - count > 0 ? reinterpret_cast(&joints->Get(0)->bind_pose().orientation()) : nullptr; - return struct_field_view(self, first, joint_stride, count, 4); - }, - "Joint bind orientations (XYZW) as a (J, 4) float32 strided view.") - .def_property_readonly( - "joint_parents", - [=, offset = FBS_FIELD_OFFSET(TactileJoint, parent)](py::object self) - { - const auto* joints = joints_of(self); - const size_t count = joints != nullptr ? joints->size() : 0; - const uint32_t* first = count > 0 ? fbs_field_address(*joints->Get(0), offset) : nullptr; - return struct_field_view(self, first, joint_stride, count, 0); - }, - "Parent joint per joint as a (J,) uint32 strided view; ROOT_PARENT marks a root.") - .def_property_readonly( - "taxel_positions", - [=](py::object self) - { - const auto* taxels = taxels_of(self); - const size_t count = taxels != nullptr ? taxels->size() : 0; - const float* first = count > 0 ? reinterpret_cast(&taxels->Get(0)->position()) : nullptr; - return struct_field_view(self, first, taxel_stride, count, 3); - }, - "Taxel bind positions as an (N, 3) float32 strided view.") - .def_property_readonly( - "taxel_normals", - [=](py::object self) - { - const auto* taxels = taxels_of(self); - const size_t count = taxels != nullptr ? taxels->size() : 0; - const float* first = count > 0 ? reinterpret_cast(&taxels->Get(0)->normal()) : nullptr; - return struct_field_view(self, first, taxel_stride, count, 3); - }, - "Taxel unit normals as an (N, 3) float32 strided view.") - .def_property_readonly( - "taxel_tangents", - [=](py::object self) - { - const auto* taxels = taxels_of(self); - const size_t count = taxels != nullptr ? taxels->size() : 0; - const float* first = count > 0 ? reinterpret_cast(&taxels->Get(0)->tangent()) : nullptr; - return struct_field_view(self, first, taxel_stride, count, 3); - }, - "Taxel unit tangents as an (N, 3) float32 strided view.") - .def_property_readonly( - "taxel_joint_indices", - [=](py::object self) - { - const auto* taxels = taxels_of(self); - const size_t count = taxels != nullptr ? taxels->size() : 0; - const uint32_t* first = count > 0 ? taxels->Get(0)->joint_indices()->data() : nullptr; - return struct_field_view(self, first, taxel_stride, count, 4); - }, - "Skinning joint per slot as an (N, 4) uint32 strided view.") - .def_property_readonly( - "taxel_joint_weights", - [=](py::object self) - { - const auto* taxels = taxels_of(self); - const size_t count = taxels != nullptr ? taxels->size() : 0; - const float* first = count > 0 ? taxels->Get(0)->joint_weights()->data() : nullptr; - return struct_field_view(self, first, taxel_stride, count, 4); - }, - "Skinning weight per slot as an (N, 4) float32 strided view.") - .def("__repr__", - [](const Serialized& self) - { - const auto* joints = self->joints(); - const auto* taxels = self->taxels(); - return "TactileShape(joints=" + std::to_string(joints != nullptr ? joints->size() : 0) + - ", taxels=" + std::to_string(taxels != nullptr ? taxels->size() : 0) + ")"; - }); - shape.attr("ROOT_PARENT") = TACTILE_ROOT_PARENT; -} - -} // namespace core diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py index ff5153723e..3aaabddc6a 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py @@ -29,8 +29,8 @@ SomaHandJointPoses, SomaHandJointRotations, MessageChannelMessagesTracked, - TactileData, - TactileShape, + TactileDataArray, + TactileMetadata, ) @@ -160,15 +160,15 @@ class SomaHandJointPosesTrackedType(_PayloadTensorType): class TactileDataTrackedType(_PayloadTensorType): - """TactileData payload from DeviceIO TactileTracker.""" + """TactileDataArray payload from DeviceIO TactileTracker.""" - _payload_cls = TactileData + _payload_cls = TactileDataArray class TactileShapeTrackedType(_PayloadTensorType): - """TactileShape metadata from DeviceIO TactileTracker.""" + """TactileMetadata metadata from DeviceIO TactileTracker.""" - _payload_cls = TactileShape + _payload_cls = TactileMetadata class MessageChannelMessagesTrackedType(_RequiredPayloadTensorType): @@ -331,8 +331,8 @@ def DeviceIOTactileTracked() -> TensorGroupType: """Tracked tactile frame and its shape from DeviceIO TactileTracker. Contains: - tactile_tracked: TactileData handle, or None when inactive - tactile_shape: TactileShape metadata handle, or None until it arrives + tactile_tracked: TactileDataArray handle, or None when inactive + tactile_shape: TactileMetadata metadata handle, or None until it arrives """ return TensorGroupType( "deviceio_tactile", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 0bbddb02b6..6da125fd9e 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -3,8 +3,8 @@ """Tactile Source Node - DeviceIO to Retargeting Engine converter. -``TactileSource`` turns per-frame ``TactileData`` into a force array, and the tracker's -``TactileShape`` metadata into the arrays that lay those forces out. Taxel and joint counts are +``TactileSource`` turns per-frame ``TactileDataArray`` into a force array, and the tracker's +``TactileMetadata`` metadata into the arrays that lay those forces out. Taxel and joint counts are fixed at construction so the retargeting graph has a static spec. """ @@ -24,7 +24,7 @@ if TYPE_CHECKING: from isaaccapture.deviceio_trackers import ITracker - from isaaccapture.schema import TactileData, TactileShape + from isaaccapture.schema import TactileDataArray, TactileMetadata def _float_array(name: str, rows: int, width: int) -> NDArrayType: @@ -46,33 +46,27 @@ def TactileForces(num_taxels: int) -> TensorGroupType: class TactileShapeIndex(IntEnum): """Positions of the arrays in a :func:`TactileShapeLayout` group.""" - JOINT_BIND_POSITIONS = 0 - JOINT_BIND_ORIENTATIONS = 1 - JOINT_PARENTS = 2 - TAXEL_POSITIONS = 3 - TAXEL_NORMALS = 4 - TAXEL_TANGENTS = 5 - TAXEL_JOINT_INDICES = 6 - TAXEL_JOINT_WEIGHTS = 7 + SEGMENT_JOINTS = 0 + SEGMENT_BIND_POSE = 1 + TAXEL_BIND_POSE = 2 + TAXEL_SEGMENT_INDICES = 3 + TAXEL_SEGMENT_WEIGHTS = 4 def TactileShapeLayout(num_joints: int, num_taxels: int) -> TensorGroupType: - """``TactileShape`` as arrays, ordered by :class:`TactileShapeIndex`. + """``TactileMetadata`` as arrays, ordered by :class:`TactileShapeIndex`. - Each array is named after the ``TactileShape`` property it is read from. Joint parents and - joint indices are int32; the root parent stays ``TactileShape.ROOT_PARENT``. + HandJoint identifiers and segment indices are int32. Pose rows contain + position XYZ followed by quaternion XYZW, all in hand-root space. """ return TensorGroupType( f"tactile_shape_{num_joints}x{num_taxels}", [ - _float_array("joint_bind_positions", num_joints, 3), - _float_array("joint_bind_orientations", num_joints, 4), - _int_array("joint_parents", (num_joints,)), - _float_array("taxel_positions", num_taxels, 3), - _float_array("taxel_normals", num_taxels, 3), - _float_array("taxel_tangents", num_taxels, 3), - _int_array("taxel_joint_indices", (num_taxels, 4)), - _float_array("taxel_joint_weights", num_taxels, 4), + _int_array("segment_joints", (num_joints,)), + _float_array("segment_bind_pose", num_joints, 7), + _float_array("taxel_bind_pose", num_taxels, 7), + _int_array("taxel_segment_indices", (num_taxels, 4)), + _float_array("taxel_segment_weights", num_taxels, 4), ], ) @@ -81,7 +75,7 @@ class TactileSource(IDeviceIOSource): """Converter: DeviceIO ``TactileTracker`` -> per-taxel forces and the shape that lays them out. Outputs (Optional): - - :data:`FORCES`: :func:`TactileForces` of ``num_taxels``, in ``TactileShape`` taxel + - :data:`FORCES`: :func:`TactileForces` of ``num_taxels``, in ``TactileMetadata`` taxel order; absent when the device is inactive. - :data:`SHAPE`: :func:`TactileShapeLayout` of ``num_joints`` x ``num_taxels``; absent until the tracker holds the device's shape. The tracker publishes no forces before then. @@ -104,7 +98,7 @@ def __init__( Args: name: Unique name for this source node. - collection_id: Tensor collection ID of the producer's ``TactileData`` stream. + collection_id: Tensor collection ID of the producer's ``TactileDataArray`` stream. num_joints: Joint count of the device skeleton; fixes the shape output spec. num_taxels: Taxel count of the device; fixes the output specs. max_flatbuffer_size: Override for ``TactileTracker``'s default collection size. @@ -146,8 +140,8 @@ def output_spec(self) -> RetargeterIOType: def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: """Publish the frame's forces and a copy of the shape; ``set_none()`` whichever is absent.""" - data: "TactileData | None" = inputs["deviceio_tactile"][0] - shape: "TactileShape | None" = inputs["deviceio_tactile"][1] + data: "TactileDataArray | None" = inputs["deviceio_tactile"][0] + shape: "TactileMetadata | None" = inputs["deviceio_tactile"][1] forces_out = outputs[self.FORCES] if data is None: @@ -164,14 +158,26 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N if shape is None: shape_out.set_none() return - joints, taxels = len(shape.joint_parents), len(shape.taxel_positions) + algorithm = shape.algorithm + segments = shape.segments + if algorithm is None or segments is None: + raise ValueError( + f"{self.name}: metadata must specify an algorithm and segments" + ) + joints, taxels = len(segments.joints), len(algorithm.taxel_bind_pose) if (joints, taxels) != (self._num_joints, self._num_taxels): raise ValueError( f"{self.name}: shape has {joints} joints and {taxels} taxels, " f"source expects {self._num_joints} and {self._num_taxels}" ) - for index, tensor_type in enumerate(shape_out.group_type.types): - value = getattr(shape, tensor_type.name) + values = ( + np.asarray(segments.joints, dtype=np.int32), + algorithm.segment_bind_pose, + algorithm.taxel_bind_pose, + algorithm.taxel_segment_indices, + algorithm.taxel_segment_weights, + ) + for index, value in enumerate(values): # Copied: the views alias the metadata the tracker holds and compares later pushes to. shape_out[index] = value.astype( np.int32 if value.dtype == np.uint32 else value.dtype diff --git a/src/python/isaaccapture/schema/__init__.py b/src/python/isaaccapture/schema/__init__.py index a64904dffa..b2a94c440e 100644 --- a/src/python/isaaccapture/schema/__init__.py +++ b/src/python/isaaccapture/schema/__init__.py @@ -57,9 +57,13 @@ OgloGloveSample, OgloGloveSampleRecord, # Tactile types. - TactileData, - TactileDataRecord, - TactileShape, + TactileDataArray, + TactileDataArrayRecord, + TactileMetadata, + HandSkeletonRef, + SegmentSource, + AlgorithmSource, + LBS, # Tracker metadata, for producers of a tracker's metadata. TRACKER_METADATA_MAX_SIZE, TRACKER_METADATA_TENSOR_IDENTIFIER, @@ -197,9 +201,13 @@ def __getattr__(name: str): "OgloGloveSample", "OgloGloveSampleRecord", # Tactile types. - "TactileData", - "TactileDataRecord", - "TactileShape", + "TactileDataArray", + "TactileDataArrayRecord", + "TactileMetadata", + "HandSkeletonRef", + "SegmentSource", + "AlgorithmSource", + "LBS", # Tracker metadata. "TRACKER_METADATA_MAX_SIZE", "TRACKER_METADATA_TENSOR_IDENTIFIER", diff --git a/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp b/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp index 557dab36a3..98be2a9815 100644 --- a/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp +++ b/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp @@ -23,8 +23,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -33,6 +33,7 @@ #include #include #include +#include #include #ifdef _WIN32 @@ -829,7 +830,7 @@ TEST_CASE("ReplaySession: keyboard tracker round-trip ignores live provider inpu } // ============================================================================= -// Tracker with metadata — TactileTracker (TactileShape on its own channel) +// Tracker with metadata — TactileTracker (TactileMetadata on its own channel) // ============================================================================= namespace @@ -839,26 +840,34 @@ namespace void write_tactile_recording(const std::string& path, const std::string& base_name, bool with_metadata) { auto writer = open_writer(path); - core::McapTrackerChannels data( + core::McapTrackerChannels data( *writer, base_name, to_string_vec(core::TactileRecordingTraits::recording_channels)); - core::McapTrackerChannels metadata( + core::McapTrackerChannels metadata( *writer, base_name, to_string_vec(core::TactileRecordingTraits::metadata_channels)); if (with_metadata) { - core::TactileShapeT shape; - shape.joints = { core::TactileJoint(make_pose(0, 0, 0), 0x7FFFFFFF) }; - shape.taxels = { core::TactileTaxel(), core::TactileTaxel() }; - metadata.write(0, core::pack(shape), 500); + core::HandSkeletonRefT segments; + segments.joints = { core::HandJoint_WRIST }; + core::LBST lbs; + lbs.segment_bind_pose = { make_pose(0, 0, 0) }; + lbs.taxel_bind_pose = { make_pose(0, 0, 0), make_pose(0, 0, 0) }; + lbs.taxel_segment_indices.resize(2); + const std::array weights = { 1, 0, 0, 0 }; + lbs.taxel_segment_weights.assign(2, core::Float4(flatbuffers::span(weights.data(), 4))); + core::TactileMetadataT shape; + shape.algorithm.Set(std::move(lbs)); + shape.segments.Set(std::move(segments)); + metadata.write(0, core::pack(shape), 500); } for (int i = 0; i < 2; ++i) { - core::TactileDataT frame; + core::TactileDataArrayT frame; frame.taxel_force = { core::Point(static_cast(i), 0, 0), core::Point(0, 0, 1) }; const int64_t time_ns = (i + 1) * 1000; - const auto record = - core::pack_record(&frame, core::DeviceDataTimestamp(time_ns, time_ns, time_ns)); + const auto record = core::pack_record( + &frame, core::DeviceDataTimestamp(time_ns, time_ns, time_ns)); data.write(0, record); data.write(1, record); } @@ -870,7 +879,7 @@ void write_tactile_recording(const std::string& path, const std::string& base_na TEST_CASE("ReplaySession: tracker metadata is replayed before its data", "[replay][session][metadata]") { // Frozen wire strings, as for the record channels above. - CHECK(std::string_view(core::TactileShape::GetFullyQualifiedName()) == "core.TactileShape"); + CHECK(std::string_view(core::TactileMetadata::GetFullyQualifiedName()) == "core.TactileMetadata"); REQUIRE(core::TactileRecordingTraits::metadata_channels.size() == 1); CHECK(std::string_view(core::TactileRecordingTraits::metadata_channels[0]) == "tactile_metadata"); @@ -887,7 +896,8 @@ TEST_CASE("ReplaySession: tracker metadata is replayed before its data", "[repla // Read when the session opens, so it is there before the first frame. const auto& shape = tracker.get_metadata(*session); REQUIRE(shape); - CHECK(shape->taxels()->size() == 2); + REQUIRE(shape->algorithm_as_LBS()); + CHECK(shape->algorithm_as_LBS()->taxel_bind_pose()->size() == 2); for (int i = 0; i < 2; ++i) { diff --git a/tests/cpp/core/schema/CMakeLists.txt b/tests/cpp/core/schema/CMakeLists.txt index d2f839e50f..d1e59cd488 100644 --- a/tests/cpp/core/schema/CMakeLists.txt +++ b/tests/cpp/core/schema/CMakeLists.txt @@ -22,8 +22,8 @@ add_executable(schema_tests test_soma_body_joint_poses.cpp test_soma_hand_joint_rotations.cpp test_soma_hand_joint_poses.cpp - test_tactile.cpp - test_tactile_shape.cpp + test_tactile_array.cpp + test_tactile_metadata.cpp test_schema_conform.cpp ) target_link_libraries(schema_tests PRIVATE diff --git a/tests/cpp/core/schema/test_tactile.cpp b/tests/cpp/core/schema/test_tactile_array.cpp similarity index 53% rename from tests/cpp/core/schema/test_tactile.cpp rename to tests/cpp/core/schema/test_tactile_array.cpp index 3f15e08326..42e0c3acd1 100644 --- a/tests/cpp/core/schema/test_tactile.cpp +++ b/tests/cpp/core/schema/test_tactile_array.cpp @@ -1,13 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Unit tests for the generated TactileData FlatBuffer message. +// Unit tests for the generated TactileDataArray FlatBuffer message. #include #include #include #include -#include +#include #include #include @@ -15,9 +15,9 @@ // VT_ == (id + 2) * 2. #define VT(field) (field + 2) * 2 -static_assert(core::TactileData::VT_TAXEL_FORCE == VT(0)); -static_assert(core::TactileDataRecord::VT_DATA == VT(0)); -static_assert(core::TactileDataRecord::VT_TIMESTAMP == VT(1)); +static_assert(core::TactileDataArray::VT_TAXEL_FORCE == VT(0)); +static_assert(core::TactileDataArrayRecord::VT_DATA == VT(0)); +static_assert(core::TactileDataArrayRecord::VT_TIMESTAMP == VT(1)); // Producers copy XrVector3f arrays straight into taxel_force. static_assert(sizeof(core::Point) == sizeof(XrVector3f)); @@ -25,18 +25,18 @@ static_assert(sizeof(core::Point) == sizeof(XrVector3f)); namespace { -core::TactileDataT make_frame() +core::TactileDataArrayT make_frame() { - core::TactileDataT native; + core::TactileDataArrayT native; native.taxel_force = { core::Point(0.0f, 0.0f, 1.0f), core::Point(0.5f, -0.5f, 2.0f) }; return native; } } // namespace -TEST_CASE("TactileData round-trips every field", "[unit][tactile]") +TEST_CASE("TactileDataArray round-trips every field", "[unit][tactile]") { - const auto data = core::pack(make_frame()); + const auto data = core::pack(make_frame()); REQUIRE(data); REQUIRE(data->taxel_force() != nullptr); @@ -45,26 +45,26 @@ TEST_CASE("TactileData round-trips every field", "[unit][tactile]") CHECK(data->taxel_force()->Get(1)->z() == 2.0f); } -TEST_CASE("TactileData leaves an empty frame's vector absent", "[unit][tactile]") +TEST_CASE("TactileDataArray leaves an empty frame's vector absent", "[unit][tactile]") { - const auto data = core::pack(core::TactileDataT{}); + const auto data = core::pack(core::TactileDataArrayT{}); REQUIRE(data); CHECK(data->taxel_force() == nullptr); } -TEST_CASE("TactileDataRecord verifies and carries its timestamp", "[unit][tactile]") +TEST_CASE("TactileDataArrayRecord verifies and carries its timestamp", "[unit][tactile]") { - core::TactileDataRecordT record; - record.data = std::make_shared(make_frame()); + core::TactileDataArrayRecordT record; + record.data = std::make_shared(make_frame()); record.timestamp = std::make_shared(1, 2, 3); flatbuffers::FlatBufferBuilder builder; - builder.Finish(core::TactileDataRecord::Pack(builder, &record)); + builder.Finish(core::TactileDataArrayRecord::Pack(builder, &record)); flatbuffers::Verifier verifier(builder.GetBufferPointer(), builder.GetSize()); - REQUIRE(core::VerifyTactileDataRecordBuffer(verifier)); - const auto* read = core::GetTactileDataRecord(builder.GetBufferPointer()); + REQUIRE(core::VerifyTactileDataArrayRecordBuffer(verifier)); + const auto* read = core::GetTactileDataArrayRecord(builder.GetBufferPointer()); CHECK(read->timestamp()->sample_time_local_common_clock() == 2); CHECK(read->data()->taxel_force()->size() == 2); } diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp new file mode 100644 index 0000000000..1a3156ed32 --- /dev/null +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include +#include +#include +#include + +#define VT(field) (field + 2) * 2 +static_assert(core::TactileMetadata::VT_ALGORITHM_TYPE == VT(0)); +static_assert(core::TactileMetadata::VT_ALGORITHM == VT(1)); +static_assert(core::TactileMetadata::VT_SEGMENTS_TYPE == VT(2)); +static_assert(core::TactileMetadata::VT_SEGMENTS == VT(3)); +static_assert(core::HandSkeletonRef::VT_JOINTS == VT(0)); +static_assert(core::LBS::VT_SEGMENT_BIND_POSE == VT(0)); +static_assert(core::LBS::VT_TAXEL_BIND_POSE == VT(1)); +static_assert(core::LBS::VT_TAXEL_SEGMENT_INDICES == VT(2)); +static_assert(core::LBS::VT_TAXEL_SEGMENT_WEIGHTS == VT(3)); +static_assert(sizeof(core::UInt4) == 4 * sizeof(uint32_t)); +static_assert(sizeof(core::Float4) == 4 * sizeof(float)); + +namespace +{ + +core::TactileMetadataT make_metadata() +{ + core::HandSkeletonRefT segments; + segments.joints = { core::HandJoint_WRIST, core::HandJoint_INDEX_TIP }; + core::LBST lbs; + lbs.segment_bind_pose = { core::Pose(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)), + core::Pose(core::Point(0, 0.1f, 0), core::Quaternion(0, 0, 0, 1)) }; + lbs.taxel_bind_pose = { core::Pose(core::Point(0, 0.09f, 0.01f), core::Quaternion(1, 0, 0, 0)) }; + const std::array indices = { 1, 0, 0, 0 }; + const std::array weights = { 0.75f, 0.25f, 0.0f, 0.0f }; + lbs.taxel_segment_indices.emplace_back(flatbuffers::span(indices.data(), 4)); + lbs.taxel_segment_weights.emplace_back(flatbuffers::span(weights.data(), 4)); + core::TactileMetadataT native; + native.algorithm.Set(std::move(lbs)); + native.segments.Set(std::move(segments)); + return native; +} + +} // namespace + +TEST_CASE("TactileMetadata round-trips union tags and segment-local indices", "[unit][tactile_metadata]") +{ + const auto metadata = core::pack(make_metadata()); + CHECK(metadata->algorithm_type() == core::AlgorithmSource_LBS); + CHECK(metadata->segments_type() == core::SegmentSource_HandSkeletonRef); + const auto* segments = metadata->segments_as_HandSkeletonRef(); + const auto* lbs = metadata->algorithm_as_LBS(); + REQUIRE(segments); + REQUIRE(lbs); + REQUIRE(segments->joints()->size() == 2); + CHECK(segments->joints()->Get(1) == core::HandJoint_INDEX_TIP); + CHECK(lbs->taxel_segment_indices()->Get(0)->values()->Get(0) == 1); + CHECK(lbs->taxel_segment_weights()->Get(0)->values()->Get(1) == 0.25f); + CHECK(lbs->segment_bind_pose()->Get(1)->position().y() == 0.1f); + CHECK(lbs->taxel_bind_pose()->Get(0)->orientation().x() == 1.0f); +} + +TEST_CASE("TactileMetadata verifies and retains nested union views", "[unit][tactile_metadata]") +{ + auto metadata = core::pack(make_metadata()); + const auto bytes = metadata.buffer(); + flatbuffers::Verifier verifier(bytes.data(), bytes.size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK(std::string(core::TactileMetadata::GetFullyQualifiedName()) == "core.TactileMetadata"); + auto lbs = metadata.narrow(metadata->algorithm_as_LBS()); + metadata.reset(); + REQUIRE(lbs); + CHECK(lbs->taxel_segment_weights()->Get(0)->values()->Get(0) == 0.75f); +} diff --git a/tests/cpp/core/schema/test_tactile_shape.cpp b/tests/cpp/core/schema/test_tactile_shape.cpp deleted file mode 100644 index 787f5ab7e4..0000000000 --- a/tests/cpp/core/schema/test_tactile_shape.cpp +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Unit tests for the generated TactileShape FlatBuffer message. - -#include -#include -#include -#include -#include - -#include -#include -#include - -// VT_ == (id + 2) * 2. -#define VT(field) (field + 2) * 2 -static_assert(core::TactileShape::VT_JOINTS == VT(0)); -static_assert(core::TactileShape::VT_TAXELS == VT(1)); - -// Producers copy OpenXR values straight into the structs. -static_assert(sizeof(core::Point) == sizeof(XrVector3f)); -static_assert(sizeof(core::TactileJoint) == sizeof(XrPosef) + sizeof(XrHandJointEXT)); -static_assert(sizeof(core::TactileTaxel) == 3 * sizeof(XrVector3f) + 4 * sizeof(XrHandJointEXT) + sizeof(XrVector4f)); - -namespace -{ - -constexpr uint32_t kRootParent = 0x7FFFFFFF; - -// Two joints (wrist -> index tip) and one taxel skinned to both. -core::TactileShapeT make_shape() -{ - core::TactileShapeT native; - native.joints = { core::TactileJoint(core::Pose(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)), kRootParent), - core::TactileJoint(core::Pose(core::Point(0, 0.1f, 0), core::Quaternion(0, 0, 0, 1)), 0) }; - - const std::array indices = { 1, 0, 0, 0 }; - const std::array weights = { 0.75f, 0.25f, 0.0f, 0.0f }; - native.taxels = { core::TactileTaxel(core::Point(0, 0.09f, 0.01f), core::Point(0, 0, 1), core::Point(1, 0, 0), - ::flatbuffers::span(indices.data(), 4), - ::flatbuffers::span(weights.data(), 4)) }; - return native; -} - -} // namespace - -TEST_CASE("TactileShape round-trips skeleton, taxels and skinning", "[unit][tactile_shape]") -{ - const auto shape = core::pack(make_shape()); - - REQUIRE(shape); - REQUIRE(shape->joints()->size() == 2); - CHECK(shape->joints()->Get(0)->parent() == kRootParent); - CHECK(shape->joints()->Get(1)->parent() == 0); - CHECK(shape->joints()->Get(1)->bind_pose().position().y() == 0.1f); - REQUIRE(shape->taxels()->size() == 1); - const auto* taxel = shape->taxels()->Get(0); - CHECK(taxel->normal().z() == 1.0f); - CHECK(taxel->joint_indices()->Get(0) == 1); - CHECK(taxel->joint_weights()->Get(1) == 0.25f); -} - -TEST_CASE("TactileShape is its own root and verifies", "[unit][tactile_shape]") -{ - const auto shape = core::pack(make_shape()); - const auto bytes = shape.buffer(); - - flatbuffers::Verifier verifier(bytes.data(), bytes.size()); - REQUIRE(core::VerifyTactileShapeBuffer(verifier)); - CHECK(std::string(core::TactileShape::GetFullyQualifiedName()) == "core.TactileShape"); - - core::TactileShapeT unpacked; - core::GetTactileShape(bytes.data())->UnPackTo(&unpacked); - REQUIRE(unpacked.taxels.size() == 1); - CHECK(unpacked.taxels[0].joint_weights()->Get(0) == 0.75f); -} diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 5b4e37652d..81aeb51c26 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -16,7 +16,13 @@ ) from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup -from isaaccapture.schema import TactileData, TactileShape +from isaaccapture.schema import ( + HandJoint, + HandSkeletonRef, + LBS, + TactileDataArray, + TactileMetadata, +) NUM_JOINTS = 2 NUM_TAXELS = 3 @@ -41,19 +47,20 @@ def _outputs(source) -> dict: return {name: _make_output_group(gt) for name, gt in source.output_spec().items()} -def _shape(num_joints: int = NUM_JOINTS, num_taxels: int = NUM_TAXELS) -> TactileShape: +def _shape( + num_joints: int = NUM_JOINTS, num_taxels: int = NUM_TAXELS +) -> TactileMetadata: indices = np.zeros((num_taxels, 4), np.uint32) weights = np.zeros((num_taxels, 4), np.float32) weights[:, 0] = 1.0 - return TactileShape( - joint_bind_positions=np.zeros((num_joints, 3), np.float32), - joint_bind_orientations=np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1)), - joint_parents=[TactileShape.ROOT_PARENT] + list(range(num_joints - 1)), - taxel_positions=np.full((num_taxels, 3), 0.01, np.float32), - taxel_normals=np.tile([0.0, 0.0, 1.0], (num_taxels, 1)), - taxel_tangents=np.tile([1.0, 0.0, 0.0], (num_taxels, 1)), - taxel_joint_indices=indices, - taxel_joint_weights=weights, + segment_poses = np.zeros((num_joints, 7), np.float32) + segment_poses[:, 6] = 1 + taxel_poses = np.zeros((num_taxels, 7), np.float32) + taxel_poses[:, :3] = 0.01 + taxel_poses[:, 6] = 1 + return TactileMetadata( + LBS(segment_poses, taxel_poses, indices, weights), + HandSkeletonRef([HandJoint(i) for i in range(num_joints)]), ) @@ -89,7 +96,7 @@ def test_index_matches_layout_order(self): def test_poll_reads_data_and_metadata_from_one_tracker(self): src = _source() - data = TactileData(np.zeros((NUM_TAXELS, 3), np.float32)) + data = TactileDataArray(np.zeros((NUM_TAXELS, 3), np.float32)) shape = _shape() src._tracker = _FakeTracker(data, shape) @@ -102,22 +109,22 @@ def test_active_conversion(self): src = _source() forces = np.arange(9, dtype=np.float32).reshape(3, 3) outputs = _outputs(src) - src.compute(_inputs(src, TactileData(forces), _shape()), outputs) + src.compute(_inputs(src, TactileDataArray(forces), _shape()), outputs) np.testing.assert_array_equal( np.asarray(outputs[TactileSource.FORCES][0]), forces ) group = outputs[TactileSource.SHAPE] assert not group.is_none - parents = np.asarray(group[TactileShapeIndex.JOINT_PARENTS]) - assert parents.dtype == np.int32 - assert parents.tolist() == [TactileShape.ROOT_PARENT, 0] + joints = np.asarray(group[TactileShapeIndex.SEGMENT_JOINTS]) + assert joints.dtype == np.int32 + assert joints.tolist() == [0, 1] assert ( - np.asarray(group[TactileShapeIndex.TAXEL_JOINT_INDICES]).dtype == np.int32 + np.asarray(group[TactileShapeIndex.TAXEL_SEGMENT_INDICES]).dtype == np.int32 ) np.testing.assert_array_equal( - np.asarray(group[TactileShapeIndex.TAXEL_NORMALS]), - np.tile([0.0, 0.0, 1.0], (NUM_TAXELS, 1)), + np.asarray(group[TactileShapeIndex.TAXEL_BIND_POSE])[:, 3:], + np.tile([0.0, 0.0, 0.0, 1.0], (NUM_TAXELS, 1)), ) def test_shape_outputs_do_not_alias_the_metadata(self): @@ -149,7 +156,7 @@ def test_inactive_sets_none(self): def test_taxel_count_mismatch_raises(self): src = _source(num_taxels=4) - frame = TactileData(np.zeros((3, 3), np.float32)) + frame = TactileDataArray(np.zeros((3, 3), np.float32)) with pytest.raises(ValueError, match="3 taxels"): src.compute(_inputs(src, frame, None), _outputs(src)) diff --git a/tests/python/core/schema/test_tactile.py b/tests/python/core/schema/test_tactile_array.py similarity index 60% rename from tests/python/core/schema/test_tactile.py rename to tests/python/core/schema/test_tactile_array.py index f6d91cb243..a48678bdc7 100644 --- a/tests/python/core/schema/test_tactile.py +++ b/tests/python/core/schema/test_tactile_array.py @@ -1,51 +1,57 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for TactileData / TactileDataRecord in isaaccapture.schema.""" +"""Unit tests for TactileDataArray / TactileDataArrayRecord in isaaccapture.schema.""" import gc import numpy as np import pytest -from isaaccapture.schema import DeviceDataTimestamp, TactileData, TactileDataRecord +from isaaccapture.schema import ( + DeviceDataTimestamp, + TactileDataArray, + TactileDataArrayRecord, +) FORCES = np.array([[0.0, 0.0, 1.0], [0.5, -0.5, 2.0], [0.0, 0.1, 0.0]], np.float32) -class TestTactileData: +class TestTactileDataArray: def test_default_construction(self): - data = TactileData() + data = TactileDataArray() assert data.taxel_force.shape == (0, 3) def test_round_trip(self): - data = TactileData(FORCES) + data = TactileDataArray(FORCES) assert data.taxel_force.dtype == np.float32 np.testing.assert_array_equal(data.taxel_force, FORCES) def test_accepts_nested_lists(self): - data = TactileData([[1, 2, 3]]) + data = TactileDataArray([[1, 2, 3]]) np.testing.assert_array_equal(data.taxel_force, [[1.0, 2.0, 3.0]]) def test_view_outlives_its_table(self): - forces = TactileData(FORCES).taxel_force + forces = TactileDataArray(FORCES).taxel_force gc.collect() np.testing.assert_array_equal(forces, FORCES) def test_rejects_wrong_width(self): with pytest.raises(ValueError, match="taxel_force"): - TactileData(np.zeros((2, 4), np.float32)) + TactileDataArray(np.zeros((2, 4), np.float32)) def test_repr(self): - assert "taxels=3" in repr(TactileData(FORCES)) + assert "taxels=3" in repr(TactileDataArray(FORCES)) -class TestTactileDataRecord: +class TestTactileDataArrayRecord: def test_record_carries_payload_and_timestamp(self): - record = TactileDataRecord(TactileData(FORCES), DeviceDataTimestamp(1, 2, 3)) + record = TactileDataArrayRecord( + TactileDataArray(FORCES), DeviceDataTimestamp(1, 2, 3) + ) np.testing.assert_array_equal(record.data.taxel_force, FORCES) assert record.timestamp.sample_time_raw_device_clock == 3 def test_payload_less_record(self): - record = TactileDataRecord(None, DeviceDataTimestamp(1, 2, 3)) + record = TactileDataArrayRecord(None, DeviceDataTimestamp(1, 2, 3)) assert record.data is None diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py new file mode 100644 index 0000000000..2e9191eb01 --- /dev/null +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tactile metadata union composition, binding invariants, and nested view lifetime.""" + +import gc + +import numpy as np +import pytest + +from isaaccapture.schema import ( + AlgorithmSource, + HandJoint, + HandSkeletonRef, + LBS, + SegmentSource, + TactileMetadata, +) + + +def lbs_arrays(num_joints: int = 2, num_taxels: int = 3) -> dict: + segments = np.zeros((num_joints, 7), np.float32) + segments[:, 6] = 1 + taxels = np.zeros((num_taxels, 7), np.float32) + taxels[:, :3] = 0.01 + taxels[:, 6] = 1 + indices = np.zeros((num_taxels, 4), np.uint32) + indices[:, 0] = num_joints - 1 + weights = np.zeros((num_taxels, 4), np.float32) + weights[:, 0] = 1 + return dict( + segment_bind_pose=segments, + taxel_bind_pose=taxels, + taxel_segment_indices=indices, + taxel_segment_weights=weights, + ) + + +def test_metadata_round_trip(): + arrays = lbs_arrays() + metadata = TactileMetadata( + LBS(**arrays), HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) + assert metadata.algorithm_type == AlgorithmSource.LBS + assert metadata.segments_type == SegmentSource.HandSkeletonRef + assert metadata.segments.joints == [HandJoint.WRIST, HandJoint.INDEX_TIP] + for name, expected in arrays.items(): + actual = getattr(metadata.algorithm, name) + assert actual.dtype == expected.dtype + np.testing.assert_array_equal(actual, expected) + assert metadata.algorithm.taxel_segment_indices[0, 0] == 1 + + +def test_nested_views_and_arrays_retain_the_buffer(): + metadata = TactileMetadata( + LBS(**lbs_arrays()), HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) + algorithm = metadata.algorithm + joints = metadata.segments + rows = algorithm.taxel_bind_pose + encoded = algorithm.to_bytes() + del metadata, algorithm + gc.collect() + assert joints.joints == [HandJoint.WRIST, HandJoint.INDEX_TIP] + np.testing.assert_array_equal(rows, lbs_arrays()["taxel_bind_pose"]) + assert encoded + + +def test_rejects_mismatched_taxel_lengths(): + arrays = lbs_arrays() + arrays["taxel_bind_pose"] = arrays["taxel_bind_pose"][:2] + with pytest.raises(ValueError, match="same length"): + LBS(**arrays) + + +def test_rejects_mismatched_segment_mapping(): + with pytest.raises(ValueError, match="same length"): + TactileMetadata(LBS(**lbs_arrays()), HandSkeletonRef([HandJoint.WRIST])) + + +def test_rejects_out_of_range_zero_weight_slot(): + arrays = lbs_arrays() + arrays["taxel_segment_indices"][0, 3] = 2 + with pytest.raises(ValueError, match="taxel_segment_indices"): + LBS(**arrays) + + +@pytest.mark.parametrize("weight", [-1.0, np.inf, np.nan, 0.5]) +def test_rejects_invalid_weights(weight): + arrays = lbs_arrays() + arrays["taxel_segment_weights"][0, 0] = weight + with pytest.raises(ValueError, match="taxel_segment_weights"): + LBS(**arrays) + + +@pytest.mark.parametrize("field", ["segment_bind_pose", "taxel_bind_pose"]) +def test_rejects_nonunit_quaternions(field): + arrays = lbs_arrays() + arrays[field][0, 6] = 0 + with pytest.raises(ValueError, match="unit quaternions"): + LBS(**arrays) + + +def test_rejects_nonfinite_bind_positions(): + arrays = lbs_arrays() + arrays["segment_bind_pose"][0, 0] = np.nan + with pytest.raises(ValueError, match="finite"): + LBS(**arrays) + + +def test_rejects_wrong_influence_width(): + arrays = lbs_arrays() + arrays["taxel_segment_weights"] = np.zeros((3, 3), np.float32) + with pytest.raises(ValueError, match="taxel_segment_weights"): + LBS(**arrays) + + +@pytest.mark.parametrize( + "joints", [[HandJoint.WRIST, HandJoint.WRIST], [HandJoint.NUM_JOINTS]] +) +def test_rejects_invalid_joint_mapping(joints): + with pytest.raises(ValueError, match="joints"): + HandSkeletonRef(joints) diff --git a/tests/python/core/schema/test_tactile_shape.py b/tests/python/core/schema/test_tactile_shape.py deleted file mode 100644 index 9bac9efee9..0000000000 --- a/tests/python/core/schema/test_tactile_shape.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for TactileShape in isaaccapture.schema.""" - -import numpy as np -import pytest - -from isaaccapture.schema import TactileShape - -ROOT = TactileShape.ROOT_PARENT - - -def shape_arrays(num_joints: int = 2, num_taxels: int = 3) -> dict: - """A joint chain rooted at joint 0, with every taxel skinned to its last joint.""" - indices = np.zeros((num_taxels, 4), np.uint32) - indices[:, 0] = num_joints - 1 - weights = np.zeros((num_taxels, 4), np.float32) - weights[:, 0] = 1.0 - return dict( - joint_bind_positions=np.arange(num_joints * 3, dtype=np.float32).reshape(-1, 3), - joint_bind_orientations=np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1)).astype( - np.float32 - ), - joint_parents=np.array([ROOT] + list(range(num_joints - 1)), np.uint32), - taxel_positions=np.full((num_taxels, 3), 0.01, np.float32), - taxel_normals=np.tile([0.0, 0.0, 1.0], (num_taxels, 1)).astype(np.float32), - taxel_tangents=np.tile([1.0, 0.0, 0.0], (num_taxels, 1)).astype(np.float32), - taxel_joint_indices=indices, - taxel_joint_weights=weights, - ) - - -class TestTactileShape: - def test_root_parent_is_the_openxr_max_enum(self): - assert ROOT == 0x7FFFFFFF - - def test_round_trip(self): - arrays = shape_arrays() - shape = TactileShape(**arrays) - - for name, expected in arrays.items(): - value = getattr(shape, name) - assert value.dtype == expected.dtype, name - np.testing.assert_array_equal(value, expected, err_msg=name) - - def test_rejects_mismatched_joint_lengths(self): - arrays = shape_arrays() - arrays["joint_parents"] = arrays["joint_parents"][:1] - with pytest.raises(ValueError, match="same length"): - TactileShape(**arrays) - - def test_rejects_mismatched_taxel_lengths(self): - arrays = shape_arrays() - arrays["taxel_normals"] = arrays["taxel_normals"][:2] - with pytest.raises(ValueError, match="same length"): - TactileShape(**arrays) - - def test_rejects_parent_out_of_range(self): - arrays = shape_arrays() - arrays["joint_parents"][1] = 5 - with pytest.raises(ValueError, match="joint_parents"): - TactileShape(**arrays) - - def test_rejects_skinning_joint_out_of_range(self): - arrays = shape_arrays() - arrays["taxel_joint_indices"][0, 3] = 2 - with pytest.raises(ValueError, match="taxel_joint_indices"): - TactileShape(**arrays) - - def test_rejects_wrong_width(self): - arrays = shape_arrays() - arrays["taxel_joint_weights"] = np.zeros((3, 3), np.float32) - with pytest.raises(ValueError, match="taxel_joint_weights"): - TactileShape(**arrays) - - def test_repr(self): - assert ( - repr(TactileShape(**shape_arrays())) == "TactileShape(joints=2, taxels=3)" - ) From 5348889ae54c176a89487f73b97de53268885834 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 21 Oct 2026 14:48:03 -0700 Subject: [PATCH 07/45] Variable naming Signed-off-by: Lotus Li --- .../isaaccapture_examples/tactile/common.py | 2 +- .../isaaccapture_examples/tactile/feeder.py | 48 +++++----- .../isaaccapture_examples/tactile/printer.py | 25 +++--- scripts/check_licenses.py | 3 + src/core/codegen/generate_trackers.py | 3 + src/core/codegen/templates.py | 2 + src/core/schema/fbs/tactile_array.fbs | 6 +- src/core/schema/fbs/tactile_metadata.fbs | 21 ++--- .../schema/python/tactile_array_bindings.h | 14 +-- .../schema/python/tactile_metadata_bindings.h | 79 ++++++++-------- .../deviceio_source_nodes/__init__.py | 12 +-- .../deviceio_tensor_types.py | 8 +- .../deviceio_source_nodes/tactile_source.py | 89 ++++++++++--------- .../test_replay_session.cpp | 30 +++---- tests/cpp/core/schema/test_tactile_array.cpp | 19 ++-- .../cpp/core/schema/test_tactile_metadata.cpp | 48 +++++----- .../test_tactile_sources.py | 73 +++++++-------- .../python/core/schema/test_tactile_array.py | 14 +-- .../core/schema/test_tactile_metadata.py | 46 +++++----- 19 files changed, 282 insertions(+), 260 deletions(-) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/common.py b/examples/tactile/python/isaaccapture_examples/tactile/common.py index 1535dd0152..6174ba63cf 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/common.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/common.py @@ -11,4 +11,4 @@ DEFAULT_NUM_TAXELS = 200 # TactileTracker's default max_flatbuffer_size: the reader rejects a larger collection. -TACTILE_SAMPLE_SIZE = 16384 +TACTILE_MAX_FRAME_SIZE = 16384 diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py index 34fbdba2c4..68016786f9 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -38,7 +38,7 @@ DEFAULT_NUM_TAXELS, MAX_NUM_JOINTS, MIN_NUM_JOINTS, - TACTILE_SAMPLE_SIZE, + TACTILE_MAX_FRAME_SIZE, ) logger = logging.getLogger("isaaccapture.examples.tactile.feeder") @@ -49,12 +49,12 @@ FINGER_DISTAL_JOINTS = (4, 9, 14, 19, 24) -def mock_shape(num_joints: int, num_taxels: int) -> TactileMetadata: - """Build a flat hand in hand-root space with taxels on its distal segments.""" +def mock_metadata(num_joints: int, num_taxels: int) -> TactileMetadata: + """Build metadata for a flat hand with taxels on its distal segments.""" positions = np.zeros((num_joints, 3), np.float32) positions[0] = (0.0, 0.05, 0.0) - taxel_joints = [] + taxel_joint_ids = [] for finger, first in enumerate(FINGER_FIRST_JOINTS): if first >= num_joints: break @@ -68,25 +68,25 @@ def mock_shape(num_joints: int, num_taxels: int) -> TactileMetadata: 0.02 + 0.025 * (joint - first), 0.0, ) - taxel_joints.append(min(FINGER_DISTAL_JOINTS[finger], last)) + taxel_joint_ids.append(min(FINGER_DISTAL_JOINTS[finger], last)) - if not taxel_joints: - taxel_joints.append(WRIST) + if not taxel_joint_ids: + taxel_joint_ids.append(WRIST) taxel_positions = np.zeros((num_taxels, 3), np.float32) - joint_indices = np.zeros((num_taxels, 4), np.uint32) - joint_weights = np.zeros((num_taxels, 4), np.float32) - per_segment = math.ceil(num_taxels / len(taxel_joints)) + taxel_segment_indices = np.zeros((num_taxels, 4), np.uint32) + taxel_segment_weights = np.zeros((num_taxels, 4), np.float32) + per_segment = math.ceil(num_taxels / len(taxel_joint_ids)) for taxel in range(num_taxels): - segment, slot = taxel % len(taxel_joints), taxel // len(taxel_joints) - joint = taxel_joints[segment] + segment, slot = taxel % len(taxel_joint_ids), taxel // len(taxel_joint_ids) + joint = taxel_joint_ids[segment] taxel_positions[taxel] = positions[joint] + ( 0.0, 0.02 * slot / per_segment, -0.005, ) - joint_indices[taxel] = joint - joint_weights[taxel, 0] = 1.0 + taxel_segment_indices[taxel] = joint + taxel_segment_weights[taxel, 0] = 1.0 segment_poses = np.column_stack( (positions, np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1))) @@ -95,8 +95,10 @@ def mock_shape(num_joints: int, num_taxels: int) -> TactileMetadata: (taxel_positions, np.tile([1.0, 0.0, 0.0, 0.0], (num_taxels, 1))) ) return TactileMetadata( - algorithm=LBS(segment_poses, taxel_poses, joint_indices, joint_weights), - segments=HandSkeletonRef([HandJoint(i) for i in range(num_joints)]), + geometry_mapping=LBS( + segment_poses, taxel_poses, taxel_segment_indices, taxel_segment_weights + ), + segment_source=HandSkeletonRef([HandJoint(i) for i in range(num_joints)]), ) @@ -138,10 +140,10 @@ def main(argv: list[str]) -> int: if not math.isfinite(args.metadata_interval) or args.metadata_interval <= 0: parser.error("--metadata-interval must be finite and positive") - shape = mock_shape(args.joints, args.taxels).to_bytes() - if len(shape) > TRACKER_METADATA_MAX_SIZE: + metadata_bytes = mock_metadata(args.joints, args.taxels).to_bytes() + if len(metadata_bytes) > TRACKER_METADATA_MAX_SIZE: parser.error( - f"{args.joints} joints and {args.taxels} taxels make a {len(shape)} B TactileMetadata; " + f"{args.joints} joints and {args.taxels} taxels make a {len(metadata_bytes)} B TactileMetadata; " f"a tracker accepts at most {TRACKER_METADATA_MAX_SIZE} B" ) logger.info( @@ -149,7 +151,7 @@ def main(argv: list[str]) -> int: args.joints, args.taxels, args.collection_id, - len(shape), + len(metadata_bytes), ) frames = 0 @@ -157,12 +159,12 @@ def main(argv: list[str]) -> int: with ExitStack() as stack: stack.enter_context(CloudXRLauncher.launch_context(args)) data_tracker = TensorPushTracker( - args.collection_id, "tactile", TACTILE_SAMPLE_SIZE + args.collection_id, "tactile", TACTILE_MAX_FRAME_SIZE ) metadata_tracker = TensorPushTracker( tracker_metadata_collection_id(args.collection_id), TRACKER_METADATA_TENSOR_IDENTIFIER, - len(shape), + len(metadata_bytes), ) trackers = [data_tracker, metadata_tracker] oxr_session = stack.enter_context( @@ -186,7 +188,7 @@ def main(argv: list[str]) -> int: session, TactileDataArray(mock_forces(args.taxels, t)).to_bytes() ) if t - last_metadata_push >= args.metadata_interval: - metadata_tracker.push(session, shape) + metadata_tracker.push(session, metadata_bytes) last_metadata_push = t frames += 1 if frames % max(1, int(args.rate * 5)) == 0: diff --git a/examples/tactile/python/isaaccapture_examples/tactile/printer.py b/examples/tactile/python/isaaccapture_examples/tactile/printer.py index eea404db9b..8bc0b849a7 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/printer.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/printer.py @@ -54,7 +54,7 @@ def print_frame(elapsed: float | None, index: int, forces: np.ndarray) -> None: def run(session, tracker: TactileTracker, args: argparse.Namespace) -> int: start = time.monotonic() - shape_seen = False + metadata_seen = False expected_taxels = None steps = empty = 0 last_print = -1.0 @@ -64,20 +64,23 @@ def run(session, tracker: TactileTracker, args: argparse.Namespace) -> int: session.update() elapsed = time.monotonic() - start - shape = tracker.get_metadata(session) - if not shape_seen and shape is not None: - algorithm, segments = shape.algorithm, shape.segments - if algorithm is None or segments is None: + metadata = tracker.get_metadata(session) + if not metadata_seen and metadata is not None: + geometry_mapping, segment_source = ( + metadata.geometry_mapping, + metadata.segment_source, + ) + if geometry_mapping is None or segment_source is None: raise ValueError( - "TactileMetadata must specify an algorithm and segments" + "TactileMetadata must specify geometry_mapping and segment_source" ) - expected_taxels = len(algorithm.taxel_bind_pose) + expected_taxels = len(geometry_mapping.taxel_bind_poses) print( - f"{GREEN}[{elapsed:6.2f}s] (shape received) " - f"{len(segments.joints)} joints, " + f"{GREEN}[{elapsed:6.2f}s] (metadata received) " + f"{len(segment_source.joint_ids)} joints, " f"{expected_taxels} taxels{RESET}" ) - shape_seen = True + metadata_seen = True data = tracker.get_data(session) if data is None: @@ -87,7 +90,7 @@ def run(session, tracker: TactileTracker, args: argparse.Namespace) -> int: else: empty = 0 steps += 1 - force_array = np.asarray(data.taxel_force) + force_array = np.asarray(data.taxel_forces) if expected_taxels is not None and len(force_array) != expected_taxels: raise ValueError( f"frame has {len(force_array)} taxels, " diff --git a/scripts/check_licenses.py b/scripts/check_licenses.py index ddf86560ab..fbbc0ac5c9 100644 --- a/scripts/check_licenses.py +++ b/scripts/check_licenses.py @@ -123,6 +123,8 @@ def fix_license(project: Project, path: Path) -> bool: if expected == EGOPOST_LICENSE and expressions == {"Apache-2.0"}: content = path.read_text() # Replace only the default header tag, retaining copyright and other notices. + # SPDX patterns are data, not license declarations for this file. + # REUSE-IgnoreStart updated, count = re.subn( r"(SPDX-License-Identifier:[ \t]*)Apache-2\.0(?=[ \t]*(?:\*/|-->)?[ \t]*$)", rf"\g<1>{EGOPOST_LICENSE}", @@ -130,6 +132,7 @@ def fix_license(project: Project, path: Path) -> bool: count=1, flags=re.MULTILINE, ) + # REUSE-IgnoreEnd if count: path.write_text(updated) print(f"{path}: updated default license to {EGOPOST_LICENSE}") diff --git a/src/core/codegen/generate_trackers.py b/src/core/codegen/generate_trackers.py index 2c1b46e5f2..acd62c73f2 100644 --- a/src/core/codegen/generate_trackers.py +++ b/src/core/codegen/generate_trackers.py @@ -15,12 +15,15 @@ from manifest import load_manifest from templates import TrackerGenContext, enrich_context +# Generated SPDX text is string data, not this file's license declaration. +# REUSE-IgnoreStart COPYRIGHT = ( "// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" "// " "SPDX-License-Identifier: Apache-2.0\n\n" "// AUTO-GENERATED by src/core/codegen/generate_trackers.py — do not edit.\n\n" ) +# REUSE-IgnoreEnd def _write_if_changed(path: Path, content: str) -> bool: diff --git a/src/core/codegen/templates.py b/src/core/codegen/templates.py index 026ced75a9..53cad9dcd2 100644 --- a/src/core/codegen/templates.py +++ b/src/core/codegen/templates.py @@ -11,6 +11,7 @@ def _header() -> str: + # REUSE-IgnoreStart return ( "// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" "// SPDX-License-Identifier: Apache-2.0\n" @@ -18,6 +19,7 @@ def _header() -> str: "// AUTO-GENERATED by src/core/codegen/generate_trackers.py — do not edit.\n" "\n" ) + # REUSE-IgnoreEnd def camel_to_snake(value: str) -> str: diff --git a/src/core/schema/fbs/tactile_array.fbs b/src/core/schema/fbs/tactile_array.fbs index 97b6d39776..4343ba8eb0 100644 --- a/src/core/schema/fbs/tactile_array.fbs +++ b/src/core/schema/fbs/tactile_array.fbs @@ -9,9 +9,9 @@ namespace core; // Per-frame tactile forces. TactileMetadata (tactile_metadata.fbs) defines the // taxel order and reference geometry for the same tracker. table TactileDataArray { - // Force vectors in newtons, expressed in taxel-local axes. For LBS, taxel_bind_pose - // defines these axes at bind; the vector length equals the metadata taxel count. - taxel_force: [Point] (id: 0); + // Force vectors in newtons, expressed in taxel-local axes. For LBS, taxel_bind_poses + // define these axes at bind; the vector length equals the metadata taxel count. + taxel_forces: [Point] (id: 0); } // MCAP recording wrapper for TactileDataArray. diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs index 432e518975..7397fe2fee 100644 --- a/src/core/schema/fbs/tactile_metadata.fbs +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -10,7 +10,7 @@ table HandSkeletonRef { // Ordered, unique HandJoint identifiers; NUM_JOINTS is not a joint. // Segment indices address this vector, not HandJoint ordinals. Bind and current // segment poses map joint-local coordinates into the same hand-root frame. - joints: [HandJoint] (id: 0); + joint_ids: [HandJoint] (id: 0); } union SegmentSource { @@ -28,14 +28,14 @@ struct Float4 { table LBS { // Segment-to-source-frame bind poses, in SegmentSource order. Positions are in // meters and orientations are unit quaternions (x, y, z, w). - segment_bind_pose: [Pose] (id: 0); + segment_bind_poses: [Pose] (id: 0); - // Taxel-local-to-source-frame bind poses, in TactileDataArray.taxel_force order. - // Uses the same units and quaternion convention as segment_bind_pose. - taxel_bind_pose: [Pose] (id: 1); + // Taxel-local-to-source-frame bind poses, in TactileDataArray.taxel_forces order. + // Uses the same units and quaternion convention as segment_bind_poses. + taxel_bind_poses: [Pose] (id: 1); // One four-slot row per taxel; all taxel_* vectors have the same length. - // Every slot indexes segment_bind_pose, including slots whose weight is zero. + // Every slot indexes segment_bind_poses, including slots whose weight is zero. taxel_segment_indices: [UInt4] (id: 2); // Finite, nonnegative weights, summing to one per taxel. Bind positions move as @@ -44,6 +44,7 @@ table LBS { taxel_segment_weights: [Float4] (id: 3); } +// Static parameters for the selected taxel geometry-mapping method. union AlgorithmSource { LBS = 1, } @@ -52,11 +53,11 @@ union AlgorithmSource { // A usable layout sets both unions; lengths, indices and weights require semantic // validation by producers/consumers, beyond FlatBuffers buffer verification. table TactileMetadata { - // The implicit algorithm_type discriminator occupies id 0. - algorithm: AlgorithmSource (id: 1); + // The implicit geometry_mapping_type discriminator occupies id 0. + geometry_mapping: AlgorithmSource (id: 1); - // The implicit segments_type discriminator occupies id 2. - segments: SegmentSource (id: 3); + // The implicit segment_source_type discriminator occupies id 2. + segment_source: SegmentSource (id: 3); } root_type TactileMetadata; diff --git a/src/core/schema/python/tactile_array_bindings.h b/src/core/schema/python/tactile_array_bindings.h index 0a140ac284..547db29d42 100644 --- a/src/core/schema/python/tactile_array_bindings.h +++ b/src/core/schema/python/tactile_array_bindings.h @@ -28,28 +28,28 @@ inline void bind_tactile_array(py::module& m) serialized_class(m, "TactileDataArray", "Encoded per-frame tactile reading.") .def(py::init( - [](const std::optional& taxel_force) + [](const std::optional& taxel_forces) { TactileDataArrayT native; - if (taxel_force.has_value()) + if (taxel_forces.has_value()) { - native.taxel_force = structs_from_rows(*taxel_force, "taxel_force"); + native.taxel_forces = structs_from_rows(*taxel_forces, "taxel_forces"); } return pack(native); }), - py::arg("taxel_force") = py::none(), "Encode a tactile frame. taxel_force is an (N, 3) array.") + py::arg("taxel_forces") = py::none(), "Encode a tactile frame. taxel_forces is an (N, 3) array.") .def_property_readonly( - "taxel_force", + "taxel_forces", [](py::object self) { const auto& view = self.cast&>(); - return struct_vector_view(self, view->taxel_force()); + return struct_vector_view(self, view->taxel_forces()); }, "Per-taxel force as an (N, 3) float32 view aliasing this object's buffer; call .copy() to own it.") .def("__repr__", [](const Serialized& self) { - const auto* forces = self->taxel_force(); + const auto* forces = self->taxel_forces(); return "TactileDataArray(taxels=" + std::to_string(forces != nullptr ? forces->size() : 0) + ")"; }); diff --git a/src/core/schema/python/tactile_metadata_bindings.h b/src/core/schema/python/tactile_metadata_bindings.h index 8163057860..be737afdc5 100644 --- a/src/core/schema/python/tactile_metadata_bindings.h +++ b/src/core/schema/python/tactile_metadata_bindings.h @@ -36,25 +36,27 @@ inline void bind_tactile_metadata(py::module& m) .value("HandSkeletonRef", SegmentSource_HandSkeletonRef); py::enum_(m, "AlgorithmSource").value("NONE", AlgorithmSource_NONE).value("LBS", AlgorithmSource_LBS); - serialized_class(m, "HandSkeletonRef", "Ordered hand joints referenced by tactile segments.") + serialized_class( + m, "HandSkeletonRef", "Ordered hand joint identifiers referenced by tactile segments.") .def(py::init( - [](const std::vector& joints) + [](const std::vector& joint_ids) { std::array seen{}; - for (const auto joint : joints) + for (const auto joint : joint_ids) { if (joint >= HandJoint_NUM_JOINTS || seen[joint]) { - throw py::value_error("joints: expected unique HandJoint identifiers, excluding NUM_JOINTS"); + throw py::value_error( + "joint_ids: expected unique HandJoint identifiers, excluding NUM_JOINTS"); } seen[joint] = true; } HandSkeletonRefT native; - native.joints = joints; + native.joint_ids = joint_ids; return pack(native); }), - py::arg("joints")) - .def_property_readonly("joints", vector_field(&HandSkeletonRef::joints)); + py::arg("joint_ids")) + .def_property_readonly("joint_ids", vector_field(&HandSkeletonRef::joint_ids)); auto poses_from_rows = [](const FloatRows& rows, const char* name) { @@ -80,17 +82,17 @@ inline void bind_tactile_metadata(py::module& m) serialized_class(m, "LBS", "Linear blend skinning bind poses and four-slot influences.") .def(py::init( - [poses_from_rows](const FloatRows& segment_bind_pose, const FloatRows& taxel_bind_pose, + [poses_from_rows](const FloatRows& segment_bind_poses, const FloatRows& taxel_bind_poses, const IndexRows& taxel_segment_indices, const FloatRows& taxel_segment_weights) { LBST native; - native.segment_bind_pose = poses_from_rows(segment_bind_pose, "segment_bind_pose"); - native.taxel_bind_pose = poses_from_rows(taxel_bind_pose, "taxel_bind_pose"); + native.segment_bind_poses = poses_from_rows(segment_bind_poses, "segment_bind_poses"); + native.taxel_bind_poses = poses_from_rows(taxel_bind_poses, "taxel_bind_poses"); native.taxel_segment_indices = structs_from_rows(taxel_segment_indices, "taxel_segment_indices"); native.taxel_segment_weights = structs_from_rows(taxel_segment_weights, "taxel_segment_weights"); - const size_t taxels = native.taxel_bind_pose.size(); + const size_t taxels = native.taxel_bind_poses.size(); if (native.taxel_segment_indices.size() != taxels || native.taxel_segment_weights.size() != taxels) { throw py::value_error("all taxel_* arrays must have the same length"); @@ -100,7 +102,7 @@ inline void bind_tactile_metadata(py::module& m) float total = 0; for (int k = 0; k < 4; ++k) { - if (native.taxel_segment_indices[i].values()->Get(k) >= native.segment_bind_pose.size()) + if (native.taxel_segment_indices[i].values()->Get(k) >= native.segment_bind_poses.size()) { throw py::value_error("taxel_segment_indices: slot is not a segment index"); } @@ -118,15 +120,15 @@ inline void bind_tactile_metadata(py::module& m) } return pack(native); }), - py::arg("segment_bind_pose"), py::arg("taxel_bind_pose"), py::arg("taxel_segment_indices"), + py::arg("segment_bind_poses"), py::arg("taxel_bind_poses"), py::arg("taxel_segment_indices"), py::arg("taxel_segment_weights"), "Encode bind poses as (J, 7)/(N, 7) XYZ/XYZW rows and influences as (N, 4) arrays.") .def_property_readonly( - "segment_bind_pose", [](py::object self) - { return struct_vector_view(self, self.cast&>()->segment_bind_pose()); }) + "segment_bind_poses", [](py::object self) + { return struct_vector_view(self, self.cast&>()->segment_bind_poses()); }) .def_property_readonly( - "taxel_bind_pose", [](py::object self) - { return struct_vector_view(self, self.cast&>()->taxel_bind_pose()); }) + "taxel_bind_poses", [](py::object self) + { return struct_vector_view(self, self.cast&>()->taxel_bind_poses()); }) .def_property_readonly("taxel_segment_indices", [](py::object self) { return struct_vector_view( @@ -138,49 +140,50 @@ inline void bind_tactile_metadata(py::module& m) serialized_class(m, "TactileMetadata", "Static tactile layout and its segment source.") .def(py::init( - [](const Serialized& algorithm, const Serialized& segments) + [](const Serialized& geometry_mapping, const Serialized& segment_source) { - const auto* poses = algorithm->segment_bind_pose(); - const auto* joints = segments->joints(); - if ((poses != nullptr ? poses->size() : 0) != (joints != nullptr ? joints->size() : 0)) + const auto* poses = geometry_mapping->segment_bind_poses(); + const auto* joint_ids = segment_source->joint_ids(); + if ((poses != nullptr ? poses->size() : 0) != (joint_ids != nullptr ? joint_ids->size() : 0)) { - throw py::value_error("segment_bind_pose and segments.joints must have the same length"); + throw py::value_error( + "segment_bind_poses and segment_source.joint_ids must have the same length"); } TactileMetadataT native; LBST lbs; - algorithm->UnPackTo(&lbs); - native.algorithm.Set(std::move(lbs)); + geometry_mapping->UnPackTo(&lbs); + native.geometry_mapping.Set(std::move(lbs)); HandSkeletonRefT hand; - segments->UnPackTo(&hand); - native.segments.Set(std::move(hand)); + segment_source->UnPackTo(&hand); + native.segment_source.Set(std::move(hand)); return pack(native); }), - py::arg("algorithm"), py::arg("segments")) - .def_property_readonly("algorithm_type", field(&TactileMetadata::algorithm_type)) - .def_property_readonly("segments_type", field(&TactileMetadata::segments_type)) - .def_property_readonly("algorithm", + py::arg("geometry_mapping"), py::arg("segment_source")) + .def_property_readonly("geometry_mapping_type", field(&TactileMetadata::geometry_mapping_type)) + .def_property_readonly("segment_source_type", field(&TactileMetadata::segment_source_type)) + .def_property_readonly("geometry_mapping", [](const Serialized& self) -> std::optional> { - if (self->algorithm_type() == AlgorithmSource_NONE) + if (self->geometry_mapping_type() == AlgorithmSource_NONE) { return std::nullopt; } - if (const auto* algorithm = self->algorithm_as_LBS()) + if (const auto* geometry_mapping = self->geometry_mapping_as_LBS()) { - return self.narrow(algorithm); + return self.narrow(geometry_mapping); } - throw py::value_error("unsupported tactile algorithm"); + throw py::value_error("unsupported tactile geometry mapping"); }) - .def_property_readonly("segments", + .def_property_readonly("segment_source", [](const Serialized& self) -> std::optional> { - if (self->segments_type() == SegmentSource_NONE) + if (self->segment_source_type() == SegmentSource_NONE) { return std::nullopt; } - if (const auto* segments = self->segments_as_HandSkeletonRef()) + if (const auto* segment_source = self->segment_source_as_HandSkeletonRef()) { - return self.narrow(segments); + return self.narrow(segment_source); } throw py::value_error("unsupported tactile segment source"); }); diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index c6233c7941..6009c94f32 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -44,8 +44,8 @@ from .haptic_sink import HapticSink from .tactile_source import ( TactileForces, - TactileShapeIndex, - TactileShapeLayout, + TactileLBSLayoutIndex, + TactileLBSLayout, TactileSource, ) from .deviceio_tensor_types import ( @@ -63,7 +63,7 @@ SomaHandJointRotationsTrackedType, SomaHandJointPosesTrackedType, TactileDataTrackedType, - TactileShapeTrackedType, + TactileMetadataTrackedType, DeviceIOHeadPoseTracked, DeviceIOHandPoseTracked, DeviceIOControllerSnapshotTracked, @@ -120,9 +120,9 @@ "messageChannelConfig", "HapticSink", "TactileSource", - "TactileShapeIndex", + "TactileLBSLayoutIndex", "TactileForces", - "TactileShapeLayout", + "TactileLBSLayout", "HeadPoseTrackedType", "HandPoseTrackedType", "ControllerSnapshotTrackedType", @@ -137,7 +137,7 @@ "SomaHandJointRotationsTrackedType", "SomaHandJointPosesTrackedType", "TactileDataTrackedType", - "TactileShapeTrackedType", + "TactileMetadataTrackedType", "MessageChannelMessagesTrackedType", "MessageChannelConnectionStatus", "MessageChannelStatusType", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py index 3aaabddc6a..4516836d26 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py @@ -165,7 +165,7 @@ class TactileDataTrackedType(_PayloadTensorType): _payload_cls = TactileDataArray -class TactileShapeTrackedType(_PayloadTensorType): +class TactileMetadataTrackedType(_PayloadTensorType): """TactileMetadata metadata from DeviceIO TactileTracker.""" _payload_cls = TactileMetadata @@ -328,17 +328,17 @@ def DeviceIOSomaHandJointPosesTracked() -> TensorGroupType: def DeviceIOTactileTracked() -> TensorGroupType: - """Tracked tactile frame and its shape from DeviceIO TactileTracker. + """Tracked tactile frame and its metadata from DeviceIO TactileTracker. Contains: tactile_tracked: TactileDataArray handle, or None when inactive - tactile_shape: TactileMetadata metadata handle, or None until it arrives + tactile_metadata: TactileMetadata metadata handle, or None until it arrives """ return TensorGroupType( "deviceio_tactile", [ TactileDataTrackedType("tactile_tracked"), - TactileShapeTrackedType("tactile_shape"), + TactileMetadataTrackedType("tactile_metadata"), ], ) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 6da125fd9e..5e74d068e8 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -39,32 +39,32 @@ def TactileForces(num_taxels: int) -> TensorGroupType: """Per-taxel force vectors: one ``(num_taxels, 3)`` float32 array.""" return TensorGroupType( f"tactile_forces_{num_taxels}", - [_float_array("taxel_force", num_taxels, 3)], + [_float_array("taxel_forces", num_taxels, 3)], ) -class TactileShapeIndex(IntEnum): - """Positions of the arrays in a :func:`TactileShapeLayout` group.""" +class TactileLBSLayoutIndex(IntEnum): + """Positions of the arrays in a :func:`TactileLBSLayout` group.""" - SEGMENT_JOINTS = 0 - SEGMENT_BIND_POSE = 1 - TAXEL_BIND_POSE = 2 + SEGMENT_JOINT_IDS = 0 + SEGMENT_BIND_POSES = 1 + TAXEL_BIND_POSES = 2 TAXEL_SEGMENT_INDICES = 3 TAXEL_SEGMENT_WEIGHTS = 4 -def TactileShapeLayout(num_joints: int, num_taxels: int) -> TensorGroupType: - """``TactileMetadata`` as arrays, ordered by :class:`TactileShapeIndex`. +def TactileLBSLayout(num_segments: int, num_taxels: int) -> TensorGroupType: + """LBS and HandSkeletonRef arrays, ordered by :class:`TactileLBSLayoutIndex`. HandJoint identifiers and segment indices are int32. Pose rows contain position XYZ followed by quaternion XYZW, all in hand-root space. """ return TensorGroupType( - f"tactile_shape_{num_joints}x{num_taxels}", + f"tactile_lbs_layout_{num_segments}x{num_taxels}", [ - _int_array("segment_joints", (num_joints,)), - _float_array("segment_bind_pose", num_joints, 7), - _float_array("taxel_bind_pose", num_taxels, 7), + _int_array("segment_joint_ids", (num_segments,)), + _float_array("segment_bind_poses", num_segments, 7), + _float_array("taxel_bind_poses", num_taxels, 7), _int_array("taxel_segment_indices", (num_taxels, 4)), _float_array("taxel_segment_weights", num_taxels, 4), ], @@ -72,25 +72,25 @@ def TactileShapeLayout(num_joints: int, num_taxels: int) -> TensorGroupType: class TactileSource(IDeviceIOSource): - """Converter: DeviceIO ``TactileTracker`` -> per-taxel forces and the shape that lays them out. + """Convert tactile frames and metadata into per-taxel forces and an LBS layout. Outputs (Optional): - :data:`FORCES`: :func:`TactileForces` of ``num_taxels``, in ``TactileMetadata`` taxel order; absent when the device is inactive. - - :data:`SHAPE`: :func:`TactileShapeLayout` of ``num_joints`` x ``num_taxels``; absent - until the tracker holds the device's shape. The tracker publishes no forces before then. + - :data:`LAYOUT`: :func:`TactileLBSLayout` of ``num_segments`` x ``num_taxels``; absent + until the tracker holds the device's metadata. The tracker publishes no forces before then. - A frame or shape whose counts differ from the constructor's raises ``ValueError``. + A frame or metadata whose counts differ from the constructor's raises ``ValueError``. """ FORCES = "taxel_forces" - SHAPE = "tactile_shape" + LAYOUT = "tactile_layout" def __init__( self, name: str, collection_id: str, - num_joints: int, + num_segments: int, num_taxels: int, max_flatbuffer_size: Optional[int] = None, ) -> None: @@ -99,7 +99,7 @@ def __init__( Args: name: Unique name for this source node. collection_id: Tensor collection ID of the producer's ``TactileDataArray`` stream. - num_joints: Joint count of the device skeleton; fixes the shape output spec. + num_segments: Referenced joint count; fixes the LBS layout output spec. num_taxels: Taxel count of the device; fixes the output specs. max_flatbuffer_size: Override for ``TactileTracker``'s default collection size. """ @@ -110,7 +110,7 @@ def __init__( if max_flatbuffer_size is None else TactileTracker(collection_id, max_flatbuffer_size) ) - self._num_joints = int(num_joints) + self._num_segments = int(num_segments) self._num_taxels = int(num_taxels) super().__init__(name) @@ -130,55 +130,58 @@ def input_spec(self) -> RetargeterIOType: return {"deviceio_tactile": DeviceIOTactileTracked()} def output_spec(self) -> RetargeterIOType: - """Declare the force and shape outputs (Optional -- may be absent).""" + """Declare the force and LBS layout outputs (Optional -- may be absent).""" return { self.FORCES: OptionalType(TactileForces(self._num_taxels)), - self.SHAPE: OptionalType( - TactileShapeLayout(self._num_joints, self._num_taxels) + self.LAYOUT: OptionalType( + TactileLBSLayout(self._num_segments, self._num_taxels) ), } def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: - """Publish the frame's forces and a copy of the shape; ``set_none()`` whichever is absent.""" + """Publish forces and a copied LBS layout; mark absent outputs with ``set_none()``.""" data: "TactileDataArray | None" = inputs["deviceio_tactile"][0] - shape: "TactileMetadata | None" = inputs["deviceio_tactile"][1] + metadata: "TactileMetadata | None" = inputs["deviceio_tactile"][1] forces_out = outputs[self.FORCES] if data is None: forces_out.set_none() else: - forces = data.taxel_force + forces = data.taxel_forces if len(forces) != self._num_taxels: raise ValueError( f"{self.name}: frame has {len(forces)} taxels, source expects {self._num_taxels}" ) forces_out[0] = forces - shape_out = outputs[self.SHAPE] - if shape is None: - shape_out.set_none() + layout_out = outputs[self.LAYOUT] + if metadata is None: + layout_out.set_none() return - algorithm = shape.algorithm - segments = shape.segments - if algorithm is None or segments is None: + geometry_mapping = metadata.geometry_mapping + segment_source = metadata.segment_source + if geometry_mapping is None or segment_source is None: raise ValueError( - f"{self.name}: metadata must specify an algorithm and segments" + f"{self.name}: metadata must specify geometry_mapping and segment_source" ) - joints, taxels = len(segments.joints), len(algorithm.taxel_bind_pose) - if (joints, taxels) != (self._num_joints, self._num_taxels): + num_segments, num_taxels = ( + len(segment_source.joint_ids), + len(geometry_mapping.taxel_bind_poses), + ) + if (num_segments, num_taxels) != (self._num_segments, self._num_taxels): raise ValueError( - f"{self.name}: shape has {joints} joints and {taxels} taxels, " - f"source expects {self._num_joints} and {self._num_taxels}" + f"{self.name}: metadata has {num_segments} segments and {num_taxels} taxels, " + f"source expects {self._num_segments} and {self._num_taxels}" ) values = ( - np.asarray(segments.joints, dtype=np.int32), - algorithm.segment_bind_pose, - algorithm.taxel_bind_pose, - algorithm.taxel_segment_indices, - algorithm.taxel_segment_weights, + np.asarray(segment_source.joint_ids, dtype=np.int32), + geometry_mapping.segment_bind_poses, + geometry_mapping.taxel_bind_poses, + geometry_mapping.taxel_segment_indices, + geometry_mapping.taxel_segment_weights, ) for index, value in enumerate(values): # Copied: the views alias the metadata the tracker holds and compares later pushes to. - shape_out[index] = value.astype( + layout_out[index] = value.astype( np.int32 if value.dtype == np.uint32 else value.dtype ) diff --git a/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp b/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp index 98be2a9815..4759840d0e 100644 --- a/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp +++ b/tests/cpp/core/replay_deviceio_session/test_replay_session.cpp @@ -847,24 +847,24 @@ void write_tactile_recording(const std::string& path, const std::string& base_na if (with_metadata) { - core::HandSkeletonRefT segments; - segments.joints = { core::HandJoint_WRIST }; + core::HandSkeletonRefT segment_source; + segment_source.joint_ids = { core::HandJoint_WRIST }; core::LBST lbs; - lbs.segment_bind_pose = { make_pose(0, 0, 0) }; - lbs.taxel_bind_pose = { make_pose(0, 0, 0), make_pose(0, 0, 0) }; + lbs.segment_bind_poses = { make_pose(0, 0, 0) }; + lbs.taxel_bind_poses = { make_pose(0, 0, 0), make_pose(0, 0, 0) }; lbs.taxel_segment_indices.resize(2); const std::array weights = { 1, 0, 0, 0 }; lbs.taxel_segment_weights.assign(2, core::Float4(flatbuffers::span(weights.data(), 4))); - core::TactileMetadataT shape; - shape.algorithm.Set(std::move(lbs)); - shape.segments.Set(std::move(segments)); - metadata.write(0, core::pack(shape), 500); + core::TactileMetadataT native; + native.geometry_mapping.Set(std::move(lbs)); + native.segment_source.Set(std::move(segment_source)); + metadata.write(0, core::pack(native), 500); } for (int i = 0; i < 2; ++i) { core::TactileDataArrayT frame; - frame.taxel_force = { core::Point(static_cast(i), 0, 0), core::Point(0, 0, 1) }; + frame.taxel_forces = { core::Point(static_cast(i), 0, 0), core::Point(0, 0, 1) }; const int64_t time_ns = (i + 1) * 1000; const auto record = core::pack_record( &frame, core::DeviceDataTimestamp(time_ns, time_ns, time_ns)); @@ -894,18 +894,18 @@ TEST_CASE("ReplaySession: tracker metadata is replayed before its data", "[repla auto session = core::ReplaySession::run(config); // Read when the session opens, so it is there before the first frame. - const auto& shape = tracker.get_metadata(*session); - REQUIRE(shape); - REQUIRE(shape->algorithm_as_LBS()); - CHECK(shape->algorithm_as_LBS()->taxel_bind_pose()->size() == 2); + const auto& metadata = tracker.get_metadata(*session); + REQUIRE(metadata); + REQUIRE(metadata->geometry_mapping_as_LBS()); + CHECK(metadata->geometry_mapping_as_LBS()->taxel_bind_poses()->size() == 2); for (int i = 0; i < 2; ++i) { session->update(); const auto& frame = tracker.get_data(*session); REQUIRE(frame); - CHECK(frame->taxel_force()->Get(0)->x() == static_cast(i)); - CHECK(tracker.get_metadata(*session).get() == shape.get()); + CHECK(frame->taxel_forces()->Get(0)->x() == static_cast(i)); + CHECK(tracker.get_metadata(*session).get() == metadata.get()); } session->update(); diff --git a/tests/cpp/core/schema/test_tactile_array.cpp b/tests/cpp/core/schema/test_tactile_array.cpp index 42e0c3acd1..7c2e4cdf32 100644 --- a/tests/cpp/core/schema/test_tactile_array.cpp +++ b/tests/cpp/core/schema/test_tactile_array.cpp @@ -15,11 +15,12 @@ // VT_ == (id + 2) * 2. #define VT(field) (field + 2) * 2 -static_assert(core::TactileDataArray::VT_TAXEL_FORCE == VT(0)); +// Field renames change generated VT_* symbols without changing field IDs. +static_assert(core::TactileDataArray::VT_TAXEL_FORCES == VT(0)); static_assert(core::TactileDataArrayRecord::VT_DATA == VT(0)); static_assert(core::TactileDataArrayRecord::VT_TIMESTAMP == VT(1)); -// Producers copy XrVector3f arrays straight into taxel_force. +// Producers copy XrVector3f arrays straight into taxel_forces. static_assert(sizeof(core::Point) == sizeof(XrVector3f)); namespace @@ -28,7 +29,7 @@ namespace core::TactileDataArrayT make_frame() { core::TactileDataArrayT native; - native.taxel_force = { core::Point(0.0f, 0.0f, 1.0f), core::Point(0.5f, -0.5f, 2.0f) }; + native.taxel_forces = { core::Point(0.0f, 0.0f, 1.0f), core::Point(0.5f, -0.5f, 2.0f) }; return native; } @@ -39,10 +40,10 @@ TEST_CASE("TactileDataArray round-trips every field", "[unit][tactile]") const auto data = core::pack(make_frame()); REQUIRE(data); - REQUIRE(data->taxel_force() != nullptr); - REQUIRE(data->taxel_force()->size() == 2); - CHECK(data->taxel_force()->Get(1)->x() == 0.5f); - CHECK(data->taxel_force()->Get(1)->z() == 2.0f); + REQUIRE(data->taxel_forces() != nullptr); + REQUIRE(data->taxel_forces()->size() == 2); + CHECK(data->taxel_forces()->Get(1)->x() == 0.5f); + CHECK(data->taxel_forces()->Get(1)->z() == 2.0f); } TEST_CASE("TactileDataArray leaves an empty frame's vector absent", "[unit][tactile]") @@ -50,7 +51,7 @@ TEST_CASE("TactileDataArray leaves an empty frame's vector absent", "[unit][tact const auto data = core::pack(core::TactileDataArrayT{}); REQUIRE(data); - CHECK(data->taxel_force() == nullptr); + CHECK(data->taxel_forces() == nullptr); } TEST_CASE("TactileDataArrayRecord verifies and carries its timestamp", "[unit][tactile]") @@ -66,5 +67,5 @@ TEST_CASE("TactileDataArrayRecord verifies and carries its timestamp", "[unit][t REQUIRE(core::VerifyTactileDataArrayRecordBuffer(verifier)); const auto* read = core::GetTactileDataArrayRecord(builder.GetBufferPointer()); CHECK(read->timestamp()->sample_time_local_common_clock() == 2); - CHECK(read->data()->taxel_force()->size() == 2); + CHECK(read->data()->taxel_forces()->size() == 2); } diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index 1a3156ed32..b5be4149cd 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -12,13 +12,13 @@ #include #define VT(field) (field + 2) * 2 -static_assert(core::TactileMetadata::VT_ALGORITHM_TYPE == VT(0)); -static_assert(core::TactileMetadata::VT_ALGORITHM == VT(1)); -static_assert(core::TactileMetadata::VT_SEGMENTS_TYPE == VT(2)); -static_assert(core::TactileMetadata::VT_SEGMENTS == VT(3)); -static_assert(core::HandSkeletonRef::VT_JOINTS == VT(0)); -static_assert(core::LBS::VT_SEGMENT_BIND_POSE == VT(0)); -static_assert(core::LBS::VT_TAXEL_BIND_POSE == VT(1)); +static_assert(core::TactileMetadata::VT_GEOMETRY_MAPPING_TYPE == VT(0)); +static_assert(core::TactileMetadata::VT_GEOMETRY_MAPPING == VT(1)); +static_assert(core::TactileMetadata::VT_SEGMENT_SOURCE_TYPE == VT(2)); +static_assert(core::TactileMetadata::VT_SEGMENT_SOURCE == VT(3)); +static_assert(core::HandSkeletonRef::VT_JOINT_IDS == VT(0)); +static_assert(core::LBS::VT_SEGMENT_BIND_POSES == VT(0)); +static_assert(core::LBS::VT_TAXEL_BIND_POSES == VT(1)); static_assert(core::LBS::VT_TAXEL_SEGMENT_INDICES == VT(2)); static_assert(core::LBS::VT_TAXEL_SEGMENT_WEIGHTS == VT(3)); static_assert(sizeof(core::UInt4) == 4 * sizeof(uint32_t)); @@ -29,19 +29,19 @@ namespace core::TactileMetadataT make_metadata() { - core::HandSkeletonRefT segments; - segments.joints = { core::HandJoint_WRIST, core::HandJoint_INDEX_TIP }; + core::HandSkeletonRefT segment_source; + segment_source.joint_ids = { core::HandJoint_WRIST, core::HandJoint_INDEX_TIP }; core::LBST lbs; - lbs.segment_bind_pose = { core::Pose(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)), - core::Pose(core::Point(0, 0.1f, 0), core::Quaternion(0, 0, 0, 1)) }; - lbs.taxel_bind_pose = { core::Pose(core::Point(0, 0.09f, 0.01f), core::Quaternion(1, 0, 0, 0)) }; + lbs.segment_bind_poses = { core::Pose(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)), + core::Pose(core::Point(0, 0.1f, 0), core::Quaternion(0, 0, 0, 1)) }; + lbs.taxel_bind_poses = { core::Pose(core::Point(0, 0.09f, 0.01f), core::Quaternion(1, 0, 0, 0)) }; const std::array indices = { 1, 0, 0, 0 }; const std::array weights = { 0.75f, 0.25f, 0.0f, 0.0f }; lbs.taxel_segment_indices.emplace_back(flatbuffers::span(indices.data(), 4)); lbs.taxel_segment_weights.emplace_back(flatbuffers::span(weights.data(), 4)); core::TactileMetadataT native; - native.algorithm.Set(std::move(lbs)); - native.segments.Set(std::move(segments)); + native.geometry_mapping.Set(std::move(lbs)); + native.segment_source.Set(std::move(segment_source)); return native; } @@ -50,18 +50,18 @@ core::TactileMetadataT make_metadata() TEST_CASE("TactileMetadata round-trips union tags and segment-local indices", "[unit][tactile_metadata]") { const auto metadata = core::pack(make_metadata()); - CHECK(metadata->algorithm_type() == core::AlgorithmSource_LBS); - CHECK(metadata->segments_type() == core::SegmentSource_HandSkeletonRef); - const auto* segments = metadata->segments_as_HandSkeletonRef(); - const auto* lbs = metadata->algorithm_as_LBS(); - REQUIRE(segments); + CHECK(metadata->geometry_mapping_type() == core::AlgorithmSource_LBS); + CHECK(metadata->segment_source_type() == core::SegmentSource_HandSkeletonRef); + const auto* segment_source = metadata->segment_source_as_HandSkeletonRef(); + const auto* lbs = metadata->geometry_mapping_as_LBS(); + REQUIRE(segment_source); REQUIRE(lbs); - REQUIRE(segments->joints()->size() == 2); - CHECK(segments->joints()->Get(1) == core::HandJoint_INDEX_TIP); + REQUIRE(segment_source->joint_ids()->size() == 2); + CHECK(segment_source->joint_ids()->Get(1) == core::HandJoint_INDEX_TIP); CHECK(lbs->taxel_segment_indices()->Get(0)->values()->Get(0) == 1); CHECK(lbs->taxel_segment_weights()->Get(0)->values()->Get(1) == 0.25f); - CHECK(lbs->segment_bind_pose()->Get(1)->position().y() == 0.1f); - CHECK(lbs->taxel_bind_pose()->Get(0)->orientation().x() == 1.0f); + CHECK(lbs->segment_bind_poses()->Get(1)->position().y() == 0.1f); + CHECK(lbs->taxel_bind_poses()->Get(0)->orientation().x() == 1.0f); } TEST_CASE("TactileMetadata verifies and retains nested union views", "[unit][tactile_metadata]") @@ -71,7 +71,7 @@ TEST_CASE("TactileMetadata verifies and retains nested union views", "[unit][tac flatbuffers::Verifier verifier(bytes.data(), bytes.size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); CHECK(std::string(core::TactileMetadata::GetFullyQualifiedName()) == "core.TactileMetadata"); - auto lbs = metadata.narrow(metadata->algorithm_as_LBS()); + auto lbs = metadata.narrow(metadata->geometry_mapping_as_LBS()); metadata.reset(); REQUIRE(lbs); CHECK(lbs->taxel_segment_weights()->Get(0)->values()->Get(0) == 0.75f); diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 81aeb51c26..6ba468218a 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -10,8 +10,8 @@ import pytest from isaaccapture.retargeting_engine.deviceio_source_nodes import ( - TactileShapeIndex, - TactileShapeLayout, + TactileLBSLayoutIndex, + TactileLBSLayout, TactileSource, ) from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group @@ -24,22 +24,22 @@ TactileMetadata, ) -NUM_JOINTS = 2 +NUM_SEGMENTS = 2 NUM_TAXELS = 3 def _source(**kwargs) -> TactileSource: args = dict( - collection_id="tactile/right", num_joints=NUM_JOINTS, num_taxels=NUM_TAXELS + collection_id="tactile/right", num_segments=NUM_SEGMENTS, num_taxels=NUM_TAXELS ) args.update(kwargs) return TactileSource("tactile", **args) -def _inputs(source, data, shape) -> dict: +def _inputs(source, data, metadata) -> dict: tg = TensorGroup(source.input_spec()["deviceio_tactile"]) tg[0] = data - tg[1] = shape + tg[1] = metadata return {"deviceio_tactile": tg} @@ -47,20 +47,20 @@ def _outputs(source) -> dict: return {name: _make_output_group(gt) for name, gt in source.output_spec().items()} -def _shape( - num_joints: int = NUM_JOINTS, num_taxels: int = NUM_TAXELS +def _metadata( + num_segments: int = NUM_SEGMENTS, num_taxels: int = NUM_TAXELS ) -> TactileMetadata: indices = np.zeros((num_taxels, 4), np.uint32) weights = np.zeros((num_taxels, 4), np.float32) weights[:, 0] = 1.0 - segment_poses = np.zeros((num_joints, 7), np.float32) + segment_poses = np.zeros((num_segments, 7), np.float32) segment_poses[:, 6] = 1 taxel_poses = np.zeros((num_taxels, 7), np.float32) taxel_poses[:, :3] = 0.01 taxel_poses[:, 6] = 1 return TactileMetadata( LBS(segment_poses, taxel_poses, indices, weights), - HandSkeletonRef([HandJoint(i) for i in range(num_joints)]), + HandSkeletonRef([HandJoint(i) for i in range(num_segments)]), ) @@ -88,71 +88,72 @@ def test_specs(self): forces = src.output_spec()[TactileSource.FORCES] assert forces.is_optional assert forces.inner_type.types[0].shape == (NUM_TAXELS, 3) - assert src.output_spec()[TactileSource.SHAPE].is_optional + assert src.output_spec()[TactileSource.LAYOUT].is_optional def test_index_matches_layout_order(self): - names = [t.name for t in TactileShapeLayout(NUM_JOINTS, NUM_TAXELS).types] - assert names == [member.name.lower() for member in TactileShapeIndex] + names = [t.name for t in TactileLBSLayout(NUM_SEGMENTS, NUM_TAXELS).types] + assert names == [member.name.lower() for member in TactileLBSLayoutIndex] def test_poll_reads_data_and_metadata_from_one_tracker(self): src = _source() data = TactileDataArray(np.zeros((NUM_TAXELS, 3), np.float32)) - shape = _shape() - src._tracker = _FakeTracker(data, shape) + metadata = _metadata() + src._tracker = _FakeTracker(data, metadata) polled = src.poll_tracker(None)["deviceio_tactile"] assert polled[0] is data - assert polled[1] is shape + assert polled[1] is metadata def test_active_conversion(self): src = _source() forces = np.arange(9, dtype=np.float32).reshape(3, 3) outputs = _outputs(src) - src.compute(_inputs(src, TactileDataArray(forces), _shape()), outputs) + src.compute(_inputs(src, TactileDataArray(forces), _metadata()), outputs) np.testing.assert_array_equal( np.asarray(outputs[TactileSource.FORCES][0]), forces ) - group = outputs[TactileSource.SHAPE] + group = outputs[TactileSource.LAYOUT] assert not group.is_none - joints = np.asarray(group[TactileShapeIndex.SEGMENT_JOINTS]) + joints = np.asarray(group[TactileLBSLayoutIndex.SEGMENT_JOINT_IDS]) assert joints.dtype == np.int32 assert joints.tolist() == [0, 1] assert ( - np.asarray(group[TactileShapeIndex.TAXEL_SEGMENT_INDICES]).dtype == np.int32 + np.asarray(group[TactileLBSLayoutIndex.TAXEL_SEGMENT_INDICES]).dtype + == np.int32 ) np.testing.assert_array_equal( - np.asarray(group[TactileShapeIndex.TAXEL_BIND_POSE])[:, 3:], + np.asarray(group[TactileLBSLayoutIndex.TAXEL_BIND_POSES])[:, 3:], np.tile([0.0, 0.0, 0.0, 1.0], (NUM_TAXELS, 1)), ) - def test_shape_outputs_do_not_alias_the_metadata(self): + def test_layout_outputs_do_not_alias_the_metadata(self): src = _source() - shape = _shape() - encoded = shape.to_bytes() + metadata = _metadata() + encoded = metadata.to_bytes() outputs = _outputs(src) - src.compute(_inputs(src, None, shape), outputs) + src.compute(_inputs(src, None, metadata), outputs) - group = outputs[TactileSource.SHAPE] - for index in TactileShapeIndex: + group = outputs[TactileSource.LAYOUT] + for index in TactileLBSLayoutIndex: array = np.asarray(group[index]) array[...] = 7 - assert shape.to_bytes() == encoded + assert metadata.to_bytes() == encoded - def test_shape_without_data(self): + def test_metadata_without_data(self): src = _source() outputs = _outputs(src) - src.compute(_inputs(src, None, _shape()), outputs) + src.compute(_inputs(src, None, _metadata()), outputs) assert outputs[TactileSource.FORCES].is_none - assert not outputs[TactileSource.SHAPE].is_none + assert not outputs[TactileSource.LAYOUT].is_none def test_inactive_sets_none(self): src = _source() outputs = _outputs(src) src.compute(_inputs(src, None, None), outputs) assert outputs[TactileSource.FORCES].is_none - assert outputs[TactileSource.SHAPE].is_none + assert outputs[TactileSource.LAYOUT].is_none def test_taxel_count_mismatch_raises(self): src = _source(num_taxels=4) @@ -160,7 +161,7 @@ def test_taxel_count_mismatch_raises(self): with pytest.raises(ValueError, match="3 taxels"): src.compute(_inputs(src, frame, None), _outputs(src)) - def test_shape_count_mismatch_raises(self): - src = _source(num_joints=NUM_JOINTS + 1) - with pytest.raises(ValueError, match="2 joints"): - src.compute(_inputs(src, None, _shape()), _outputs(src)) + def test_metadata_count_mismatch_raises(self): + src = _source(num_segments=NUM_SEGMENTS + 1) + with pytest.raises(ValueError, match="2 segments"): + src.compute(_inputs(src, None, _metadata()), _outputs(src)) diff --git a/tests/python/core/schema/test_tactile_array.py b/tests/python/core/schema/test_tactile_array.py index a48678bdc7..bc57ea775f 100644 --- a/tests/python/core/schema/test_tactile_array.py +++ b/tests/python/core/schema/test_tactile_array.py @@ -20,24 +20,24 @@ class TestTactileDataArray: def test_default_construction(self): data = TactileDataArray() - assert data.taxel_force.shape == (0, 3) + assert data.taxel_forces.shape == (0, 3) def test_round_trip(self): data = TactileDataArray(FORCES) - assert data.taxel_force.dtype == np.float32 - np.testing.assert_array_equal(data.taxel_force, FORCES) + assert data.taxel_forces.dtype == np.float32 + np.testing.assert_array_equal(data.taxel_forces, FORCES) def test_accepts_nested_lists(self): data = TactileDataArray([[1, 2, 3]]) - np.testing.assert_array_equal(data.taxel_force, [[1.0, 2.0, 3.0]]) + np.testing.assert_array_equal(data.taxel_forces, [[1.0, 2.0, 3.0]]) def test_view_outlives_its_table(self): - forces = TactileDataArray(FORCES).taxel_force + forces = TactileDataArray(FORCES).taxel_forces gc.collect() np.testing.assert_array_equal(forces, FORCES) def test_rejects_wrong_width(self): - with pytest.raises(ValueError, match="taxel_force"): + with pytest.raises(ValueError, match="taxel_forces"): TactileDataArray(np.zeros((2, 4), np.float32)) def test_repr(self): @@ -49,7 +49,7 @@ def test_record_carries_payload_and_timestamp(self): record = TactileDataArrayRecord( TactileDataArray(FORCES), DeviceDataTimestamp(1, 2, 3) ) - np.testing.assert_array_equal(record.data.taxel_force, FORCES) + np.testing.assert_array_equal(record.data.taxel_forces, FORCES) assert record.timestamp.sample_time_raw_device_clock == 3 def test_payload_less_record(self): diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py index 2e9191eb01..a71e552492 100644 --- a/tests/python/core/schema/test_tactile_metadata.py +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -19,8 +19,8 @@ def lbs_arrays(num_joints: int = 2, num_taxels: int = 3) -> dict: - segments = np.zeros((num_joints, 7), np.float32) - segments[:, 6] = 1 + segment_bind_poses = np.zeros((num_joints, 7), np.float32) + segment_bind_poses[:, 6] = 1 taxels = np.zeros((num_taxels, 7), np.float32) taxels[:, :3] = 0.01 taxels[:, 6] = 1 @@ -29,8 +29,8 @@ def lbs_arrays(num_joints: int = 2, num_taxels: int = 3) -> dict: weights = np.zeros((num_taxels, 4), np.float32) weights[:, 0] = 1 return dict( - segment_bind_pose=segments, - taxel_bind_pose=taxels, + segment_bind_poses=segment_bind_poses, + taxel_bind_poses=taxels, taxel_segment_indices=indices, taxel_segment_weights=weights, ) @@ -41,34 +41,34 @@ def test_metadata_round_trip(): metadata = TactileMetadata( LBS(**arrays), HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) ) - assert metadata.algorithm_type == AlgorithmSource.LBS - assert metadata.segments_type == SegmentSource.HandSkeletonRef - assert metadata.segments.joints == [HandJoint.WRIST, HandJoint.INDEX_TIP] + assert metadata.geometry_mapping_type == AlgorithmSource.LBS + assert metadata.segment_source_type == SegmentSource.HandSkeletonRef + assert metadata.segment_source.joint_ids == [HandJoint.WRIST, HandJoint.INDEX_TIP] for name, expected in arrays.items(): - actual = getattr(metadata.algorithm, name) + actual = getattr(metadata.geometry_mapping, name) assert actual.dtype == expected.dtype np.testing.assert_array_equal(actual, expected) - assert metadata.algorithm.taxel_segment_indices[0, 0] == 1 + assert metadata.geometry_mapping.taxel_segment_indices[0, 0] == 1 def test_nested_views_and_arrays_retain_the_buffer(): metadata = TactileMetadata( LBS(**lbs_arrays()), HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) ) - algorithm = metadata.algorithm - joints = metadata.segments - rows = algorithm.taxel_bind_pose - encoded = algorithm.to_bytes() - del metadata, algorithm + geometry_mapping = metadata.geometry_mapping + segment_source = metadata.segment_source + rows = geometry_mapping.taxel_bind_poses + encoded = geometry_mapping.to_bytes() + del metadata, geometry_mapping gc.collect() - assert joints.joints == [HandJoint.WRIST, HandJoint.INDEX_TIP] - np.testing.assert_array_equal(rows, lbs_arrays()["taxel_bind_pose"]) + assert segment_source.joint_ids == [HandJoint.WRIST, HandJoint.INDEX_TIP] + np.testing.assert_array_equal(rows, lbs_arrays()["taxel_bind_poses"]) assert encoded def test_rejects_mismatched_taxel_lengths(): arrays = lbs_arrays() - arrays["taxel_bind_pose"] = arrays["taxel_bind_pose"][:2] + arrays["taxel_bind_poses"] = arrays["taxel_bind_poses"][:2] with pytest.raises(ValueError, match="same length"): LBS(**arrays) @@ -93,7 +93,7 @@ def test_rejects_invalid_weights(weight): LBS(**arrays) -@pytest.mark.parametrize("field", ["segment_bind_pose", "taxel_bind_pose"]) +@pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) def test_rejects_nonunit_quaternions(field): arrays = lbs_arrays() arrays[field][0, 6] = 0 @@ -103,7 +103,7 @@ def test_rejects_nonunit_quaternions(field): def test_rejects_nonfinite_bind_positions(): arrays = lbs_arrays() - arrays["segment_bind_pose"][0, 0] = np.nan + arrays["segment_bind_poses"][0, 0] = np.nan with pytest.raises(ValueError, match="finite"): LBS(**arrays) @@ -116,8 +116,8 @@ def test_rejects_wrong_influence_width(): @pytest.mark.parametrize( - "joints", [[HandJoint.WRIST, HandJoint.WRIST], [HandJoint.NUM_JOINTS]] + "joint_ids", [[HandJoint.WRIST, HandJoint.WRIST], [HandJoint.NUM_JOINTS]] ) -def test_rejects_invalid_joint_mapping(joints): - with pytest.raises(ValueError, match="joints"): - HandSkeletonRef(joints) +def test_rejects_invalid_joint_mapping(joint_ids): + with pytest.raises(ValueError, match="joint_ids"): + HandSkeletonRef(joint_ids) From 306ee0994fe721caaaa37a8af8d448786069bb68 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 21 Oct 2026 15:18:01 -0700 Subject: [PATCH 08/45] Make fields required Signed-off-by: Lotus Li --- src/core/schema/fbs/tactile_metadata.fbs | 18 +++++++++--------- src/core/schema/golden/tactile_metadata.bfbs | Bin 4968 -> 5040 bytes 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs index 7397fe2fee..474622bdeb 100644 --- a/src/core/schema/fbs/tactile_metadata.fbs +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -10,7 +10,7 @@ table HandSkeletonRef { // Ordered, unique HandJoint identifiers; NUM_JOINTS is not a joint. // Segment indices address this vector, not HandJoint ordinals. Bind and current // segment poses map joint-local coordinates into the same hand-root frame. - joint_ids: [HandJoint] (id: 0); + joint_ids: [HandJoint] (id: 0, required); } union SegmentSource { @@ -28,20 +28,20 @@ struct Float4 { table LBS { // Segment-to-source-frame bind poses, in SegmentSource order. Positions are in // meters and orientations are unit quaternions (x, y, z, w). - segment_bind_poses: [Pose] (id: 0); + segment_bind_poses: [Pose] (id: 0, required); // Taxel-local-to-source-frame bind poses, in TactileDataArray.taxel_forces order. // Uses the same units and quaternion convention as segment_bind_poses. - taxel_bind_poses: [Pose] (id: 1); + taxel_bind_poses: [Pose] (id: 1, required); // One four-slot row per taxel; all taxel_* vectors have the same length. // Every slot indexes segment_bind_poses, including slots whose weight is zero. - taxel_segment_indices: [UInt4] (id: 2); + taxel_segment_indices: [UInt4] (id: 2, required); // Finite, nonnegative weights, summing to one per taxel. Bind positions move as // p(t) = sum_k w_k * T_current[j_k] * inverse(T_bind[j_k]) * p_bind. // LBS blends transformed points; it does not define a rigid animated taxel frame. - taxel_segment_weights: [Float4] (id: 3); + taxel_segment_weights: [Float4] (id: 3, required); } // Static parameters for the selected taxel geometry-mapping method. @@ -50,14 +50,14 @@ union AlgorithmSource { } // Static layout published as tracker metadata, without a timestamp or Record wrapper. -// A usable layout sets both unions; lengths, indices and weights require semantic -// validation by producers/consumers, beyond FlatBuffers buffer verification. +// Required fields enforce presence only; empty vectors still pass verification. +// Lengths, indices and weights require semantic validation by producers/consumers. table TactileMetadata { // The implicit geometry_mapping_type discriminator occupies id 0. - geometry_mapping: AlgorithmSource (id: 1); + geometry_mapping: AlgorithmSource (id: 1, required); // The implicit segment_source_type discriminator occupies id 2. - segment_source: SegmentSource (id: 3); + segment_source: SegmentSource (id: 3, required); } root_type TactileMetadata; diff --git a/src/core/schema/golden/tactile_metadata.bfbs b/src/core/schema/golden/tactile_metadata.bfbs index 07ee0d1b7f0ca3e3d4baa09bb85fa574a6591387..355bac21ed7a5d7a5eec3cb68786797d5a9d858c 100644 GIT binary patch delta 784 zcmZuv&ubG=5T3V9*6!w+#x{Y1LV8duv%*`=F;fkJ!m z4^SqDAP81G^q>&vr57OwkK##h3Lbjsp`r)z2a3<{y=_9PKA2_aoB7`N&FpR;Z;##& zPfZOFO{RznX`;?SqKk;8B-#e{&LNr}B)XOXeu!wfpQ!X;Ex$7TQKB^5&G2LRS{(OR0`i?6;O-{_CsYqsQUHAhF6^r!%D48{KL6$CikC8d@s@F zulkUBh7lI`$VFr()x>spRc!JFgZxs2@toqk(v;* z{5&-(&hcBkZ?Yp#iw>Wc6JnB=WPUhnZ0ZkfYbBhdvC1tup4PtZ1QmR@N-3g^*YctL E2Vlytu>b%7 delta 735 zcmZWmJxE(o6h8Nv1oQG@RBUk&lfjxHQd%rjp)T!cKtu6O|(ZL~d z9YX1liVlKbvCpwo-ZCISkQ7!KLhJuaVLv7OaXy=DgJ) zTG#hhqKS0s1k0lsHLjIcWHq~I%=o;Sv@!@OccqjDwhJ-WFW1Huc0FsbRYMcC-lt9E zrMj0Om+jR}*}56d@6GrvVsr7-Lc*Gzmu~%43>Wg7srM>Z|6cGCo;*qs_ykM<9RQC$ zlUgtzhhL5%q{7Yr>t4$}J@lYdKjFpfix`mK_0a3UZB6sX*p&UnT#JvV Date: Wed, 21 Oct 2026 15:27:52 -0700 Subject: [PATCH 09/45] Validate tactile metadata before adoption and recording Signed-off-by: Lotus Li --- .../live_trackers/schema_metadata_tracker.hpp | 10 ++ .../schema/tactile_metadata_validation.hpp | 61 +++++++++++ .../cpp/core/schema/test_tactile_metadata.cpp | 101 ++++++++++++++++++ .../test_tracker_metadata.cpp | 47 ++++++++ 4 files changed, 219 insertions(+) create mode 100644 src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp index dadf1b51f8..48de6ee30d 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp @@ -10,12 +10,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include namespace core @@ -100,6 +102,14 @@ class SchemaMetadataTracker : public SchemaTrackerBase MetadataT::GetFullyQualifiedName()); } + if constexpr (std::is_same_v) + { + if (!verify_tactile_metadata(*flatbuffers::GetRoot(sample.buffer.data()))) + { + throw std::runtime_error(collection_id + ": tactile metadata failed semantic verification"); + } + } + if (!metadata_) { const int64_t log_time_ns = sample.timestamp.available_time_local_common_clock(); diff --git a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp new file mode 100644 index 0000000000..0570938531 --- /dev/null +++ b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include + +namespace core +{ + +//! Checks the semantics of the supported LBS/HandSkeletonRef layout after FlatBuffers verification. +//! Weight sums may differ from one by at most 1e-5 to allow floating-point rounding. +[[nodiscard]] inline bool verify_tactile_metadata(const TactileMetadata& metadata) +{ + const auto* lbs = metadata.geometry_mapping_as_LBS(); + const auto* hand = metadata.segment_source_as_HandSkeletonRef(); + if (lbs == nullptr || hand == nullptr) + { + return false; + } + + const auto taxel_count = lbs->taxel_bind_poses()->size(); + if (lbs->taxel_segment_indices()->size() != taxel_count || lbs->taxel_segment_weights()->size() != taxel_count || + lbs->segment_bind_poses()->size() != hand->joint_ids()->size()) + { + return false; + } + + std::array seen{}; + for (const auto joint_id : *hand->joint_ids()) + { + if (joint_id >= HandJoint_NUM_JOINTS || seen[joint_id]) + { + return false; + } + seen[joint_id] = true; + } + + for (const auto* weights : *lbs->taxel_segment_weights()) + { + double sum = 0.0; + for (const float weight : *weights->values()) + { + if (!std::isfinite(weight) || weight < 0.0f) + { + return false; + } + sum += weight; + } + if (std::abs(sum - 1.0) > 1e-5) + { + return false; + } + } + return true; +} + +} // namespace core diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index b5be4149cd..665ae7bcea 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include +#include #include #include @@ -76,3 +78,102 @@ TEST_CASE("TactileMetadata verifies and retains nested union views", "[unit][tac REQUIRE(lbs); CHECK(lbs->taxel_segment_weights()->Get(0)->values()->Get(0) == 0.75f); } + +TEST_CASE("TactileMetadata semantic verification accepts valid metadata", "[unit][tactile_metadata]") +{ + const auto metadata = core::pack(make_metadata()); + CHECK(core::verify_tactile_metadata(*metadata)); +} + +TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", "[unit][tactile_metadata]") +{ + auto native = make_metadata(); + auto& lbs = *native.geometry_mapping.AsLBS(); + SECTION("taxel bind poses") + { + lbs.taxel_bind_poses.push_back(lbs.taxel_bind_poses.front()); + } + SECTION("taxel segment indices") + { + lbs.taxel_segment_indices.clear(); + } + SECTION("taxel segment weights") + { + lbs.taxel_segment_weights.clear(); + } + SECTION("joint identifiers and segment bind poses") + { + native.segment_source.AsHandSkeletonRef()->joint_ids.pop_back(); + } + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK_FALSE(core::verify_tactile_metadata(*metadata)); +} + +TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joints", "[unit][tactile_metadata]") +{ + auto native = make_metadata(); + auto& joint_ids = native.segment_source.AsHandSkeletonRef()->joint_ids; + SECTION("sentinel") + { + joint_ids[1] = core::HandJoint_NUM_JOINTS; + } + SECTION("unknown enum value") + { + joint_ids[1] = static_cast(255); + } + SECTION("duplicate identifier") + { + joint_ids[1] = joint_ids[0]; + } + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK_FALSE(core::verify_tactile_metadata(*metadata)); +} + +TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", "[unit][tactile_metadata]") +{ + auto native = make_metadata(); + auto& lbs = *native.geometry_mapping.AsLBS(); + std::array weights = { 0.75f, 0.25f, 0.0f, 0.0f }; + bool valid = false; + SECTION("negative weight with sum one") + { + weights = { 1.25f, -0.25f, 0.0f, 0.0f }; + } + SECTION("sum below one") + { + weights = { 0.25f, 0.25f, 0.0f, 0.0f }; + } + SECTION("sum above one") + { + weights = { 0.75f, 0.75f, 0.0f, 0.0f }; + } + SECTION("NaN") + { + weights[3] = std::numeric_limits::quiet_NaN(); + } + SECTION("infinity") + { + weights[3] = std::numeric_limits::infinity(); + } + SECTION("rounding within tolerance") + { + weights = { 0.75f, 0.250005f, 0.0f, 0.0f }; + valid = true; + } + SECTION("rounding outside tolerance") + { + weights = { 0.75f, 0.25002f, 0.0f, 0.0f }; + } + // Place the changed weights after a valid taxel to exercise the whole vector. + lbs.taxel_bind_poses.push_back(lbs.taxel_bind_poses.front()); + lbs.taxel_segment_indices.push_back(lbs.taxel_segment_indices.front()); + lbs.taxel_segment_weights.emplace_back(flatbuffers::span(weights.data(), 4)); + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK(core::verify_tactile_metadata(*metadata) == valid); +} diff --git a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp index 44bdd45385..37bebfea0b 100644 --- a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp +++ b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp @@ -19,16 +19,19 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include #include #include +#include #include namespace @@ -267,6 +270,50 @@ TEST_CASE("Replay withholds data recorded without metadata", "[unit][tracker_met #if defined(XR_USE_TIMESPEC) +TEST_CASE("Live tactile metadata is semantically verified before adoption and recording", "[unit][tracker_metadata]") +{ + const auto path = temp_mcap_path("test_tactile_metadata"); + const TempFileCleanup cleanup(path); + { + FakeTensorRuntime runtime; + auto writer = open_writer(path); + core::McapTrackerChannels channels(*writer, BASE_NAME, { "metadata" }); + core::SchemaMetadataTracker tracker(runtime.handles(), DATA_COLLECTION, &channels); + runtime.create(METADATA_COLLECTION, core::TRACKER_METADATA_MAX_SIZE); + + core::HandSkeletonRefT hand; + hand.joint_ids = { core::HandJoint_WRIST }; + core::LBST lbs; + lbs.segment_bind_poses.emplace_back(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)); + lbs.taxel_bind_poses = lbs.segment_bind_poses; + const std::array indices{}; + const std::array zero_weights{}; + lbs.taxel_segment_indices.emplace_back(flatbuffers::span(indices.data(), 4)); + lbs.taxel_segment_weights.emplace_back(flatbuffers::span(zero_weights.data(), 4)); + core::TactileMetadataT native; + native.geometry_mapping.Set(std::move(lbs)); + native.segment_source.Set(std::move(hand)); + const auto invalid = core::pack(native); + flatbuffers::Verifier verifier(invalid.buffer().data(), invalid.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + + runtime.push(METADATA_COLLECTION, invalid.buffer(), 1); + CHECK_THROWS_WITH(tracker.update(), Catch::Matchers::ContainsSubstring("semantic verification")); + CHECK_FALSE(tracker.get()); + + const std::array weights = { 1.0f, 0.0f, 0.0f, 0.0f }; + native.geometry_mapping.AsLBS()->taxel_segment_weights[0] = + core::Float4(flatbuffers::span(weights.data(), 4)); + const auto valid = core::pack(native); + runtime.push(METADATA_COLLECTION, valid.buffer(), 2); + CHECK(tracker.update()); + REQUIRE(tracker.get()); + CHECK(tracker.get()->geometry_mapping_as_LBS()->taxel_segment_weights()->Get(0)->values()->Get(0) == 1.0f); + writer->close(); + } + CHECK(read_channel(path, "metadata").size() == 1); +} + TEST_CASE("Live publishes and records data only once the producer's metadata arrives", "[unit][tracker_metadata]") { const auto path = temp_mcap_path("test_tracker_metadata"); From e95d1d9705ed384ab661a4d1a5b77b30518fd75c Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 00:28:05 -0700 Subject: [PATCH 10/45] Fix tactile layout NumPy type inference Signed-off-by: Lotus Li --- .../deviceio_source_nodes/tactile_source.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 5e74d068e8..f2a6d8fb5c 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -181,7 +181,9 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N geometry_mapping.taxel_segment_weights, ) for index, value in enumerate(values): + # Schema stubs have mixed array types; normalize before NumPy operations. + array = np.asarray(value) # Copied: the views alias the metadata the tracker holds and compares later pushes to. - layout_out[index] = value.astype( - np.int32 if value.dtype == np.uint32 else value.dtype + layout_out[index] = array.astype( + np.int32 if array.dtype == np.uint32 else array.dtype ) From 52efcbebfba5f5fd507486aa6fb2a1c36defe6ab Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 00:28:44 -0700 Subject: [PATCH 11/45] Reject out-of-range tactile segment indices Signed-off-by: Lotus Li --- .../schema/tactile_metadata_validation.hpp | 11 ++++++++++ .../cpp/core/schema/test_tactile_metadata.cpp | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp index 0570938531..6f9c8375e4 100644 --- a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp +++ b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp @@ -39,6 +39,17 @@ namespace core seen[joint_id] = true; } + for (const auto* indices : *lbs->taxel_segment_indices()) + { + for (const auto segment_index : *indices->values()) + { + if (segment_index >= lbs->segment_bind_poses()->size()) + { + return false; + } + } + } + for (const auto* weights : *lbs->taxel_segment_weights()) { double sum = 0.0; diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index 665ae7bcea..3ae2a1796a 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -133,6 +133,26 @@ TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joi CHECK_FALSE(core::verify_tactile_metadata(*metadata)); } +TEST_CASE("TactileMetadata semantic verification rejects out-of-range segment indices", "[unit][tactile_metadata]") +{ + auto native = make_metadata(); + auto& lbs = *native.geometry_mapping.AsLBS(); + std::array indices = { 1, 0, 0, 0 }; + SECTION("nonzero-weight slot") + { + indices[0] = 2; + } + SECTION("zero-weight slot") + { + indices[3] = 2; + } + lbs.taxel_segment_indices[0] = core::UInt4(flatbuffers::span(indices.data(), 4)); + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK_FALSE(core::verify_tactile_metadata(*metadata)); +} + TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", "[unit][tactile_metadata]") { auto native = make_metadata(); From 695a7e95f86fcaf94d5ded74fc9c4aaa75a52c1c Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 00:30:31 -0700 Subject: [PATCH 12/45] Preserve HandJoint types in tactile metadata bindings Signed-off-by: Lotus Li --- src/core/schema/python/tactile_metadata_bindings.h | 13 ++++++++++++- tests/python/core/schema/test_tactile_metadata.py | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/core/schema/python/tactile_metadata_bindings.h b/src/core/schema/python/tactile_metadata_bindings.h index be737afdc5..be21c3b752 100644 --- a/src/core/schema/python/tactile_metadata_bindings.h +++ b/src/core/schema/python/tactile_metadata_bindings.h @@ -56,7 +56,18 @@ inline void bind_tactile_metadata(py::module& m) return pack(native); }), py::arg("joint_ids")) - .def_property_readonly("joint_ids", vector_field(&HandSkeletonRef::joint_ids)); + .def_property_readonly("joint_ids", + [](const Serialized& self) + { + // FlatBuffers exposes enum-vector storage as uint8_t. + std::vector joint_ids; + joint_ids.reserve(self->joint_ids()->size()); + for (const auto joint : *self->joint_ids()) + { + joint_ids.push_back(static_cast(joint)); + } + return joint_ids; + }); auto poses_from_rows = [](const FloatRows& rows, const char* name) { diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py index a71e552492..991ca31f3f 100644 --- a/tests/python/core/schema/test_tactile_metadata.py +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -51,6 +51,15 @@ def test_metadata_round_trip(): assert metadata.geometry_mapping.taxel_segment_indices[0, 0] == 1 +@pytest.mark.parametrize("joint_ids", [[], [HandJoint.WRIST, HandJoint.INDEX_TIP]]) +def test_joint_ids_can_reconstruct_hand_reference(joint_ids): + hand = HandSkeletonRef(joint_ids) + rebuilt = HandSkeletonRef(hand.joint_ids) + assert all(isinstance(joint, HandJoint) for joint in hand.joint_ids) + assert rebuilt.joint_ids == joint_ids + assert rebuilt.to_bytes() == hand.to_bytes() + + def test_nested_views_and_arrays_retain_the_buffer(): metadata = TactileMetadata( LBS(**lbs_arrays()), HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) From 898b765110fe869047a4f7b5153b33864a4f7125 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 00:32:38 -0700 Subject: [PATCH 13/45] Clarify tactile example OpenXR system requirement Signed-off-by: Lotus Li --- examples/tactile/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/tactile/README.md b/examples/tactile/README.md index a974d067b3..6e86241b42 100644 --- a/examples/tactile/README.md +++ b/examples/tactile/README.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # Tactile -A mock tactile device and a consumer for exercising the tactile tracker without hardware. +A mock tactile device and a consumer for exercising the tactile tracker without tactile hardware. - `feeder` publishes a 26-joint hand with 200 taxels by default. Each frame carries one `TactileDataArray`; it publishes `TactileMetadata` when it starts and every second after that. @@ -26,7 +26,8 @@ Install the example: uv pip install -e ./examples/tactile ``` -Start the printer first; it starts the CloudXR runtime, and no headset needs to connect: +Start the printer first; it starts the CloudXR runtime. Live initialization requires an +OpenXR HMD system; if none is available, it waits for a headset client to connect: ```bash python -m isaaccapture_examples.tactile.printer --accept-eula --duration 0 From 5181e655591db9b3a38b8b1be4cc419e20c514cb Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 00:54:59 -0700 Subject: [PATCH 14/45] Remove unrelated maintenance from tactile schema PR Signed-off-by: Lotus Li --- scripts/check_licenses.py | 3 --- src/core/codegen/generate_trackers.py | 3 --- src/core/codegen/templates.py | 2 -- 3 files changed, 8 deletions(-) diff --git a/scripts/check_licenses.py b/scripts/check_licenses.py index fbbc0ac5c9..ddf86560ab 100644 --- a/scripts/check_licenses.py +++ b/scripts/check_licenses.py @@ -123,8 +123,6 @@ def fix_license(project: Project, path: Path) -> bool: if expected == EGOPOST_LICENSE and expressions == {"Apache-2.0"}: content = path.read_text() # Replace only the default header tag, retaining copyright and other notices. - # SPDX patterns are data, not license declarations for this file. - # REUSE-IgnoreStart updated, count = re.subn( r"(SPDX-License-Identifier:[ \t]*)Apache-2\.0(?=[ \t]*(?:\*/|-->)?[ \t]*$)", rf"\g<1>{EGOPOST_LICENSE}", @@ -132,7 +130,6 @@ def fix_license(project: Project, path: Path) -> bool: count=1, flags=re.MULTILINE, ) - # REUSE-IgnoreEnd if count: path.write_text(updated) print(f"{path}: updated default license to {EGOPOST_LICENSE}") diff --git a/src/core/codegen/generate_trackers.py b/src/core/codegen/generate_trackers.py index acd62c73f2..2c1b46e5f2 100644 --- a/src/core/codegen/generate_trackers.py +++ b/src/core/codegen/generate_trackers.py @@ -15,15 +15,12 @@ from manifest import load_manifest from templates import TrackerGenContext, enrich_context -# Generated SPDX text is string data, not this file's license declaration. -# REUSE-IgnoreStart COPYRIGHT = ( "// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" "// " "SPDX-License-Identifier: Apache-2.0\n\n" "// AUTO-GENERATED by src/core/codegen/generate_trackers.py — do not edit.\n\n" ) -# REUSE-IgnoreEnd def _write_if_changed(path: Path, content: str) -> bool: diff --git a/src/core/codegen/templates.py b/src/core/codegen/templates.py index 53cad9dcd2..026ced75a9 100644 --- a/src/core/codegen/templates.py +++ b/src/core/codegen/templates.py @@ -11,7 +11,6 @@ def _header() -> str: - # REUSE-IgnoreStart return ( "// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" "// SPDX-License-Identifier: Apache-2.0\n" @@ -19,7 +18,6 @@ def _header() -> str: "// AUTO-GENERATED by src/core/codegen/generate_trackers.py — do not edit.\n" "\n" ) - # REUSE-IgnoreEnd def camel_to_snake(value: str) -> str: From 3570b331bb164ec8aa249561a2a4d15ea3460ee5 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 01:12:12 -0700 Subject: [PATCH 15/45] Dispatch metadata semantic validation through the schema layer Signed-off-by: Lotus Li --- .../live_trackers/schema_metadata_tracker.hpp | 12 ++++------ .../cpp/inc/schema/metadata_validation.hpp | 24 +++++++++++++++++++ .../cpp/core/schema/test_tactile_metadata.cpp | 13 +++++----- 3 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 src/core/schema/cpp/inc/schema/metadata_validation.hpp diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp index 48de6ee30d..db6b37845e 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp @@ -9,15 +9,14 @@ #include #include #include +#include #include -#include #include #include #include #include #include -#include #include namespace core @@ -102,12 +101,11 @@ class SchemaMetadataTracker : public SchemaTrackerBase MetadataT::GetFullyQualifiedName()); } - if constexpr (std::is_same_v) + // Schema-specific dispatch belongs to the schema layer, not this generic reader. + if (!verify_metadata(*flatbuffers::GetRoot(sample.buffer.data()))) { - if (!verify_tactile_metadata(*flatbuffers::GetRoot(sample.buffer.data()))) - { - throw std::runtime_error(collection_id + ": tactile metadata failed semantic verification"); - } + throw std::runtime_error(collection_id + ": metadata failed semantic verification for " + + MetadataT::GetFullyQualifiedName()); } if (!metadata_) diff --git a/src/core/schema/cpp/inc/schema/metadata_validation.hpp b/src/core/schema/cpp/inc/schema/metadata_validation.hpp new file mode 100644 index 0000000000..df047f941d --- /dev/null +++ b/src/core/schema/cpp/inc/schema/metadata_validation.hpp @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "tactile_metadata_validation.hpp" + +namespace core +{ + +//! Semantic verification after FlatBuffers verification; schemas without extra rules pass. +template +[[nodiscard]] inline bool verify_metadata(const MetadataT&) +{ + return true; +} + +template <> +[[nodiscard]] inline bool verify_metadata(const TactileMetadata& metadata) +{ + return verify_tactile_metadata(metadata); +} + +} // namespace core diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index 3ae2a1796a..1b914a563c 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -3,9 +3,9 @@ #include #include +#include #include #include -#include #include #include @@ -82,7 +82,7 @@ TEST_CASE("TactileMetadata verifies and retains nested union views", "[unit][tac TEST_CASE("TactileMetadata semantic verification accepts valid metadata", "[unit][tactile_metadata]") { const auto metadata = core::pack(make_metadata()); - CHECK(core::verify_tactile_metadata(*metadata)); + CHECK(core::verify_metadata(*metadata)); } TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", "[unit][tactile_metadata]") @@ -108,7 +108,7 @@ TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK_FALSE(core::verify_tactile_metadata(*metadata)); + CHECK_FALSE(core::verify_metadata(*metadata)); } TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joints", "[unit][tactile_metadata]") @@ -130,7 +130,7 @@ TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joi const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK_FALSE(core::verify_tactile_metadata(*metadata)); + CHECK_FALSE(core::verify_metadata(*metadata)); } TEST_CASE("TactileMetadata semantic verification rejects out-of-range segment indices", "[unit][tactile_metadata]") @@ -150,7 +150,7 @@ TEST_CASE("TactileMetadata semantic verification rejects out-of-range segment in const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK_FALSE(core::verify_tactile_metadata(*metadata)); + CHECK_FALSE(core::verify_metadata(*metadata)); } TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", "[unit][tactile_metadata]") @@ -195,5 +195,6 @@ TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", " const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK(core::verify_tactile_metadata(*metadata) == valid); + CHECK(core::verify_metadata(*metadata) == valid); + CHECK(core::verify_metadata(*metadata) == valid); } From 899238b61646d5d6cfd5c4c1af5a296eba2c4cde Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 01:29:43 -0700 Subject: [PATCH 16/45] Remove one-off review notes from agent guidance Signed-off-by: Lotus Li --- src/core/AGENTS.md | 1 - src/core/schema/AGENTS.md | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index a139c46a8d..b758b2bcfc 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -18,7 +18,6 @@ If work under **`src/core/`** went wrong—**user** correction, **pre-commit/CI* - When changing `TeleopSession` retargeting execution defaults, update config, docs, and default-behavior tests together so opt-in vs. default semantics stay aligned. - Preserve existing `TeleopSession` lifecycle flag semantics unless changing the public/context-manager contract intentionally; use tests to lock down cleanup details before altering them. - After Python test or session-manager edits, let `ruff format`/pre-commit own wrapping and rerun the hook when it modifies files. -- Schema NumPy properties are stubbed as `numpy.ndarray[numpy.float32]`, so mypy (`retargeting_mypy_type_check`) rejects `.shape[i]` on them; use `len()` or `np.asarray()` first. - **`IDeviceIOSource` leaves are only discovered when reachable from a declared graph root** — the `pipeline` combiner's outputs, the `teleop_control_pipeline`, or a subgraph registered in `TeleopSessionConfig(sinks=[...])`. `TeleopSession._discover_sources` unions the leaves of all three. An input *source* whose only purpose is a side effect (e.g. message-channel send) must therefore expose at least one output (a heartbeat boolean is the established pattern) **and** be reachable from one of those roots; silent no-discovery is the recurring footgun. **Output `IDeviceIOSink` nodes are different:** they are registered via `TeleopSessionConfig(sinks=[...])` and the session runs and flushes them explicitly each frame, so a sink needs no heartbeat output and no `OutputCombiner` reachability. - **A plugin a wheel install must be able to start is bundled into `isaaccapture/plugins//`** (binary + `plugin.yaml`, via a POST_BUILD copy and `add_dependencies(python_package ...)`), and its binary must not need shared libraries the wheel does not ship. Apps start it explicitly with `PluginConfig(search_paths=[BUNDLED_PLUGINS_DIR])`; sources never start plugins. See `src/plugins/spacemouse`. - **All C++ diagnostics go through `isaaccapture::Logger`** (`log_bridge/cpp/inc/log_bridge/logger.hpp`), never `std::cout`/`std::cerr`/`printf`, and the logger name is always `isaaccapture..`. The env-var contract and wire format are shared with the Python half, so a change to either must land in both — see [`log_bridge/AGENTS.md`](log_bridge/AGENTS.md) before touching the logging machinery, and the repo root [`AGENTS.md`](../../AGENTS.md) "Logging" section for the rules that apply to every call site. diff --git a/src/core/schema/AGENTS.md b/src/core/schema/AGENTS.md index e44c412fcb..2977b70417 100644 --- a/src/core/schema/AGENTS.md +++ b/src/core/schema/AGENTS.md @@ -29,6 +29,3 @@ fails. The short version: existing offset survives; `check_schema_compat()` catches it on the struct's size. - **Never refresh `golden/` to clear a failing test.** The failure is the signal. Refresh only once breaking existing recordings is a decision you have made and stated. -- For tactile metadata union changes, validate C++ and Python against the same freshly - generated schemas. Run validators and hooks through the documented uv-managed Python - environment; the macOS system Python may be too old for the hook dependencies. From 7ffcd3a39d2c2374166df56cede5e2d48a5441d2 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 01:49:48 -0700 Subject: [PATCH 17/45] Document tactile coordinate conventions and data flow Specify right-handed taxel axes, OpenXR hand joint frames, and wrist-relative bind poses. Explain handedness, LBS mapping, metadata transport, and current interpretation limits without changing the wire format. Signed-off-by: Lotus Li --- docs/source/device/tactile/index.rst | 215 +++++++++++++++++++++++ src/core/schema/fbs/hand.fbs | 10 +- src/core/schema/fbs/tactile_array.fbs | 2 + src/core/schema/fbs/tactile_metadata.fbs | 19 +- 4 files changed, 239 insertions(+), 7 deletions(-) diff --git a/docs/source/device/tactile/index.rst b/docs/source/device/tactile/index.rst index 57b8522573..578caf29cf 100644 --- a/docs/source/device/tactile/index.rst +++ b/docs/source/device/tactile/index.rst @@ -7,6 +7,221 @@ Tactile devices Tactile devices capture contact measurements. For actuator output, see :doc:`/references/haptic_feedback`. +The common tactile interface separates each frame's force measurements from the +device's static layout. A *taxel* is one tactile sensing element: its force has +three components, and its layout describes where it is attached to a skeleton. +The current geometry mapping is linear blend skinning (LBS) over hand joints. + +.. contents:: On this page + :local: + :depth: 2 + +Data contract +------------- + +The contract is defined by +:code-file:`src/core/schema/fbs/tactile_array.fbs` and +:code-file:`src/core/schema/fbs/tactile_metadata.fbs`: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Type + - Meaning + * - ``TactileDataArray`` + - One force vector per taxel, in newtons, ordered exactly as the metadata. + The Python view is an ``(N, 3)`` float32 array. + * - ``TactileMetadata`` + - The static ``geometry_mapping`` and ``segment_source`` for that stream. + Currently these select ``LBS`` and ``HandSkeletonRef``. + * - ``HandSkeletonRef`` + - An ordered list of unique ``HandJoint`` identifiers from + :code-file:`src/core/schema/fbs/hand.fbs`. ``NUM_JOINTS`` is a count, + not a usable joint. + * - ``LBS`` + - ``J`` segment bind poses, ``N`` taxel bind poses, and four segment indices + and weights per taxel. An index addresses the ordered segment list, + rather than a ``HandJoint`` ordinal. + +Python pose arrays have shape ``(count, 7)``: position ``(x, y, z)`` in meters, +then a unit quaternion ``(x, y, z, w)``. Identity orientation is ``(0, 0, 0, 1)``. +All frames are right-handed. A pose maps coordinates from its local frame into +its reference frame: ``p_reference = R(q) * p_local + position``. + +Coordinate conventions +---------------------- + +Hand joints and the skeleton root +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``HandSkeletonRef`` uses ``HandJoint.WRIST`` as the skeleton root. Its origin +is the wrist pivot. The joint names and local orientations follow +`OpenXR hand-joint conventions +`_: + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Root axis + - Anatomical direction + * - ``+Y`` + - Out through the back of the hand at the wrist. + * - ``+Z`` + - Along the line through the wrist and middle-finger metacarpal, pointing + toward the forearm, away from the fingertips. + * - ``+X`` + - ``+Y cross +Z``, completing the right-handed basis. + +Finger-joint local ``+Z`` follows its bone away from the fingertip; local ``+Y`` +points toward the nail/back side in the fully open hand. The palm uses the +middle metacarpal for its longitudinal axis. These local frames move with the +joints when the hand bends; their axes are not fixed world directions. + +``segment_bind_poses[j]`` maps joint-local coordinates into the wrist frame in +the calibrated bind configuration. All segments share that root; the poses are +not relative to each joint's parent. If WRIST appears in the segment list, its +bind pose is identity. ``taxel_bind_poses`` use the same root and bind +configuration. The bind configuration must match the device's calibration; +the schema does not prescribe joint angles or a particular hand size. + +**HandPose is not automatically wrist-relative.** The live ``HandTracker`` +copies OpenXR joint poses in the session base space, currently ``STAGE``. +To obtain current joint poses in the tactile skeleton's wrist frame, use the +wrist and joint poses from the same valid hand sample: + +.. code-block:: text + + T_root_joint(t) = inverse(T_base_wrist(t)) * T_base_joint(t) + +Here ``T_A_B`` maps a point from frame B to frame A. This conversion uses both +the wrist translation and rotation. Subtracting the wrist position alone is +insufficient. A downstream consumer must perform this conversion when using +root-relative joint poses; ``TactileSource`` does not rebase a hand stream. + +Taxel frames +~~~~~~~~~~~~ + +A taxel pose's origin is its sensing location. Its orientation defines: + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Taxel axis + - Direction + * - ``+Z = N`` + - Unit normal pointing out of the sensing surface, away from the device. + * - ``+X = T`` + - A calibrated unit tangent perpendicular to ``N``. + * - ``+Y`` + - ``N cross T``. Thus ``+X cross +Y = +Z``. + +The producer chooses and calibrates ``T`` for each taxel, then stores the full +orientation in ``taxel_bind_poses``. A normal alone cannot determine the tangent +direction. In particular, taxel ``+Z`` is a surface normal, whereas hand-joint +``+Z`` follows a bone; the two local frames generally differ. + +Forces ``(fx, fy, fz)`` are components in these taxel-local axes, rather than in +the wrist or world frame. Positive ``fz`` points outward; this does not imply +that a vendor's positive compression reading can be copied directly. The schema +does not identify which body the reported force acts on. A device integration +must document that choice and calibrate its units, axes, and signs accordingly. +Raw pressure or ADC counts require conversion to force before publication. + +Left and right hands +~~~~~~~~~~~~~~~~~~~~ + +Both hands use the same joint identifiers, right-handed algebra, units, +quaternion ordering, and taxel normal/tangent rules. Hand side does not reverse +the cross product or automatically negate a force component. Anatomical geometry +and the calibrated poses can differ between hands. For example, with an open +hand, wrist ``+X`` points toward the thumb side on the left hand and the little +finger side on the right hand. + +``HandSkeletonRef`` carries neither a side nor the identity of a hand tracker. +The application must associate each tactile stream with the correct hand stream +and calibration, using configuration and distinct collection IDs such as +``tactile/left`` and ``tactile/right``. The runtime does not infer side from +these names. Mirroring positions alone is insufficient: a reflection cannot be +encoded as a rotation quaternion, so each hand needs a proper right-handed +orientation for every joint and taxel. + +Mapping taxels onto the moving hand +----------------------------------- + +For a taxel's homogeneous bind position ``p_bind`` in the skeleton root, LBS +computes its current root-relative position as: + +.. code-block:: text + + p_root(t) = sum_k w[k] * C[j[k]](t) * inverse(B[j[k]]) * p_bind + +``B[j]`` is ``segment_bind_poses[j]`` and ``C[j](t)`` is the current pose of +``joint_ids[j]`` in the current wrist frame. Each taxel has four slots; weights +must be finite, nonnegative, and sum to one (validation tolerance ``1e-5``). +Every slot's index must be valid, including slots with zero weight. A taxel +rigidly attached to one segment can use weights ``(1, 0, 0, 0)`` and repeat that +segment's index in all four slots. + +To express the resulting position in the hand tracker's base space: + +.. code-block:: text + + p_base(t) = T_base_wrist(t) * p_root(t) + +LBS specifies blended positions; it does not define a unique rigid taxel frame +after blending multiple segment rotations. A consumer that needs animated +normals or world-space forces must choose an orientation/normal mapping rule. +For a single rigid attachment, the taxel transform is unambiguous: +``C[j](t) * inverse(B[j]) * T_root_taxel_bind``. Transform force directions using +rotation only, without adding the pose translation. + +Publishing and consuming +------------------------ + +A C++ plugin publishes serialized samples with ``SchemaPusher``; a Python +producer can use ``TensorPushTracker``. The paired streams are: + +.. code-block:: text + + tensor "tactile" -> TactileDataArray + /metadata tensor "metadata" -> TactileMetadata + +Publish metadata on startup and periodically so consumers that start later can +discover it. Publish the same encoded metadata throughout the tracker session. +``TactileTracker`` validates and adopts the first metadata, records it once, +and exposes it through ``get_metadata()``. It withholds force data until metadata +is confirmed. Different or invalid metadata causes an error; after a producer +restart, its metadata must be confirmed again. See :ref:`tracker-metadata`. + +The default force-buffer size is 16,384 bytes; larger frames require a matching +``max_flatbuffer_size`` on the producer and consumer. Metadata has its own +``TRACKER_METADATA_MAX_SIZE`` limit of 68,864 bytes, including FlatBuffer overhead. +The data array length must match the metadata's taxel count. + +Applications can read ``TactileTracker`` directly for live or replay data. +Retargeting graphs can instead use ``TactileSource(name, collection_id, +num_segments, num_taxels)`` from +``isaaccapture.retargeting_engine.deviceio_source_nodes``. It emits +``taxel_forces`` and ``tactile_layout`` arrays; constructor counts fix their +shapes. It adapts the frame and static layout to graph tensors; the application +supplies hand motion and evaluates the geometry mapping separately. + +The runnable feeder, direct tracker consumer, and MCAP replay commands are in +:code-file:`examples/tactile/README.md`. The feeder uses synthetic geometry to +exercise transport; it is not an anatomical calibration model. Live operation +requires an OpenXR runtime and an available HMD system, including when using a +headless session; replay does not require the runtime. + +Vendor integrations +------------------- + +The guides below describe individual devices. The existing OGLO integration +uses its own ``OgloGloveSample`` schema with raw measurements; it does not +automatically publish the common force-and-layout contract described above. + .. toctree:: :maxdepth: 1 :titlesonly: diff --git a/src/core/schema/fbs/hand.fbs b/src/core/schema/fbs/hand.fbs index 18908155eb..93540bafce 100644 --- a/src/core/schema/fbs/hand.fbs +++ b/src/core/schema/fbs/hand.fbs @@ -44,9 +44,15 @@ enum HandJoint : uint8 { NUM_JOINTS = 26, } -// Describe a HandJoint pose. +// OpenXR joint-local axes: +Z follows the bone away from the fingertip; +Y +// points toward the nail/back of the open hand; +X = +Y cross +Z. +// At WRIST, +Z follows the wrist-to-middle-metacarpal line toward the forearm; +// at PALM, it follows the middle metacarpal toward the wrist. +// Both hands use right-handed axes, meters, and unit quaternions (x, y, z, w). +// https://registry.khronos.org/OpenXR/specs/1.1/html/xrspec.html#convention-of-hand-joints struct HandJointPose { - // The concrete pose data + // Joint-local to the shared tracking reference frame (the session base space + // for live HandTracker), not parent-local or automatically wrist-relative. pose: Pose; // Whether the hand pose data is valid diff --git a/src/core/schema/fbs/tactile_array.fbs b/src/core/schema/fbs/tactile_array.fbs index 4343ba8eb0..358548cabb 100644 --- a/src/core/schema/fbs/tactile_array.fbs +++ b/src/core/schema/fbs/tactile_array.fbs @@ -11,6 +11,8 @@ namespace core; table TactileDataArray { // Force vectors in newtons, expressed in taxel-local axes. For LBS, taxel_bind_poses // define these axes at bind; the vector length equals the metadata taxel count. + // +X is the calibrated tangent T, +Z is the outward normal N, +Y = N cross T. + // Component signs follow these axes for both hands, without a side-based flip. taxel_forces: [Point] (id: 0); } diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs index 474622bdeb..54807532a8 100644 --- a/src/core/schema/fbs/tactile_metadata.fbs +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -6,11 +6,15 @@ include "pose.fbs"; namespace core; +// The skeleton root is WRIST, with the OpenXR axes from hand.fbs: +Y toward +// the back of the hand, +Z toward the forearm, +X = +Y cross +Z. +// Both hands use these right-handed rules. Hand side and source hand stream +// are associated by the application; this table does not imply mirroring. table HandSkeletonRef { // Ordered, unique HandJoint identifiers; NUM_JOINTS is not a joint. // Segment indices address this vector, not HandJoint ordinals. Bind and current // segment poses map joint-local coordinates into the same hand-root frame. - joint_ids: [HandJoint] (id: 0, required); + joint_ids: [HandJoint] (id: 0, required); // src/core/schema/fbs/hand.fbs } union SegmentSource { @@ -26,12 +30,17 @@ struct Float4 { } table LBS { - // Segment-to-source-frame bind poses, in SegmentSource order. Positions are in - // meters and orientations are unit quaternions (x, y, z, w). + // Segment-local-to-skeleton-root bind poses, in SegmentSource order. + // For HandSkeletonRef the root is the WRIST frame at bind, not a joint's + // parent or the session/world frame. Joint-local axes follow hand.fbs. + // Positions are in meters; orientations are unit quaternions (x, y, z, w). segment_bind_poses: [Pose] (id: 0, required); - // Taxel-local-to-source-frame bind poses, in TactileDataArray.taxel_forces order. - // Uses the same units and quaternion convention as segment_bind_poses. + // Taxel-local-to-skeleton-root bind poses, in TactileDataArray.taxel_forces order. + // The origin is the sensing location. +Z is the outward unit surface normal N; + // +X is a calibrated unit tangent T orthogonal to N; +Y = N cross T. + // These axes are right-handed for either hand; T is stored by the orientation, + // not inferred from hand side. Units and root match segment_bind_poses. taxel_bind_poses: [Pose] (id: 1, required); // One four-slot row per taxel; all taxel_* vectors have the same length. From 3791111a1b4b926443a7abc358fece639ca86e7c Mon Sep 17 00:00:00 2001 From: lotusl-code Date: Thu, 22 Oct 2026 16:37:47 -0700 Subject: [PATCH 18/45] Apply suggestion from @aristarkhovNV Co-authored-by: Andrei Aristarkhov Signed-off-by: Lotus Li --- src/core/schema/fbs/tactile_metadata.fbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs index 54807532a8..380ffe6c7c 100644 --- a/src/core/schema/fbs/tactile_metadata.fbs +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -14,7 +14,7 @@ table HandSkeletonRef { // Ordered, unique HandJoint identifiers; NUM_JOINTS is not a joint. // Segment indices address this vector, not HandJoint ordinals. Bind and current // segment poses map joint-local coordinates into the same hand-root frame. - joint_ids: [HandJoint] (id: 0, required); // src/core/schema/fbs/hand.fbs + joint_ids: [HandJoint] (id: 0, required); // defined in hand.fbs } union SegmentSource { From c7dca688240f8578980a15dee01ce8b3c427c6bd Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 22:28:09 -0700 Subject: [PATCH 19/45] Preserve metadata validation across tensor collection recreation Track runtime collection identities so recreating a collection at the same list index resets its sample cursor and metadata confirmation. Add a focused regression covering matching metadata, missing metadata, and changed metadata after recreation. Signed-off-by: Lotus Li --- .../live_trackers/schema_metadata_tracker.hpp | 6 ++++ .../inc/live_trackers/schema_tracker_base.hpp | 8 +++++ .../live_trackers/cpp/schema_tracker_base.cpp | 4 ++- .../tracker_metadata/fake_tensor_runtime.hpp | 6 +++- .../test_tracker_metadata.cpp | 35 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp index db6b37845e..1595b8e9ff 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp @@ -64,8 +64,14 @@ class SchemaMetadataTracker : public SchemaTrackerBase */ bool update() { + const auto previous_collection = collection_instance_id(); std::vector samples; const bool present = read_all_samples(samples); + if (collection_instance_id() != previous_collection) + { + // Recreated collections must confirm their own sample, even without an observed gap. + confirmed_ = false; + } for (auto& sample : samples) { accept(sample); diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp index 119cfe3ba5..75305b494d 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp @@ -115,6 +115,13 @@ class SchemaTrackerBase */ const SchemaTrackerConfig& config() const; +protected: + //! Runtime identity of the discovered collection, distinct from its reusable list index. + std::optional collection_instance_id() const + { + return m_target_collection_index.has_value() ? m_target_collection_id : std::nullopt; + } + private: void initialize_tensor_data_functions(); void create_tensor_list(); @@ -137,6 +144,7 @@ class SchemaTrackerBase PFN_xrDestroyTensorListNV m_destroy_list_fn{ nullptr }; std::optional m_target_collection_index; + std::optional m_target_collection_id; uint32_t m_sample_batch_stride{ 0 }; uint32_t m_sample_size{ 0 }; diff --git a/src/core/live_trackers/cpp/schema_tracker_base.cpp b/src/core/live_trackers/cpp/schema_tracker_base.cpp index d99cdb39b1..bb688bd9ad 100644 --- a/src/core/live_trackers/cpp/schema_tracker_base.cpp +++ b/src/core/live_trackers/cpp/schema_tracker_base.cpp @@ -143,12 +143,14 @@ bool SchemaTrackerBase::ensure_collection() m_sample_batch_stride = coll_props.sampleBatchStride; m_sample_size = coll_props.data.totalSampleSize; - if (i != m_target_collection_index) + // A recreated collection can reuse its list index, but never its runtime ID. + if (i != m_target_collection_index || coll_props.tensorCollectionId != m_target_collection_id) { m_last_sample_index.reset(); m_logger->info("Found target collection at index {}", i); } m_target_collection_index = i; + m_target_collection_id = coll_props.tensorCollectionId; return true; } diff --git a/tests/cpp/core/tracker_metadata/fake_tensor_runtime.hpp b/tests/cpp/core/tracker_metadata/fake_tensor_runtime.hpp index 1aca73e27d..b0c01d7ff3 100644 --- a/tests/cpp/core/tracker_metadata/fake_tensor_runtime.hpp +++ b/tests/cpp/core/tracker_metadata/fake_tensor_runtime.hpp @@ -58,7 +58,8 @@ class FakeTensorRuntime //! Creates `collection_id` with samples of `sample_size` bytes, as a producer starting up does. void create(const std::string& collection_id, uint32_t sample_size) { - m_collections.push_back(std::make_shared(Collection{ collection_id, sample_size })); + m_collections.push_back( + std::make_shared(Collection{ collection_id, sample_size, m_next_collection_id++ })); ++m_generation; } @@ -96,6 +97,7 @@ class FakeTensorRuntime { std::string id; uint32_t sample_size; + XrTensorCollectionIDNV instance_id; std::vector samples{}; int64_t next_index = 0; bool destroyed = false; @@ -160,6 +162,7 @@ class FakeTensorRuntime properties->data.tensorCount = 1; properties->data.totalSampleSize = collection.sample_size; properties->sampleBatchStride = collection.sample_size; + properties->tensorCollectionId = collection.instance_id; return XR_SUCCESS; } @@ -231,6 +234,7 @@ class FakeTensorRuntime std::vector> m_collections; std::map m_lists; uintptr_t m_next_list = 1; + XrTensorCollectionIDNV m_next_collection_id = 1; uint64_t m_generation = 1; }; diff --git a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp index 37bebfea0b..85bc7a9e73 100644 --- a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp +++ b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp @@ -390,6 +390,41 @@ TEST_CASE("A restarted producer's data waits for its metadata, however it is pad CHECK(live.step() == 0.3f); } +TEST_CASE("A recreated metadata collection is rechecked at the same list index", "[unit][tracker_metadata]") +{ + LiveSession live; + live.start_data(); + live.start_metadata(); + live.push_metadata(make_metadata("glove")); + live.push_data(0.1f); + REQUIRE(live.step() == 0.1f); + const auto first = live.impl->get_metadata(); + + // No polling between destruction and recreation: the list position is reused. + live.runtime.destroy(METADATA_COLLECTION); + live.start_metadata(); + live.push_data(0.2f); + + SECTION("the original metadata is accepted") + { + live.push_metadata(make_metadata("glove")); + CHECK(live.step() == 0.2f); + CHECK(live.impl->get_metadata().buffer().data() == first.buffer().data()); + } + SECTION("data waits for the recreated collection's sample") + { + CHECK_FALSE(live.step()); + live.push_metadata(make_metadata("glove")); + live.push_data(0.3f); + CHECK(live.step() == 0.3f); + } + SECTION("different metadata is rejected") + { + live.push_metadata(make_metadata("other")); + CHECK_THROWS_WITH(live.step(), Catch::Matchers::ContainsSubstring("differs")); + } +} + TEST_CASE("Different metadata is an error naming its collection", "[unit][tracker_metadata]") { using Catch::Matchers::ContainsSubstring; From fb817ca116e4217c391150f1ad239805a5661610 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 23:13:31 -0700 Subject: [PATCH 20/45] Expose default tracker buffer sizes to Python Signed-off-by: Lotus Li --- .../isaaccapture_examples/tactile/common.py | 3 -- .../isaaccapture_examples/tactile/feeder.py | 5 ++-- .../templates/fragments/pybind_pull.template | 5 +++- tests/python/core/CMakeLists.txt | 1 + .../core/deviceio_trackers/CMakeLists.txt | 12 ++++++++ .../core/deviceio_trackers/pyproject.toml | 13 ++++++++ .../core/deviceio_trackers/test_defaults.py | 30 +++++++++++++++++++ 7 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 tests/python/core/deviceio_trackers/CMakeLists.txt create mode 100644 tests/python/core/deviceio_trackers/pyproject.toml create mode 100644 tests/python/core/deviceio_trackers/test_defaults.py diff --git a/examples/tactile/python/isaaccapture_examples/tactile/common.py b/examples/tactile/python/isaaccapture_examples/tactile/common.py index 6174ba63cf..12bb0466a7 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/common.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/common.py @@ -9,6 +9,3 @@ MAX_NUM_JOINTS = 26 DEFAULT_NUM_JOINTS = MAX_NUM_JOINTS DEFAULT_NUM_TAXELS = 200 - -# TactileTracker's default max_flatbuffer_size: the reader rejects a larger collection. -TACTILE_MAX_FRAME_SIZE = 16384 diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py index 68016786f9..162ba7165c 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -19,7 +19,7 @@ from isaaccapture.cloudxr import CloudXRLauncher from isaaccapture.deviceio_session import DeviceIOSession -from isaaccapture.deviceio_trackers import TensorPushTracker +from isaaccapture.deviceio_trackers import TactileTracker, TensorPushTracker from isaaccapture.oxr import OpenXRSession from isaaccapture.schema import ( TRACKER_METADATA_MAX_SIZE, @@ -38,7 +38,6 @@ DEFAULT_NUM_TAXELS, MAX_NUM_JOINTS, MIN_NUM_JOINTS, - TACTILE_MAX_FRAME_SIZE, ) logger = logging.getLogger("isaaccapture.examples.tactile.feeder") @@ -159,7 +158,7 @@ def main(argv: list[str]) -> int: with ExitStack() as stack: stack.enter_context(CloudXRLauncher.launch_context(args)) data_tracker = TensorPushTracker( - args.collection_id, "tactile", TACTILE_MAX_FRAME_SIZE + args.collection_id, "tactile", TactileTracker.DEFAULT_MAX_FLATBUFFER_SIZE ) metadata_tracker = TensorPushTracker( tracker_metadata_collection_id(args.collection_id), diff --git a/src/core/codegen/templates/fragments/pybind_pull.template b/src/core/codegen/templates/fragments/pybind_pull.template index f059b299c4..389e886a73 100644 --- a/src/core/codegen/templates/fragments/pybind_pull.template +++ b/src/core/codegen/templates/fragments/pybind_pull.template @@ -1,4 +1,7 @@ - py::class_>(m, "@CLASS@") + py::class_> @NAME@_pull(m, "@CLASS@"); + @NAME@_pull.attr("DEFAULT_MAX_FLATBUFFER_SIZE") = + static_cast(core::@CLASS@::DEFAULT_MAX_FLATBUFFER_SIZE); + @NAME@_pull .def(py::init(), py::arg("collection_id"), py::arg("max_flatbuffer_size") = core::@CLASS@::DEFAULT_MAX_FLATBUFFER_SIZE) .def( diff --git a/tests/python/core/CMakeLists.txt b/tests/python/core/CMakeLists.txt index 1c96b71c32..c96d5abc4c 100644 --- a/tests/python/core/CMakeLists.txt +++ b/tests/python/core/CMakeLists.txt @@ -6,6 +6,7 @@ add_subdirectory(retargeting_engine) add_subdirectory(teleop_session_manager) if(BUILD_PYTHON_BINDINGS) + add_subdirectory(deviceio_trackers) add_subdirectory(cloudxr) # Needs `import isaaccapture`, and drives the C++ half through the emitter # built under tests/cpp/core/log_bridge. diff --git a/tests/python/core/deviceio_trackers/CMakeLists.txt b/tests/python/core/deviceio_trackers/CMakeLists.txt new file mode 100644 index 0000000000..101a9234ba --- /dev/null +++ b/tests/python/core/deviceio_trackers/CMakeLists.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_test( + NAME deviceio_trackers_test_defaults + COMMAND uv run --python ${ISAAC_TELEOP_PYTHON_VERSION} --extra dev pytest -v --tb=short test_defaults.py + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" +) + +set_tests_properties(deviceio_trackers_test_defaults PROPERTIES + ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/python_package/$" +) diff --git a/tests/python/core/deviceio_trackers/pyproject.toml b/tests/python/core/deviceio_trackers/pyproject.toml new file mode 100644 index 0000000000..eea42f28a9 --- /dev/null +++ b/tests/python/core/deviceio_trackers/pyproject.toml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[project] +name = "isaaccapture-deviceio-trackers-tests" +version = "0.0.0" +requires-python = ">=3.11,<3.14" + +[project.optional-dependencies] +dev = ["pytest", "numpy", "pyyaml"] + +[tool.pytest.ini_options] +pythonpath = [".", "../.."] diff --git a/tests/python/core/deviceio_trackers/test_defaults.py b/tests/python/core/deviceio_trackers/test_defaults.py new file mode 100644 index 0000000000..fe77503ede --- /dev/null +++ b/tests/python/core/deviceio_trackers/test_defaults.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Check generated tracker defaults through the real Python bindings.""" + +import runpy + +import pytest +from repo_paths import repo_root + +from isaaccapture import deviceio_trackers + +_core = repo_root() / "src" / "core" +_manifest = runpy.run_path(str(_core / "codegen" / "manifest.py")) +_entries = _manifest["load_manifest"]( + _core / "deviceio_trackers" / "trackers.toml", + _core / "deviceio_trackers" / "defaults.toml", +) + + +@pytest.mark.parametrize( + "entry", + [entry for entry in _entries if entry["direction"] == "pull"], + ids=lambda entry: entry["class"], +) +def test_default_flatbuffer_size_matches_manifest(entry): + tracker_type = getattr(deviceio_trackers, entry["class"]) + size = tracker_type.DEFAULT_MAX_FLATBUFFER_SIZE + assert isinstance(size, int) + assert size == entry["max_flatbuffer_size"] From 1434c2a97d8f617d73c1dd49d6baadec7e2a0402 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 23:20:40 -0700 Subject: [PATCH 21/45] Separate tactile segment indices from hand joint ordinals Signed-off-by: Lotus Li --- .../isaaccapture_examples/tactile/feeder.py | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py index 162ba7165c..c3c19026d0 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -48,47 +48,58 @@ FINGER_DISTAL_JOINTS = (4, 9, 14, 19, 24) -def mock_metadata(num_joints: int, num_taxels: int) -> TactileMetadata: +def mock_metadata(joint_ids: list[HandJoint], num_taxels: int) -> TactileMetadata: """Build metadata for a flat hand with taxels on its distal segments.""" - positions = np.zeros((num_joints, 3), np.float32) - positions[0] = (0.0, 0.05, 0.0) - - taxel_joint_ids = [] + num_segments = len(joint_ids) + # LBS indices address joint_ids rows, not HandJoint ordinals. + segment_by_joint = {int(joint): segment for segment, joint in enumerate(joint_ids)} + positions = np.zeros((num_segments, 3), np.float32) + if int(HandJoint.PALM) in segment_by_joint: + positions[segment_by_joint[int(HandJoint.PALM)]] = (0.0, 0.05, 0.0) + + taxel_segments = [] for finger, first in enumerate(FINGER_FIRST_JOINTS): - if first >= num_joints: - break finger_end = ( - FINGER_FIRST_JOINTS[finger + 1] - 1 if finger < 4 else MAX_NUM_JOINTS - 1 + FINGER_FIRST_JOINTS[finger + 1] + if finger < len(FINGER_FIRST_JOINTS) - 1 + else int(HandJoint.NUM_JOINTS) ) - last = min(finger_end, num_joints - 1) - for joint in range(first, last + 1): - positions[joint] = ( + finger_joints = [ + joint for joint in range(first, finger_end) if joint in segment_by_joint + ] + if not finger_joints: + continue + for joint in finger_joints: + positions[segment_by_joint[joint]] = ( -0.04 + 0.02 * finger, 0.02 + 0.025 * (joint - first), 0.0, ) - taxel_joint_ids.append(min(FINGER_DISTAL_JOINTS[finger], last)) + joint = min( + finger_joints, key=lambda joint: abs(joint - FINGER_DISTAL_JOINTS[finger]) + ) + taxel_segments.append(segment_by_joint[joint]) - if not taxel_joint_ids: - taxel_joint_ids.append(WRIST) + if not taxel_segments: + taxel_segments.append(segment_by_joint.get(WRIST, 0)) taxel_positions = np.zeros((num_taxels, 3), np.float32) taxel_segment_indices = np.zeros((num_taxels, 4), np.uint32) taxel_segment_weights = np.zeros((num_taxels, 4), np.float32) - per_segment = math.ceil(num_taxels / len(taxel_joint_ids)) + per_segment = math.ceil(num_taxels / len(taxel_segments)) for taxel in range(num_taxels): - segment, slot = taxel % len(taxel_joint_ids), taxel // len(taxel_joint_ids) - joint = taxel_joint_ids[segment] - taxel_positions[taxel] = positions[joint] + ( + segment_index = taxel_segments[taxel % len(taxel_segments)] + slot = taxel // len(taxel_segments) + taxel_positions[taxel] = positions[segment_index] + ( 0.0, 0.02 * slot / per_segment, -0.005, ) - taxel_segment_indices[taxel] = joint + taxel_segment_indices[taxel] = segment_index taxel_segment_weights[taxel, 0] = 1.0 segment_poses = np.column_stack( - (positions, np.tile([0.0, 0.0, 0.0, 1.0], (num_joints, 1))) + (positions, np.tile([0.0, 0.0, 0.0, 1.0], (num_segments, 1))) ) taxel_poses = np.column_stack( (taxel_positions, np.tile([1.0, 0.0, 0.0, 0.0], (num_taxels, 1))) @@ -97,7 +108,7 @@ def mock_metadata(num_joints: int, num_taxels: int) -> TactileMetadata: geometry_mapping=LBS( segment_poses, taxel_poses, taxel_segment_indices, taxel_segment_weights ), - segment_source=HandSkeletonRef([HandJoint(i) for i in range(num_joints)]), + segment_source=HandSkeletonRef(joint_ids), ) @@ -139,7 +150,8 @@ def main(argv: list[str]) -> int: if not math.isfinite(args.metadata_interval) or args.metadata_interval <= 0: parser.error("--metadata-interval must be finite and positive") - metadata_bytes = mock_metadata(args.joints, args.taxels).to_bytes() + joint_ids = [HandJoint(i) for i in range(args.joints)] + metadata_bytes = mock_metadata(joint_ids, args.taxels).to_bytes() if len(metadata_bytes) > TRACKER_METADATA_MAX_SIZE: parser.error( f"{args.joints} joints and {args.taxels} taxels make a {len(metadata_bytes)} B TactileMetadata; " From ee5ed742bdd93a4da8f3f3dd7ce3ab9f9fa60630 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 23:31:07 -0700 Subject: [PATCH 22/45] Simplify tactile feeder context management Signed-off-by: Lotus Li --- .../isaaccapture_examples/tactile/feeder.py | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py index c3c19026d0..d2215c4b67 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -13,7 +13,6 @@ import math import sys import time -from contextlib import ExitStack import numpy as np @@ -167,26 +166,22 @@ def main(argv: list[str]) -> int: frames = 0 last_metadata_push = -math.inf - with ExitStack() as stack: - stack.enter_context(CloudXRLauncher.launch_context(args)) - data_tracker = TensorPushTracker( - args.collection_id, "tactile", TactileTracker.DEFAULT_MAX_FLATBUFFER_SIZE - ) - metadata_tracker = TensorPushTracker( - tracker_metadata_collection_id(args.collection_id), - TRACKER_METADATA_TENSOR_IDENTIFIER, - len(metadata_bytes), - ) - trackers = [data_tracker, metadata_tracker] - oxr_session = stack.enter_context( - OpenXRSession( - "TactileFeeder", DeviceIOSession.get_required_extensions(trackers) - ) - ) - session = stack.enter_context( - DeviceIOSession.run(trackers, oxr_session.get_handles()) - ) - + data_tracker = TensorPushTracker( + args.collection_id, "tactile", TactileTracker.DEFAULT_MAX_FLATBUFFER_SIZE + ) + metadata_tracker = TensorPushTracker( + tracker_metadata_collection_id(args.collection_id), + TRACKER_METADATA_TENSOR_IDENTIFIER, + len(metadata_bytes), + ) + trackers = [data_tracker, metadata_tracker] + with ( + CloudXRLauncher.launch_context(args), + OpenXRSession( + "TactileFeeder", DeviceIOSession.get_required_extensions(trackers) + ) as oxr_session, + DeviceIOSession.run(trackers, oxr_session.get_handles()) as session, + ): start = time.monotonic() try: while args.duration <= 0 or time.monotonic() - start < args.duration: From 8214cabcc113fa29d9119919d14cb002bf3536b4 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 23:41:18 -0700 Subject: [PATCH 23/45] Apply reviews on docs and example Signed-off-by: Lotus Li --- docs/source/device/tactile/index.rst | 5 ++-- examples/tactile/README.md | 4 +-- .../isaaccapture_examples/tactile/common.py | 5 ---- .../isaaccapture_examples/tactile/feeder.py | 27 ++++++++++++++----- .../isaaccapture_examples/tactile/printer.py | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/docs/source/device/tactile/index.rst b/docs/source/device/tactile/index.rst index 578caf29cf..40cb4e4fa0 100644 --- a/docs/source/device/tactile/index.rst +++ b/docs/source/device/tactile/index.rst @@ -196,9 +196,8 @@ and exposes it through ``get_metadata()``. It withholds force data until metadat is confirmed. Different or invalid metadata causes an error; after a producer restart, its metadata must be confirmed again. See :ref:`tracker-metadata`. -The default force-buffer size is 16,384 bytes; larger frames require a matching -``max_flatbuffer_size`` on the producer and consumer. Metadata has its own -``TRACKER_METADATA_MAX_SIZE`` limit of 68,864 bytes, including FlatBuffer overhead. +The force-buffer size is defined by ``max_flatbuffer_size``. Metadata has its own +``TRACKER_METADATA_MAX_SIZE`` limit, including FlatBuffer overhead. The data array length must match the metadata's taxel count. Applications can read ``TactileTracker`` directly for live or replay data. diff --git a/examples/tactile/README.md b/examples/tactile/README.md index 6e86241b42..8dc7486fe6 100644 --- a/examples/tactile/README.md +++ b/examples/tactile/README.md @@ -47,8 +47,8 @@ python -m isaaccapture_examples.tactile.printer --replay recordings/tactile_ TactileMetadata: diff --git a/examples/tactile/python/isaaccapture_examples/tactile/printer.py b/examples/tactile/python/isaaccapture_examples/tactile/printer.py index 8bc0b849a7..1fd1993b5f 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/printer.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/printer.py @@ -130,7 +130,7 @@ def main(argv: list[str]) -> int: "--print-interval", type=float, default=0.5, - help="Seconds between live prints; replay prints every 30 records", + help=f"Seconds between live prints; replay prints every {REPLAY_PRINT_EVERY_RECORDS} records", ) parser.add_argument( "--output", From 9a418fbe3e76ad37380b323bc1cd10ccbdc232eb Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 22 Oct 2026 23:53:49 -0700 Subject: [PATCH 24/45] Only verify metadata once at first reception Signed-off-by: Lotus Li --- .../live_trackers/schema_metadata_tracker.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp index 1595b8e9ff..0f4f1a687e 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp @@ -104,18 +104,18 @@ class SchemaMetadataTracker : public SchemaTrackerBase if (!verifier.VerifyBuffer(nullptr)) { throw std::runtime_error(collection_id + ": metadata sample is not a valid " + - MetadataT::GetFullyQualifiedName()); - } - - // Schema-specific dispatch belongs to the schema layer, not this generic reader. - if (!verify_metadata(*flatbuffers::GetRoot(sample.buffer.data()))) - { - throw std::runtime_error(collection_id + ": metadata failed semantic verification for " + - MetadataT::GetFullyQualifiedName()); + MetadataT::GetFullyQualifiedName()); } if (!metadata_) { + // Schema-specific dispatch belongs to the schema layer, not this generic reader. + if (!verify_metadata(*flatbuffers::GetRoot(sample.buffer.data()))) + { + throw std::runtime_error(collection_id + ": metadata failed semantic verification for " + + MetadataT::GetFullyQualifiedName()); + } + const int64_t log_time_ns = sample.timestamp.available_time_local_common_clock(); metadata_ = Serialized::adopt(std::move(sample.buffer)); if (mcap_channels_ != nullptr) From f7e2333e26b0ae33f95c0a6c277a9f7558a8b1c5 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 00:00:46 -0700 Subject: [PATCH 25/45] Validate replay tracker metadata semantics Signed-off-by: Lotus Li --- .../templates/pull/replay.cpp.template | 12 +++- .../test_tracker_metadata.cpp | 68 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/core/codegen/templates/pull/replay.cpp.template b/src/core/codegen/templates/pull/replay.cpp.template index 9f2c6e0e99..30d4cbbec9 100644 --- a/src/core/codegen/templates/pull/replay.cpp.template +++ b/src/core/codegen/templates/pull/replay.cpp.template @@ -5,10 +5,13 @@ #include #if @HAS_METADATA@ +#include #include #endif #include +#include +#include #include #include @@ -23,7 +26,14 @@ Serialized<@METADATA@> @REPLAY_IMPL@::read_metadata(std::unique_ptr viewers( std::move(reader), base_name, std::vector(@TRAITS@::metadata_channels.begin(), @TRAITS@::metadata_channels.end()), recorded); - return viewers.read(0); + auto metadata = viewers.read(0); + // MCAP verifies structure; apply the same schema semantics as live ingestion. + if (metadata && !verify_metadata(*metadata)) + { + throw std::runtime_error(std::string(base_name) + ": metadata failed semantic verification for " + + @METADATA@::GetFullyQualifiedName()); + } + return metadata; } @REPLAY_IMPL@::@REPLAY_IMPL@(std::unique_ptr reader, std::string_view base_name, diff --git a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp index 85bc7a9e73..872af17851 100644 --- a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp +++ b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp @@ -14,11 +14,15 @@ #include #include +#include #include #include #include #include +#include +#include #include +#include #include #include #include @@ -268,6 +272,70 @@ TEST_CASE("Replay withholds data recorded without metadata", "[unit][tracker_met CHECK_FALSE(replay->get_data()); } +TEST_CASE("Replay tactile metadata is semantically verified before force data is published", "[unit][tracker_metadata]") +{ + constexpr std::string_view base_name = "tactile"; + const auto path = temp_mcap_path("test_tactile_replay_metadata"); + const TempFileCleanup cleanup(path); + std::array indices{}; + std::array weights{ 1.0f, 0.0f, 0.0f, 0.0f }; + bool valid = true; + SECTION("valid metadata") + { + } + SECTION("segment index outside the single segment") + { + indices[0] = 7; + valid = false; + } + SECTION("all four weights are zero") + { + weights = {}; + valid = false; + } + + core::HandSkeletonRefT hand; + hand.joint_ids = { core::HandJoint_WRIST }; + core::LBST lbs; + lbs.segment_bind_poses.emplace_back(core::Point(0, 0, 0), core::Quaternion(0, 0, 0, 1)); + lbs.taxel_bind_poses = lbs.segment_bind_poses; + lbs.taxel_segment_indices.emplace_back(flatbuffers::span(indices.data(), 4)); + lbs.taxel_segment_weights.emplace_back(flatbuffers::span(weights.data(), 4)); + core::TactileMetadataT native; + native.geometry_mapping.Set(std::move(lbs)); + native.segment_source.Set(std::move(hand)); + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + REQUIRE(core::verify_metadata(*metadata) == valid); + + auto writer = open_writer(path); + auto channels = core::LiveTactileTrackerImpl::create_mcap_channels(*writer, base_name); + channels->metadata.write(0, metadata, 5); + core::TactileDataArrayT forces; + forces.taxel_forces.emplace_back(0.0f, 0.0f, 1.0f); + channels->data.write( + 1, core::pack_record(&forces, core::DeviceDataTimestamp(10, 10, 10))); + writer->close(); + + const auto recorded = recorded_schemas(path); + if (!valid) + { + CHECK_THROWS_WITH(core::ReplayTactileTrackerImpl::read_metadata(open_reader(path), base_name, recorded), + Catch::Matchers::ContainsSubstring("semantic verification") && + Catch::Matchers::ContainsSubstring(std::string(base_name))); + return; + } + + auto loaded = core::ReplayTactileTrackerImpl::read_metadata(open_reader(path), base_name, recorded); + core::ReplayTactileTrackerImpl replay(open_reader(path), base_name, recorded, std::move(loaded)); + REQUIRE(replay.get_metadata()); + replay.update(0); + REQUIRE(replay.get_data()); + REQUIRE(replay.get_data()->taxel_forces()->size() == 1); + CHECK(replay.get_data()->taxel_forces()->Get(0)->z() == 1.0f); +} + #if defined(XR_USE_TIMESPEC) TEST_CASE("Live tactile metadata is semantically verified before adoption and recording", "[unit][tracker_metadata]") From 71b8d1def8c75a744426f68cfc4520293f970027 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 00:38:02 -0700 Subject: [PATCH 26/45] Report specific tactile metadata validation errors Signed-off-by: Lotus Li --- .../templates/pull/replay.cpp.template | 12 +++- .../live_trackers/schema_metadata_tracker.hpp | 13 ++-- .../cpp/inc/schema/metadata_validation.hpp | 9 ++- .../schema/tactile_metadata_validation.hpp | 69 ++++++++++++++----- .../cpp/core/schema/test_tactile_metadata.cpp | 40 +++++++++-- .../test_tracker_metadata.cpp | 17 +++-- 6 files changed, 117 insertions(+), 43 deletions(-) diff --git a/src/core/codegen/templates/pull/replay.cpp.template b/src/core/codegen/templates/pull/replay.cpp.template index 30d4cbbec9..1cd0d30bc5 100644 --- a/src/core/codegen/templates/pull/replay.cpp.template +++ b/src/core/codegen/templates/pull/replay.cpp.template @@ -28,10 +28,16 @@ Serialized<@METADATA@> @REPLAY_IMPL@::read_metadata(std::unique_ptr(@TRAITS@::metadata_channels.begin(), @TRAITS@::metadata_channels.end()), recorded); auto metadata = viewers.read(0); // MCAP verifies structure; apply the same schema semantics as live ingestion. - if (metadata && !verify_metadata(*metadata)) + if (metadata) { - throw std::runtime_error(std::string(base_name) + ": metadata failed semantic verification for " + - @METADATA@::GetFullyQualifiedName()); + try + { + verify_metadata(*metadata); + } + catch (const std::runtime_error& error) + { + throw std::runtime_error(std::string(base_name) + ": " + error.what()); + } } return metadata; } diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp index 0f4f1a687e..cd13bfee9e 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_metadata_tracker.hpp @@ -104,18 +104,21 @@ class SchemaMetadataTracker : public SchemaTrackerBase if (!verifier.VerifyBuffer(nullptr)) { throw std::runtime_error(collection_id + ": metadata sample is not a valid " + - MetadataT::GetFullyQualifiedName()); + MetadataT::GetFullyQualifiedName()); } if (!metadata_) { // Schema-specific dispatch belongs to the schema layer, not this generic reader. - if (!verify_metadata(*flatbuffers::GetRoot(sample.buffer.data()))) + try { - throw std::runtime_error(collection_id + ": metadata failed semantic verification for " + - MetadataT::GetFullyQualifiedName()); + verify_metadata(*flatbuffers::GetRoot(sample.buffer.data())); } - + catch (const std::runtime_error& error) + { + throw std::runtime_error(collection_id + ": " + error.what()); + } + const int64_t log_time_ns = sample.timestamp.available_time_local_common_clock(); metadata_ = Serialized::adopt(std::move(sample.buffer)); if (mcap_channels_ != nullptr) diff --git a/src/core/schema/cpp/inc/schema/metadata_validation.hpp b/src/core/schema/cpp/inc/schema/metadata_validation.hpp index df047f941d..9979d756c9 100644 --- a/src/core/schema/cpp/inc/schema/metadata_validation.hpp +++ b/src/core/schema/cpp/inc/schema/metadata_validation.hpp @@ -8,17 +8,16 @@ namespace core { -//! Semantic verification after FlatBuffers verification; schemas without extra rules pass. +//! Throws on invalid semantics after FlatBuffers verification; schemas without extra rules are a no-op. template -[[nodiscard]] inline bool verify_metadata(const MetadataT&) +inline void verify_metadata(const MetadataT&) { - return true; } template <> -[[nodiscard]] inline bool verify_metadata(const TactileMetadata& metadata) +inline void verify_metadata(const TactileMetadata& metadata) { - return verify_tactile_metadata(metadata); + verify_tactile_metadata(metadata); } } // namespace core diff --git a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp index 6f9c8375e4..0aeb6cf15c 100644 --- a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp +++ b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp @@ -7,66 +7,101 @@ #include #include +#include +#include namespace core { //! Checks the semantics of the supported LBS/HandSkeletonRef layout after FlatBuffers verification. //! Weight sums may differ from one by at most 1e-5 to allow floating-point rounding. -[[nodiscard]] inline bool verify_tactile_metadata(const TactileMetadata& metadata) +//! Throws std::runtime_error naming the invalid field and the violated constraint. +inline void verify_tactile_metadata(const TactileMetadata& metadata) { const auto* lbs = metadata.geometry_mapping_as_LBS(); const auto* hand = metadata.segment_source_as_HandSkeletonRef(); - if (lbs == nullptr || hand == nullptr) + if (lbs == nullptr) { - return false; + throw std::runtime_error("TactileMetadata.geometry_mapping: expected LBS"); + } + if (hand == nullptr) + { + throw std::runtime_error("TactileMetadata.segment_source: expected HandSkeletonRef"); } const auto taxel_count = lbs->taxel_bind_poses()->size(); - if (lbs->taxel_segment_indices()->size() != taxel_count || lbs->taxel_segment_weights()->size() != taxel_count || - lbs->segment_bind_poses()->size() != hand->joint_ids()->size()) + if (lbs->taxel_segment_indices()->size() != taxel_count) + { + throw std::runtime_error("TactileMetadata.taxel_segment_indices: expected " + std::to_string(taxel_count) + + " rows to match taxel_bind_poses, got " + + std::to_string(lbs->taxel_segment_indices()->size())); + } + if (lbs->taxel_segment_weights()->size() != taxel_count) { - return false; + throw std::runtime_error("TactileMetadata.taxel_segment_weights: expected " + std::to_string(taxel_count) + + " rows to match taxel_bind_poses, got " + + std::to_string(lbs->taxel_segment_weights()->size())); + } + if (lbs->segment_bind_poses()->size() != hand->joint_ids()->size()) + { + throw std::runtime_error( + "TactileMetadata.segment_bind_poses: expected " + std::to_string(hand->joint_ids()->size()) + + " poses to match segment_source.joint_ids, got " + std::to_string(lbs->segment_bind_poses()->size())); } std::array seen{}; - for (const auto joint_id : *hand->joint_ids()) + for (flatbuffers::uoffset_t i = 0; i < hand->joint_ids()->size(); ++i) { - if (joint_id >= HandJoint_NUM_JOINTS || seen[joint_id]) + const auto joint_id = hand->joint_ids()->Get(i); + if (joint_id >= HandJoint_NUM_JOINTS) + { + throw std::runtime_error("TactileMetadata.joint_ids[" + std::to_string(i) + + "]: invalid HandJoint identifier " + std::to_string(joint_id)); + } + if (seen[joint_id]) { - return false; + throw std::runtime_error("TactileMetadata.joint_ids[" + std::to_string(i) + + "]: duplicate HandJoint identifier " + std::to_string(joint_id)); } seen[joint_id] = true; } - for (const auto* indices : *lbs->taxel_segment_indices()) + for (flatbuffers::uoffset_t i = 0; i < taxel_count; ++i) { - for (const auto segment_index : *indices->values()) + const auto* indices = lbs->taxel_segment_indices()->Get(i); + for (flatbuffers::uoffset_t k = 0; k < indices->values()->size(); ++k) { + const auto segment_index = indices->values()->Get(k); if (segment_index >= lbs->segment_bind_poses()->size()) { - return false; + throw std::runtime_error("TactileMetadata.taxel_segment_indices[" + std::to_string(i) + "][" + + std::to_string(k) + "]: segment index " + std::to_string(segment_index) + + " is outside segment_bind_poses (size " + + std::to_string(lbs->segment_bind_poses()->size()) + ")"); } } } - for (const auto* weights : *lbs->taxel_segment_weights()) + for (flatbuffers::uoffset_t i = 0; i < taxel_count; ++i) { + const auto* weights = lbs->taxel_segment_weights()->Get(i); double sum = 0.0; - for (const float weight : *weights->values()) + for (flatbuffers::uoffset_t k = 0; k < weights->values()->size(); ++k) { + const float weight = weights->values()->Get(k); if (!std::isfinite(weight) || weight < 0.0f) { - return false; + throw std::runtime_error("TactileMetadata.taxel_segment_weights[" + std::to_string(i) + "][" + + std::to_string(k) + "]: weights must be finite and nonnegative"); } sum += weight; } if (std::abs(sum - 1.0) > 1e-5) { - return false; + throw std::runtime_error("TactileMetadata.taxel_segment_weights[" + std::to_string(i) + + "]: weights must sum to one, got " + std::to_string(sum)); } } - return true; } } // namespace core diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index 1b914a563c..dd8cc8ae59 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include #include @@ -82,39 +83,46 @@ TEST_CASE("TactileMetadata verifies and retains nested union views", "[unit][tac TEST_CASE("TactileMetadata semantic verification accepts valid metadata", "[unit][tactile_metadata]") { const auto metadata = core::pack(make_metadata()); - CHECK(core::verify_metadata(*metadata)); + CHECK_NOTHROW(core::verify_metadata(*metadata)); + CHECK_NOTHROW(core::verify_metadata(*metadata)); } TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", "[unit][tactile_metadata]") { auto native = make_metadata(); auto& lbs = *native.geometry_mapping.AsLBS(); + std::string reason; SECTION("taxel bind poses") { lbs.taxel_bind_poses.push_back(lbs.taxel_bind_poses.front()); + reason = "taxel_segment_indices: expected 2 rows to match taxel_bind_poses, got 1"; } SECTION("taxel segment indices") { lbs.taxel_segment_indices.clear(); + reason = "taxel_segment_indices: expected 1 rows to match taxel_bind_poses, got 0"; } SECTION("taxel segment weights") { lbs.taxel_segment_weights.clear(); + reason = "taxel_segment_weights: expected 1 rows to match taxel_bind_poses, got 0"; } SECTION("joint identifiers and segment bind poses") { native.segment_source.AsHandSkeletonRef()->joint_ids.pop_back(); + reason = "segment_bind_poses: expected 1 poses to match segment_source.joint_ids, got 2"; } const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK_FALSE(core::verify_metadata(*metadata)); + CHECK_THROWS_WITH(core::verify_metadata(*metadata), "TactileMetadata." + reason); } TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joints", "[unit][tactile_metadata]") { auto native = make_metadata(); auto& joint_ids = native.segment_source.AsHandSkeletonRef()->joint_ids; + std::string reason = "invalid HandJoint identifier"; SECTION("sentinel") { joint_ids[1] = core::HandJoint_NUM_JOINTS; @@ -126,11 +134,13 @@ TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joi SECTION("duplicate identifier") { joint_ids[1] = joint_ids[0]; + reason = "duplicate HandJoint identifier"; } const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK_FALSE(core::verify_metadata(*metadata)); + CHECK_THROWS_WITH(core::verify_metadata(*metadata), + Catch::Matchers::ContainsSubstring("TactileMetadata.joint_ids[1]: " + reason)); } TEST_CASE("TactileMetadata semantic verification rejects out-of-range segment indices", "[unit][tactile_metadata]") @@ -138,19 +148,23 @@ TEST_CASE("TactileMetadata semantic verification rejects out-of-range segment in auto native = make_metadata(); auto& lbs = *native.geometry_mapping.AsLBS(); std::array indices = { 1, 0, 0, 0 }; + std::string slot; SECTION("nonzero-weight slot") { indices[0] = 2; + slot = "0"; } SECTION("zero-weight slot") { indices[3] = 2; + slot = "3"; } lbs.taxel_segment_indices[0] = core::UInt4(flatbuffers::span(indices.data(), 4)); const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK_FALSE(core::verify_metadata(*metadata)); + CHECK_THROWS_WITH(core::verify_metadata(*metadata), "TactileMetadata.taxel_segment_indices[0][" + slot + + "]: segment index 2 is outside segment_bind_poses (size 2)"); } TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", "[unit][tactile_metadata]") @@ -159,9 +173,11 @@ TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", " auto& lbs = *native.geometry_mapping.AsLBS(); std::array weights = { 0.75f, 0.25f, 0.0f, 0.0f }; bool valid = false; + std::string reason = "taxel_segment_weights[1]: weights must sum to one"; SECTION("negative weight with sum one") { weights = { 1.25f, -0.25f, 0.0f, 0.0f }; + reason = "taxel_segment_weights[1][1]: weights must be finite and nonnegative"; } SECTION("sum below one") { @@ -174,10 +190,12 @@ TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", " SECTION("NaN") { weights[3] = std::numeric_limits::quiet_NaN(); + reason = "taxel_segment_weights[1][3]: weights must be finite and nonnegative"; } SECTION("infinity") { weights[3] = std::numeric_limits::infinity(); + reason = "taxel_segment_weights[1][3]: weights must be finite and nonnegative"; } SECTION("rounding within tolerance") { @@ -195,6 +213,16 @@ TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", " const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - CHECK(core::verify_metadata(*metadata) == valid); - CHECK(core::verify_metadata(*metadata) == valid); + if (valid) + { + CHECK_NOTHROW(core::verify_metadata(*metadata)); + CHECK_NOTHROW(core::verify_metadata(*metadata)); + } + else + { + CHECK_THROWS_WITH( + core::verify_metadata(*metadata), Catch::Matchers::ContainsSubstring("TactileMetadata." + reason)); + CHECK_THROWS_WITH(core::verify_metadata(*metadata), + Catch::Matchers::ContainsSubstring("TactileMetadata." + reason)); + } } diff --git a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp index 872af17851..4cd2b6f6d9 100644 --- a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp +++ b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp @@ -279,19 +279,19 @@ TEST_CASE("Replay tactile metadata is semantically verified before force data is const TempFileCleanup cleanup(path); std::array indices{}; std::array weights{ 1.0f, 0.0f, 0.0f, 0.0f }; - bool valid = true; + std::string reason; SECTION("valid metadata") { } SECTION("segment index outside the single segment") { indices[0] = 7; - valid = false; + reason = "taxel_segment_indices[0][0]: segment index 7 is outside segment_bind_poses (size 1)"; } SECTION("all four weights are zero") { weights = {}; - valid = false; + reason = "taxel_segment_weights[0]: weights must sum to one"; } core::HandSkeletonRefT hand; @@ -307,7 +307,6 @@ TEST_CASE("Replay tactile metadata is semantically verified before force data is const auto metadata = core::pack(native); flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); - REQUIRE(core::verify_metadata(*metadata) == valid); auto writer = open_writer(path); auto channels = core::LiveTactileTrackerImpl::create_mcap_channels(*writer, base_name); @@ -319,14 +318,16 @@ TEST_CASE("Replay tactile metadata is semantically verified before force data is writer->close(); const auto recorded = recorded_schemas(path); - if (!valid) + if (!reason.empty()) { + REQUIRE_THROWS_WITH(core::verify_metadata(*metadata), Catch::Matchers::ContainsSubstring(reason)); CHECK_THROWS_WITH(core::ReplayTactileTrackerImpl::read_metadata(open_reader(path), base_name, recorded), - Catch::Matchers::ContainsSubstring("semantic verification") && + Catch::Matchers::ContainsSubstring("TactileMetadata." + reason) && Catch::Matchers::ContainsSubstring(std::string(base_name))); return; } + REQUIRE_NOTHROW(core::verify_metadata(*metadata)); auto loaded = core::ReplayTactileTrackerImpl::read_metadata(open_reader(path), base_name, recorded); core::ReplayTactileTrackerImpl replay(open_reader(path), base_name, recorded, std::move(loaded)); REQUIRE(replay.get_metadata()); @@ -366,7 +367,9 @@ TEST_CASE("Live tactile metadata is semantically verified before adoption and re REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); runtime.push(METADATA_COLLECTION, invalid.buffer(), 1); - CHECK_THROWS_WITH(tracker.update(), Catch::Matchers::ContainsSubstring("semantic verification")); + CHECK_THROWS_WITH(tracker.update(), + Catch::Matchers::ContainsSubstring(METADATA_COLLECTION + ": TactileMetadata.") && + Catch::Matchers::ContainsSubstring("taxel_segment_weights[0]: weights must sum to one")); CHECK_FALSE(tracker.get()); const std::array weights = { 1.0f, 0.0f, 0.0f, 0.0f }; From c6ad2c7ef081325f94f12baa9344b2f8e7f0a93c Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 00:40:28 -0700 Subject: [PATCH 27/45] Validate tactile bind pose finiteness and quaternion norms Signed-off-by: Lotus Li --- .../schema/tactile_metadata_validation.hpp | 28 +++++++ .../cpp/core/schema/test_tactile_metadata.cpp | 80 +++++++++++++++++++ .../core/schema/test_tactile_metadata.py | 29 ++++++- 3 files changed, 133 insertions(+), 4 deletions(-) diff --git a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp index 0aeb6cf15c..83ce528c03 100644 --- a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp +++ b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp @@ -15,6 +15,7 @@ namespace core //! Checks the semantics of the supported LBS/HandSkeletonRef layout after FlatBuffers verification. //! Weight sums may differ from one by at most 1e-5 to allow floating-point rounding. +//! Squared quaternion norms use the Python constructor's tolerance of 1e-4. //! Throws std::runtime_error naming the invalid field and the violated constraint. inline void verify_tactile_metadata(const TactileMetadata& metadata) { @@ -102,6 +103,33 @@ inline void verify_tactile_metadata(const TactileMetadata& metadata) "]: weights must sum to one, got " + std::to_string(sum)); } } + + const auto verify_poses = [](const flatbuffers::Vector* poses, const char* field) + { + for (flatbuffers::uoffset_t i = 0; i < poses->size(); ++i) + { + const auto& position = poses->Get(i)->position(); + const auto& orientation = poses->Get(i)->orientation(); + for (const float component : std::array{ position.x(), position.y(), position.z(), orientation.x(), + orientation.y(), orientation.z(), orientation.w() }) + { + if (!std::isfinite(component)) + { + throw std::runtime_error("TactileMetadata." + std::string(field) + "[" + std::to_string(i) + + "]: poses must be finite"); + } + } + const float norm2 = orientation.x() * orientation.x() + orientation.y() * orientation.y() + + orientation.z() * orientation.z() + orientation.w() * orientation.w(); + if (std::abs(norm2 - 1.0f) > 1e-4f) + { + throw std::runtime_error("TactileMetadata." + std::string(field) + "[" + std::to_string(i) + + "]: orientations must be unit quaternions (x, y, z, w)"); + } + } + }; + verify_poses(lbs->segment_bind_poses(), "segment_bind_poses"); + verify_poses(lbs->taxel_bind_poses(), "taxel_bind_poses"); } } // namespace core diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index dd8cc8ae59..5fac08e6be 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include #include @@ -9,6 +10,7 @@ #include #include +#include #include #include #include @@ -226,3 +228,81 @@ TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", " Catch::Matchers::ContainsSubstring("TactileMetadata." + reason)); } } + +TEST_CASE("TactileMetadata semantic verification rejects nonfinite bind poses", "[unit][tactile_metadata]") +{ + const bool segment = GENERATE(true, false); + const auto component = GENERATE(0, 1, 2, 3, 4, 5, 6); + const auto value = GENERATE(std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity(), + -std::numeric_limits::infinity()); + const std::string field = segment ? "segment_bind_poses" : "taxel_bind_poses"; + CAPTURE(field, component, value); + auto native = make_metadata(); + auto& lbs = *native.geometry_mapping.AsLBS(); + auto& poses = segment ? lbs.segment_bind_poses : lbs.taxel_bind_poses; + std::array row = { 0, 0, 0, 0, 0, 0, 1 }; + row[component] = value; + poses.back() = core::Pose(core::Point(row[0], row[1], row[2]), core::Quaternion(row[3], row[4], row[5], row[6])); + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + CHECK_THROWS_WITH( + core::verify_metadata(*metadata), + Catch::Matchers::ContainsSubstring(field + "[" + std::to_string(poses.size() - 1) + "]: poses must be finite")); +} + +TEST_CASE("TactileMetadata semantic verification checks bind quaternion norms", "[unit][tactile_metadata]") +{ + const bool segment = GENERATE(true, false); + const std::string field = segment ? "segment_bind_poses" : "taxel_bind_poses"; + CAPTURE(field); + auto native = make_metadata(); + auto& lbs = *native.geometry_mapping.AsLBS(); + auto& poses = segment ? lbs.segment_bind_poses : lbs.taxel_bind_poses; + core::Quaternion orientation; + bool valid = false; + SECTION("zero quaternion") + { + } + SECTION("scaled quaternion") + { + orientation = core::Quaternion(0, 0, 0, 2); + } + SECTION("finite components with overflowing squared norm") + { + orientation = core::Quaternion(std::numeric_limits::max(), 0, 0, 0); + } + SECTION("rotation with all components nonzero") + { + orientation = core::Quaternion(0.5f, 0.5f, 0.5f, 0.5f); + valid = true; + } + SECTION("negative unit quaternion") + { + orientation = core::Quaternion(0, 0, 0, -1); + valid = true; + } + SECTION("rounding within squared-norm tolerance") + { + orientation = core::Quaternion(0, 0, 0, std::sqrt(1.0f + 0.5e-4f)); + valid = true; + } + SECTION("rounding outside squared-norm tolerance") + { + orientation = core::Quaternion(0, 0, 0, std::sqrt(1.0f + 2e-4f)); + } + poses.back() = core::Pose(poses.back().position(), orientation); + const auto metadata = core::pack(native); + flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); + REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); + if (valid) + { + CHECK_NOTHROW(core::verify_metadata(*metadata)); + } + else + { + CHECK_THROWS_WITH(core::verify_metadata(*metadata), + Catch::Matchers::ContainsSubstring(field + "[" + std::to_string(poses.size() - 1) + + "]: orientations must be unit quaternions")); + } +} diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py index 991ca31f3f..9d67330951 100644 --- a/tests/python/core/schema/test_tactile_metadata.py +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -103,16 +103,37 @@ def test_rejects_invalid_weights(weight): @pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) -def test_rejects_nonunit_quaternions(field): +@pytest.mark.parametrize( + "qw", [0, 2, np.finfo(np.float32).max, np.sqrt(np.float32(1 + 2e-4))] +) +def test_rejects_nonunit_quaternions(field, qw): arrays = lbs_arrays() - arrays[field][0, 6] = 0 + arrays[field][-1, 6] = qw with pytest.raises(ValueError, match="unit quaternions"): LBS(**arrays) -def test_rejects_nonfinite_bind_positions(): +@pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) +@pytest.mark.parametrize( + "quaternion", + [ + [0.5, 0.5, 0.5, 0.5], + [0, 0, 0, -1], + [0, 0, 0, np.sqrt(np.float32(1 + 0.5e-4))], + ], +) +def test_accepts_unit_quaternions_with_rounding_tolerance(field, quaternion): + arrays = lbs_arrays() + arrays[field][-1, 3:] = quaternion + LBS(**arrays) + + +@pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) +@pytest.mark.parametrize("component", range(7)) +@pytest.mark.parametrize("value", [np.nan, np.inf, -np.inf]) +def test_rejects_nonfinite_bind_poses(field, component, value): arrays = lbs_arrays() - arrays["segment_bind_poses"][0, 0] = np.nan + arrays[field][-1, component] = value with pytest.raises(ValueError, match="finite"): LBS(**arrays) From f63878a1f27aa50334b5ce6f5e8900f550e88dce Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 00:44:36 -0700 Subject: [PATCH 28/45] Extract UInt4 and Float4 into standalone schemas Signed-off-by: Lotus Li --- src/core/schema/fbs/float4.fbs | 8 ++++++++ src/core/schema/fbs/tactile_metadata.fbs | 10 ++-------- src/core/schema/fbs/uint4.fbs | 8 ++++++++ src/core/schema/golden/float4.bfbs | Bin 0 -> 304 bytes src/core/schema/golden/uint4.bfbs | Bin 0 -> 304 bytes 5 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 src/core/schema/fbs/float4.fbs create mode 100644 src/core/schema/fbs/uint4.fbs create mode 100644 src/core/schema/golden/float4.bfbs create mode 100644 src/core/schema/golden/uint4.bfbs diff --git a/src/core/schema/fbs/float4.fbs b/src/core/schema/fbs/float4.fbs new file mode 100644 index 0000000000..695ac5a883 --- /dev/null +++ b/src/core/schema/fbs/float4.fbs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace core; + +struct Float4 { + values: [float:4]; +} diff --git a/src/core/schema/fbs/tactile_metadata.fbs b/src/core/schema/fbs/tactile_metadata.fbs index 380ffe6c7c..25f5ac9ff4 100644 --- a/src/core/schema/fbs/tactile_metadata.fbs +++ b/src/core/schema/fbs/tactile_metadata.fbs @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +include "float4.fbs"; include "hand.fbs"; include "pose.fbs"; +include "uint4.fbs"; namespace core; @@ -21,14 +23,6 @@ union SegmentSource { HandSkeletonRef = 1, } -struct UInt4 { - values: [uint32:4]; -} - -struct Float4 { - values: [float:4]; -} - table LBS { // Segment-local-to-skeleton-root bind poses, in SegmentSource order. // For HandSkeletonRef the root is the WRIST frame at bind, not a joint's diff --git a/src/core/schema/fbs/uint4.fbs b/src/core/schema/fbs/uint4.fbs new file mode 100644 index 0000000000..e752b0703e --- /dev/null +++ b/src/core/schema/fbs/uint4.fbs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace core; + +struct UInt4 { + values: [uint32:4]; +} diff --git a/src/core/schema/golden/float4.bfbs b/src/core/schema/golden/float4.bfbs new file mode 100644 index 0000000000000000000000000000000000000000..f03d7b8aa7273164b9237a686b53e426e396ebe1 GIT binary patch literal 304 zcmaKny$*vg5Q7~lEsx~rZMIm@$Aauo>$s8m@nm@O6@Nc-I7h2!FC&Q8WV3> PGoM(Vl=ta2bUp9^v4ju9 literal 0 HcmV?d00001 From da9b523dec0d3f071da5a54c603e89e646502d47 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 00:54:12 -0700 Subject: [PATCH 29/45] Share tactile metadata validation with Python constructors Signed-off-by: Lotus Li --- .../schema/tactile_metadata_validation.hpp | 2 +- .../schema/python/tactile_metadata_bindings.h | 91 ++++--------------- .../core/schema/test_tactile_metadata.py | 68 +++++++++++--- 3 files changed, 73 insertions(+), 88 deletions(-) diff --git a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp index 83ce528c03..64dde27a6a 100644 --- a/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp +++ b/src/core/schema/cpp/inc/schema/tactile_metadata_validation.hpp @@ -15,7 +15,7 @@ namespace core //! Checks the semantics of the supported LBS/HandSkeletonRef layout after FlatBuffers verification. //! Weight sums may differ from one by at most 1e-5 to allow floating-point rounding. -//! Squared quaternion norms use the Python constructor's tolerance of 1e-4. +//! Squared quaternion norms may differ from one by at most 1e-4. //! Throws std::runtime_error naming the invalid field and the violated constraint. inline void verify_tactile_metadata(const TactileMetadata& metadata) { diff --git a/src/core/schema/python/tactile_metadata_bindings.h b/src/core/schema/python/tactile_metadata_bindings.h index be21c3b752..945158955c 100644 --- a/src/core/schema/python/tactile_metadata_bindings.h +++ b/src/core/schema/python/tactile_metadata_bindings.h @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // Encoded tactile metadata views. Pose arrays use rows (x, y, z, qx, qy, qz, qw). +// Complete-layout semantics are validated by TactileMetadata using the shared schema validator. #pragma once @@ -12,12 +13,11 @@ #include #include #include +#include -#include -#include #include #include -#include +#include #include #include @@ -41,16 +41,6 @@ inline void bind_tactile_metadata(py::module& m) .def(py::init( [](const std::vector& joint_ids) { - std::array seen{}; - for (const auto joint : joint_ids) - { - if (joint >= HandJoint_NUM_JOINTS || seen[joint]) - { - throw py::value_error( - "joint_ids: expected unique HandJoint identifiers, excluding NUM_JOINTS"); - } - seen[joint] = true; - } HandSkeletonRefT native; native.joint_ids = joint_ids; return pack(native); @@ -69,66 +59,19 @@ inline void bind_tactile_metadata(py::module& m) return joint_ids; }); - auto poses_from_rows = [](const FloatRows& rows, const char* name) - { - auto poses = structs_from_rows(rows, name); - for (py::ssize_t i = 0; i < rows.shape(0); ++i) - { - const float* row = rows.data(i); - for (int c = 0; c < 7; ++c) - { - if (!std::isfinite(row[c])) - { - throw py::value_error(std::string(name) + ": poses must be finite"); - } - } - const float norm2 = row[3] * row[3] + row[4] * row[4] + row[5] * row[5] + row[6] * row[6]; - if (std::abs(norm2 - 1.0f) > 1e-4f) - { - throw py::value_error(std::string(name) + ": orientations must be unit quaternions (x, y, z, w)"); - } - } - return poses; - }; - serialized_class(m, "LBS", "Linear blend skinning bind poses and four-slot influences.") .def(py::init( - [poses_from_rows](const FloatRows& segment_bind_poses, const FloatRows& taxel_bind_poses, - const IndexRows& taxel_segment_indices, const FloatRows& taxel_segment_weights) + [](const FloatRows& segment_bind_poses, const FloatRows& taxel_bind_poses, + const IndexRows& taxel_segment_indices, const FloatRows& taxel_segment_weights) { LBST native; - native.segment_bind_poses = poses_from_rows(segment_bind_poses, "segment_bind_poses"); - native.taxel_bind_poses = poses_from_rows(taxel_bind_poses, "taxel_bind_poses"); + native.segment_bind_poses = + structs_from_rows(segment_bind_poses, "segment_bind_poses"); + native.taxel_bind_poses = structs_from_rows(taxel_bind_poses, "taxel_bind_poses"); native.taxel_segment_indices = structs_from_rows(taxel_segment_indices, "taxel_segment_indices"); native.taxel_segment_weights = structs_from_rows(taxel_segment_weights, "taxel_segment_weights"); - const size_t taxels = native.taxel_bind_poses.size(); - if (native.taxel_segment_indices.size() != taxels || native.taxel_segment_weights.size() != taxels) - { - throw py::value_error("all taxel_* arrays must have the same length"); - } - for (size_t i = 0; i < taxels; ++i) - { - float total = 0; - for (int k = 0; k < 4; ++k) - { - if (native.taxel_segment_indices[i].values()->Get(k) >= native.segment_bind_poses.size()) - { - throw py::value_error("taxel_segment_indices: slot is not a segment index"); - } - const float weight = native.taxel_segment_weights[i].values()->Get(k); - if (!std::isfinite(weight) || weight < 0) - { - throw py::value_error("taxel_segment_weights: weights must be finite and nonnegative"); - } - total += weight; - } - if (std::abs(total - 1.0f) > 1e-5f) - { - throw py::value_error("taxel_segment_weights: each row must sum to one"); - } - } return pack(native); }), py::arg("segment_bind_poses"), py::arg("taxel_bind_poses"), py::arg("taxel_segment_indices"), @@ -153,13 +96,6 @@ inline void bind_tactile_metadata(py::module& m) .def(py::init( [](const Serialized& geometry_mapping, const Serialized& segment_source) { - const auto* poses = geometry_mapping->segment_bind_poses(); - const auto* joint_ids = segment_source->joint_ids(); - if ((poses != nullptr ? poses->size() : 0) != (joint_ids != nullptr ? joint_ids->size() : 0)) - { - throw py::value_error( - "segment_bind_poses and segment_source.joint_ids must have the same length"); - } TactileMetadataT native; LBST lbs; geometry_mapping->UnPackTo(&lbs); @@ -167,7 +103,16 @@ inline void bind_tactile_metadata(py::module& m) HandSkeletonRefT hand; segment_source->UnPackTo(&hand); native.segment_source.Set(std::move(hand)); - return pack(native); + auto metadata = pack(native); + try + { + verify_tactile_metadata(*metadata); + } + catch (const std::runtime_error& error) + { + throw py::value_error(error.what()); + } + return metadata; }), py::arg("geometry_mapping"), py::arg("segment_source")) .def_property_readonly("geometry_mapping_type", field(&TactileMetadata::geometry_mapping_type)) diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py index 9d67330951..2f150a154e 100644 --- a/tests/python/core/schema/test_tactile_metadata.py +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -75,31 +75,52 @@ def test_nested_views_and_arrays_retain_the_buffer(): assert encoded -def test_rejects_mismatched_taxel_lengths(): +def test_metadata_revalidates_existing_geometry_mapping(): + geometry_mapping = LBS(**lbs_arrays()) + geometry_mapping.taxel_segment_weights[0] = 0 + with pytest.raises(ValueError, match="taxel_segment_weights"): + TactileMetadata( + geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) + + +@pytest.mark.parametrize( + "field", ["taxel_bind_poses", "taxel_segment_indices", "taxel_segment_weights"] +) +def test_rejects_mismatched_taxel_lengths(field): arrays = lbs_arrays() - arrays["taxel_bind_poses"] = arrays["taxel_bind_poses"][:2] - with pytest.raises(ValueError, match="same length"): - LBS(**arrays) + arrays[field] = arrays[field][:2] + geometry_mapping = LBS(**arrays) + with pytest.raises(ValueError, match=r"taxel_segment_(indices|weights): expected"): + TactileMetadata( + geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) def test_rejects_mismatched_segment_mapping(): - with pytest.raises(ValueError, match="same length"): + with pytest.raises(ValueError, match=r"segment_bind_poses.*joint_ids"): TactileMetadata(LBS(**lbs_arrays()), HandSkeletonRef([HandJoint.WRIST])) def test_rejects_out_of_range_zero_weight_slot(): arrays = lbs_arrays() arrays["taxel_segment_indices"][0, 3] = 2 + geometry_mapping = LBS(**arrays) with pytest.raises(ValueError, match="taxel_segment_indices"): - LBS(**arrays) + TactileMetadata( + geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) @pytest.mark.parametrize("weight", [-1.0, np.inf, np.nan, 0.5]) def test_rejects_invalid_weights(weight): arrays = lbs_arrays() arrays["taxel_segment_weights"][0, 0] = weight + geometry_mapping = LBS(**arrays) with pytest.raises(ValueError, match="taxel_segment_weights"): - LBS(**arrays) + TactileMetadata( + geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) @pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) @@ -109,8 +130,11 @@ def test_rejects_invalid_weights(weight): def test_rejects_nonunit_quaternions(field, qw): arrays = lbs_arrays() arrays[field][-1, 6] = qw + geometry_mapping = LBS(**arrays) with pytest.raises(ValueError, match="unit quaternions"): - LBS(**arrays) + TactileMetadata( + geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) @pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) @@ -125,7 +149,9 @@ def test_rejects_nonunit_quaternions(field, qw): def test_accepts_unit_quaternions_with_rounding_tolerance(field, quaternion): arrays = lbs_arrays() arrays[field][-1, 3:] = quaternion - LBS(**arrays) + TactileMetadata( + LBS(**arrays), HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) @pytest.mark.parametrize("field", ["segment_bind_poses", "taxel_bind_poses"]) @@ -134,14 +160,26 @@ def test_accepts_unit_quaternions_with_rounding_tolerance(field, quaternion): def test_rejects_nonfinite_bind_poses(field, component, value): arrays = lbs_arrays() arrays[field][-1, component] = value + geometry_mapping = LBS(**arrays) with pytest.raises(ValueError, match="finite"): - LBS(**arrays) + TactileMetadata( + geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) + ) -def test_rejects_wrong_influence_width(): +@pytest.mark.parametrize( + "field", + [ + "segment_bind_poses", + "taxel_bind_poses", + "taxel_segment_indices", + "taxel_segment_weights", + ], +) +def test_lbs_rejects_wrong_array_width(field): arrays = lbs_arrays() - arrays["taxel_segment_weights"] = np.zeros((3, 3), np.float32) - with pytest.raises(ValueError, match="taxel_segment_weights"): + arrays[field] = arrays[field][:, :-1] + with pytest.raises(ValueError, match=field): LBS(**arrays) @@ -149,5 +187,7 @@ def test_rejects_wrong_influence_width(): "joint_ids", [[HandJoint.WRIST, HandJoint.WRIST], [HandJoint.NUM_JOINTS]] ) def test_rejects_invalid_joint_mapping(joint_ids): + segment_source = HandSkeletonRef(joint_ids) + geometry_mapping = LBS(**lbs_arrays(num_joints=len(joint_ids))) with pytest.raises(ValueError, match="joint_ids"): - HandSkeletonRef(joint_ids) + TactileMetadata(geometry_mapping, segment_source) From 9a967758cf34e01fed24c6b490a36e5c17bd9fed Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 01:44:40 -0700 Subject: [PATCH 30/45] Require tactile forces and record payloads Declare tactile force arrays and record payloads required, reject missing Python constructor inputs, and test missing-field rejection and explicit empty-array support. Refresh the tactile schema golden. Replay now rejects older tactile recordings whose embedded schemas declare either field optional. Signed-off-by: Lotus Li --- src/core/schema/fbs/tactile_array.fbs | 4 +-- src/core/schema/golden/tactile_array.bfbs | Bin 1124 -> 1120 bytes src/core/schema/python/schema_serialized.h | 22 ++++++++++------ .../schema/python/tactile_array_bindings.h | 13 ++++------ tests/cpp/core/schema/test_tactile_array.cpp | 24 ++++++++++++++++-- .../python/core/schema/test_tactile_array.py | 17 +++++++++---- 6 files changed, 55 insertions(+), 25 deletions(-) diff --git a/src/core/schema/fbs/tactile_array.fbs b/src/core/schema/fbs/tactile_array.fbs index 358548cabb..9ff745e150 100644 --- a/src/core/schema/fbs/tactile_array.fbs +++ b/src/core/schema/fbs/tactile_array.fbs @@ -13,12 +13,12 @@ table TactileDataArray { // define these axes at bind; the vector length equals the metadata taxel count. // +X is the calibrated tangent T, +Z is the outward normal N, +Y = N cross T. // Component signs follow these axes for both hands, without a side-based flip. - taxel_forces: [Point] (id: 0); + taxel_forces: [Point] (id: 0, required); } // MCAP recording wrapper for TactileDataArray. table TactileDataArrayRecord { - data: TactileDataArray (id: 0); + data: TactileDataArray (id: 0, required); timestamp: DeviceDataTimestamp (id: 1); } diff --git a/src/core/schema/golden/tactile_array.bfbs b/src/core/schema/golden/tactile_array.bfbs index 636b6599ca561ff30b19fa59f25f2011c63760cd..22d12e1a699cdc6e4491967cc40e9da551e258cb 100644 GIT binary patch delta 122 zcmaFD@qlAO1nUDv1_qspvC_H$fByf!0pzeSKtKYJF9GE<0fl&&85ni|`4vEMhK)<5 z85vC`A7qpl6=4uy;9%eZs$hfaWMvSU{FhN*7$jDbSdp3&pO#;goLUUzZnkF9WC8$z C#vES& delta 126 zcmaFB@q}YS1nUz<1_pzPvC_IefByf!1LUwUKtK$TF9GE<0fhvZ85s5e`6WPcmW@lL z85wORA7qr5m0=KI;9%eZs$hfagi)*vGLyeD>hp601xgYtQgh> to_native_vector(c //! Binds a `Record` wrapper (an MCAP payload: data plus its capture timestamp). template -void bind_record(py::module& m, const char* name, const char* data_name) +void bind_record(py::module& m, const char* name, const char* data_name, bool data_required = false) { serialized_class(m, name, "Encoded MCAP record: a payload plus the timestamp it was captured at.") .def(py::init( - [](const Serialized* data, const DeviceDataTimestamp& timestamp) + [data_required](const Serialized* data, const DeviceDataTimestamp& timestamp) { + // Reject absent required data before the generated Pack asserts on it. + if (data_required && (data == nullptr || !*data)) + { + throw py::value_error("data: a payload is required"); + } typename RecordT::NativeTableType native; if (data != nullptr && *data) { @@ -171,13 +176,14 @@ void bind_record(py::module& m, const char* name, const char* data_name) native.timestamp = std::make_shared(timestamp); return pack(native); }), - py::arg("data").none(true), py::arg("timestamp"), - "Encode a record from a payload and its timestamp. `data` may be None: MCAP " - "carries payload-less records, such as the message channel's frame sentinel.") + py::arg("data").none(!data_required), py::arg("timestamp"), + data_required ? "Encode a record from a required payload and its timestamp." : + "Encode a record from a payload and its timestamp. `data` may be None: MCAP " + "carries payload-less records, such as the message channel's frame sentinel.") // No no-arg form: `timestamp` is a struct with no meaningful default, and a record - // without one is not a thing MCAP stores. A payload-less record -- the message - // channel's frame sentinel -- is spelled `Record(None, timestamp)`. + // without one is not a thing MCAP stores. For optional data fields, a payload-less + // record -- the message channel's frame sentinel -- is spelled `Record(None, timestamp)`. .def_property_readonly( "data", [](const Serialized& self) -> py::object @@ -185,7 +191,7 @@ void bind_record(py::module& m, const char* name, const char* data_name) const DataT* data = self->data(); return data != nullptr ? py::cast(self.narrow(data)) : py::none(); }, - "The recorded payload, or None when absent.") + data_required ? "The recorded payload." : "The recorded payload, or None when absent.") .def_property_readonly( "timestamp", [](const Serialized& self) { return self->timestamp(); }, py::return_value_policy::reference_internal, "Capture timestamp, or None when absent.") diff --git a/src/core/schema/python/tactile_array_bindings.h b/src/core/schema/python/tactile_array_bindings.h index 547db29d42..a3f26a94bd 100644 --- a/src/core/schema/python/tactile_array_bindings.h +++ b/src/core/schema/python/tactile_array_bindings.h @@ -14,7 +14,6 @@ #include #include -#include #include namespace py = pybind11; @@ -28,16 +27,13 @@ inline void bind_tactile_array(py::module& m) serialized_class(m, "TactileDataArray", "Encoded per-frame tactile reading.") .def(py::init( - [](const std::optional& taxel_forces) + [](const FloatRows& taxel_forces) { TactileDataArrayT native; - if (taxel_forces.has_value()) - { - native.taxel_forces = structs_from_rows(*taxel_forces, "taxel_forces"); - } + native.taxel_forces = structs_from_rows(taxel_forces, "taxel_forces"); return pack(native); }), - py::arg("taxel_forces") = py::none(), "Encode a tactile frame. taxel_forces is an (N, 3) array.") + py::arg("taxel_forces").none(false), "Encode a tactile frame. taxel_forces is an (N, 3) array.") .def_property_readonly( "taxel_forces", [](py::object self) @@ -53,7 +49,8 @@ inline void bind_tactile_array(py::module& m) return "TactileDataArray(taxels=" + std::to_string(forces != nullptr ? forces->size() : 0) + ")"; }); - bind_record(m, "TactileDataArrayRecord", "TactileDataArray"); + bind_record( + m, "TactileDataArrayRecord", "TactileDataArray", /* data_required = */ true); } } // namespace core diff --git a/tests/cpp/core/schema/test_tactile_array.cpp b/tests/cpp/core/schema/test_tactile_array.cpp index 7c2e4cdf32..c480182fd5 100644 --- a/tests/cpp/core/schema/test_tactile_array.cpp +++ b/tests/cpp/core/schema/test_tactile_array.cpp @@ -46,12 +46,32 @@ TEST_CASE("TactileDataArray round-trips every field", "[unit][tactile]") CHECK(data->taxel_forces()->Get(1)->z() == 2.0f); } -TEST_CASE("TactileDataArray leaves an empty frame's vector absent", "[unit][tactile]") +TEST_CASE("TactileDataArray keeps its required vector present for an empty frame", "[unit][tactile]") { const auto data = core::pack(core::TactileDataArrayT{}); REQUIRE(data); - CHECK(data->taxel_forces() == nullptr); + REQUIRE(data->taxel_forces() != nullptr); + CHECK(data->taxel_forces()->size() == 0); + flatbuffers::Verifier verifier(data.buffer().data(), data.buffer().size()); + CHECK(verifier.VerifyBuffer()); +} + +TEST_CASE("Tactile verification rejects missing required fields", "[unit][tactile]") +{ + flatbuffers::FlatBufferBuilder builder; + // Build an unchecked empty table to test verification without required-field builder assertions. + const auto start = builder.StartTable(); + builder.Finish(flatbuffers::Offset(builder.EndTable(start))); + flatbuffers::Verifier verifier(builder.GetBufferPointer(), builder.GetSize()); + SECTION("taxel_forces") + { + CHECK_FALSE(verifier.VerifyBuffer()); + } + SECTION("record data") + { + CHECK_FALSE(core::VerifyTactileDataArrayRecordBuffer(verifier)); + } } TEST_CASE("TactileDataArrayRecord verifies and carries its timestamp", "[unit][tactile]") diff --git a/tests/python/core/schema/test_tactile_array.py b/tests/python/core/schema/test_tactile_array.py index bc57ea775f..112badc6a5 100644 --- a/tests/python/core/schema/test_tactile_array.py +++ b/tests/python/core/schema/test_tactile_array.py @@ -18,9 +18,16 @@ class TestTactileDataArray: - def test_default_construction(self): - data = TactileDataArray() + @pytest.mark.parametrize("args", [(), (None,)]) + def test_requires_force_array(self, args): + with pytest.raises(TypeError, match="taxel_forces"): + TactileDataArray(*args) + + def test_explicit_empty_force_array(self): + data = TactileDataArray(np.empty((0, 3), np.float32)) assert data.taxel_forces.shape == (0, 3) + record = TactileDataArrayRecord(data, DeviceDataTimestamp(1, 2, 3)) + assert record.data.taxel_forces.shape == (0, 3) def test_round_trip(self): data = TactileDataArray(FORCES) @@ -52,6 +59,6 @@ def test_record_carries_payload_and_timestamp(self): np.testing.assert_array_equal(record.data.taxel_forces, FORCES) assert record.timestamp.sample_time_raw_device_clock == 3 - def test_payload_less_record(self): - record = TactileDataArrayRecord(None, DeviceDataTimestamp(1, 2, 3)) - assert record.data is None + def test_rejects_missing_payload(self): + with pytest.raises(TypeError, match="data"): + TactileDataArrayRecord(None, DeviceDataTimestamp(1, 2, 3)) From a498d6ec0cb235022bc6ba2edb2f37dd2552730c Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 01:54:18 -0700 Subject: [PATCH 31/45] Cache tactile layouts by metadata buffer Build independent LBS layout arrays on a metadata buffer change and reuse them across frames. Keep a bind-pose view alive to recognize shared storage across Python wrappers and prevent allocation-address reuse. Cover array reuse with fresh output groups, per-frame force updates, new metadata buffers, count validation, absence and recovery, and DLPack export. Signed-off-by: Lotus Li --- .../deviceio_source_nodes/tactile_source.py | 59 ++++++++++++------- .../test_tactile_sources.py | 56 ++++++++++++++++++ 2 files changed, 93 insertions(+), 22 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index f2a6d8fb5c..95a9176c50 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -81,6 +81,7 @@ class TactileSource(IDeviceIOSource): until the tracker holds the device's metadata. The tracker publishes no forces before then. A frame or metadata whose counts differ from the constructor's raises ``ValueError``. + Layout arrays are reused across frames; copy them before modifying them. """ FORCES = "taxel_forces" @@ -112,6 +113,8 @@ def __init__( ) self._num_segments = int(num_segments) self._num_taxels = int(num_taxels) + self._layout: tuple[np.ndarray, ...] | None = None + self._layout_bind_poses: np.ndarray | None = None super().__init__(name) def get_tracker(self) -> "ITracker": @@ -139,7 +142,7 @@ def output_spec(self) -> RetargeterIOType: } def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: - """Publish forces and a copied LBS layout; mark absent outputs with ``set_none()``.""" + """Publish forces and a cached LBS layout; mark absent outputs with ``set_none()``.""" data: "TactileDataArray | None" = inputs["deviceio_tactile"][0] metadata: "TactileMetadata | None" = inputs["deviceio_tactile"][1] @@ -164,26 +167,38 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N raise ValueError( f"{self.name}: metadata must specify geometry_mapping and segment_source" ) - num_segments, num_taxels = ( - len(segment_source.joint_ids), - len(geometry_mapping.taxel_bind_poses), - ) - if (num_segments, num_taxels) != (self._num_segments, self._num_taxels): - raise ValueError( - f"{self.name}: metadata has {num_segments} segments and {num_taxels} taxels, " - f"source expects {self._num_segments} and {self._num_taxels}" + # Normalize the schema stub's array type without copying its storage. + segment_bind_poses = np.asarray(geometry_mapping.segment_bind_poses) + if ( + self._layout is None + or self._layout_bind_poses is None + or not np.shares_memory(segment_bind_poses, self._layout_bind_poses) + ): + num_segments, num_taxels = ( + len(segment_source.joint_ids), + len(geometry_mapping.taxel_bind_poses), ) - values = ( - np.asarray(segment_source.joint_ids, dtype=np.int32), - geometry_mapping.segment_bind_poses, - geometry_mapping.taxel_bind_poses, - geometry_mapping.taxel_segment_indices, - geometry_mapping.taxel_segment_weights, - ) - for index, value in enumerate(values): - # Schema stubs have mixed array types; normalize before NumPy operations. - array = np.asarray(value) - # Copied: the views alias the metadata the tracker holds and compares later pushes to. - layout_out[index] = array.astype( - np.int32 if array.dtype == np.uint32 else array.dtype + if (num_segments, num_taxels) != (self._num_segments, self._num_taxels): + raise ValueError( + f"{self.name}: metadata has {num_segments} segments and {num_taxels} taxels, " + f"source expects {self._num_segments} and {self._num_taxels}" + ) + values = ( + np.asarray(segment_source.joint_ids, dtype=np.int32), + segment_bind_poses, + geometry_mapping.taxel_bind_poses, + geometry_mapping.taxel_segment_indices, + geometry_mapping.taxel_segment_weights, ) + arrays = [] + for value in values: + # Copy once: graph consumers must not alter the tracker's metadata baseline. + array = np.asarray(value) + arrays.append( + array.astype(np.int32 if array.dtype == np.uint32 else array.dtype) + ) + self._layout = tuple(arrays) + # Views keep the buffer alive and identify it across fresh Python wrappers. + self._layout_bind_poses = segment_bind_poses + for index, array in enumerate(self._layout): + layout_out[index] = array diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 6ba468218a..b81a26f7e3 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -165,3 +165,59 @@ def test_metadata_count_mismatch_raises(self): src = _source(num_segments=NUM_SEGMENTS + 1) with pytest.raises(ValueError, match="2 segments"): src.compute(_inputs(src, None, _metadata()), _outputs(src)) + + def test_layout_arrays_are_reused_while_forces_update(self): + src = _source() + metadata = _metadata() + first = src( + _inputs( + src, TactileDataArray(np.zeros((NUM_TAXELS, 3), np.float32)), metadata + ) + ) + second = src( + _inputs( + src, TactileDataArray(np.ones((NUM_TAXELS, 3), np.float32)), metadata + ) + ) + + assert first[TactileSource.LAYOUT] is not second[TactileSource.LAYOUT] + for index in TactileLBSLayoutIndex: + array = first[TactileSource.LAYOUT][index] + assert second[TactileSource.LAYOUT][index] is array + np.testing.assert_array_equal(np.from_dlpack(array), array) + np.testing.assert_array_equal(first[TactileSource.FORCES][0], 0) + np.testing.assert_array_equal(second[TactileSource.FORCES][0], 1) + + def test_layout_cache_follows_the_metadata_buffer(self): + src = _source() + metadata = _metadata() + first = src(_inputs(src, None, metadata)) + replacement = _metadata() + replacement.geometry_mapping.taxel_bind_poses[:, 0] = 0.02 + second = src(_inputs(src, None, replacement)) + + for index in TactileLBSLayoutIndex: + assert ( + second[TactileSource.LAYOUT][index] + is not first[TactileSource.LAYOUT][index] + ) + taxel_poses = TactileLBSLayoutIndex.TAXEL_BIND_POSES + np.testing.assert_array_equal( + first[TactileSource.LAYOUT][taxel_poses][:, 0], np.float32(0.01) + ) + np.testing.assert_array_equal( + second[TactileSource.LAYOUT][taxel_poses][:, 0], np.float32(0.02) + ) + + with pytest.raises(ValueError, match="4 taxels"): + src(_inputs(src, None, _metadata(num_taxels=NUM_TAXELS + 1))) + + outputs = _outputs(src) + src.compute(_inputs(src, None, None), outputs) + assert outputs[TactileSource.LAYOUT].is_none + src.compute(_inputs(src, None, replacement), outputs) + for index in TactileLBSLayoutIndex: + assert ( + outputs[TactileSource.LAYOUT][index] + is second[TactileSource.LAYOUT][index] + ) From 6246731570d90d759ae81f8684c40b78991d9918 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 01:58:43 -0700 Subject: [PATCH 32/45] Apply reviews to tests Signed-off-by: Lotus Li --- .../test_tactile_sources.py | 26 +------------------ .../core/schema/test_tactile_metadata.py | 2 -- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index b81a26f7e3..575ee844fc 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -4,6 +4,7 @@ """Tests for the TactileSource DeviceIO converter. Payloads are built through the real schema bindings; no OpenXR device is involved. +Session-backed tracker reads are covered in tests/cpp/core/replay_deviceio_session. """ import numpy as np @@ -64,20 +65,6 @@ def _metadata( ) -class _FakeTracker: - """Stands in for a TactileTracker: fixed data and metadata.""" - - def __init__(self, data, metadata): - self._data = data - self._metadata = metadata - - def get_data(self, _session): - return self._data - - def get_metadata(self, _session): - return self._metadata - - class TestTactileSource: def test_creation_and_tracker(self): assert _source().get_tracker().get_name() == "TactileTracker" @@ -94,17 +81,6 @@ def test_index_matches_layout_order(self): names = [t.name for t in TactileLBSLayout(NUM_SEGMENTS, NUM_TAXELS).types] assert names == [member.name.lower() for member in TactileLBSLayoutIndex] - def test_poll_reads_data_and_metadata_from_one_tracker(self): - src = _source() - data = TactileDataArray(np.zeros((NUM_TAXELS, 3), np.float32)) - metadata = _metadata() - src._tracker = _FakeTracker(data, metadata) - - polled = src.poll_tracker(None)["deviceio_tactile"] - - assert polled[0] is data - assert polled[1] is metadata - def test_active_conversion(self): src = _source() forces = np.arange(9, dtype=np.float32).reshape(3, 3) diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py index 2f150a154e..ef0ed38836 100644 --- a/tests/python/core/schema/test_tactile_metadata.py +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -67,12 +67,10 @@ def test_nested_views_and_arrays_retain_the_buffer(): geometry_mapping = metadata.geometry_mapping segment_source = metadata.segment_source rows = geometry_mapping.taxel_bind_poses - encoded = geometry_mapping.to_bytes() del metadata, geometry_mapping gc.collect() assert segment_source.joint_ids == [HandJoint.WRIST, HandJoint.INDEX_TIP] np.testing.assert_array_equal(rows, lbs_arrays()["taxel_bind_poses"]) - assert encoded def test_metadata_revalidates_existing_geometry_mapping(): From a25e2289b48257f2133f95c929b9346117970e35 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 02:02:13 -0700 Subject: [PATCH 33/45] Move tactile group types into tensor_types Signed-off-by: Lotus Li --- .../deviceio_source_nodes/__init__.py | 4 -- .../deviceio_source_nodes/tactile_source.py | 38 +------------------ .../tensor_types/__init__.py | 4 ++ .../tensor_types/tactile_types.py | 36 +++++++++++++++++- .../test_tactile_sources.py | 2 +- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index 6009c94f32..d0b472077a 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -43,9 +43,7 @@ ) from .haptic_sink import HapticSink from .tactile_source import ( - TactileForces, TactileLBSLayoutIndex, - TactileLBSLayout, TactileSource, ) from .deviceio_tensor_types import ( @@ -121,8 +119,6 @@ "HapticSink", "TactileSource", "TactileLBSLayoutIndex", - "TactileForces", - "TactileLBSLayout", "HeadPoseTrackedType", "HandPoseTrackedType", "ControllerSnapshotTrackedType", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 95a9176c50..a5627c21b4 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -18,8 +18,8 @@ from .interface import IDeviceIOSource from ..interface.retargeter_core_types import RetargeterIO, RetargeterIOType from ..interface.tensor_group import TensorGroup -from ..interface.tensor_group_type import OptionalType, TensorGroupType -from ..tensor_types import DLDataType, NDArrayType +from ..interface.tensor_group_type import OptionalType +from ..tensor_types import TactileForces, TactileLBSLayout from .deviceio_tensor_types import DeviceIOTactileTracked if TYPE_CHECKING: @@ -27,22 +27,6 @@ from isaaccapture.schema import TactileDataArray, TactileMetadata -def _float_array(name: str, rows: int, width: int) -> NDArrayType: - return NDArrayType(name, shape=(rows, width), dtype=DLDataType.FLOAT, dtype_bits=32) - - -def _int_array(name: str, shape: tuple[int, ...]) -> NDArrayType: - return NDArrayType(name, shape=shape, dtype=DLDataType.INT, dtype_bits=32) - - -def TactileForces(num_taxels: int) -> TensorGroupType: - """Per-taxel force vectors: one ``(num_taxels, 3)`` float32 array.""" - return TensorGroupType( - f"tactile_forces_{num_taxels}", - [_float_array("taxel_forces", num_taxels, 3)], - ) - - class TactileLBSLayoutIndex(IntEnum): """Positions of the arrays in a :func:`TactileLBSLayout` group.""" @@ -53,24 +37,6 @@ class TactileLBSLayoutIndex(IntEnum): TAXEL_SEGMENT_WEIGHTS = 4 -def TactileLBSLayout(num_segments: int, num_taxels: int) -> TensorGroupType: - """LBS and HandSkeletonRef arrays, ordered by :class:`TactileLBSLayoutIndex`. - - HandJoint identifiers and segment indices are int32. Pose rows contain - position XYZ followed by quaternion XYZW, all in hand-root space. - """ - return TensorGroupType( - f"tactile_lbs_layout_{num_segments}x{num_taxels}", - [ - _int_array("segment_joint_ids", (num_segments,)), - _float_array("segment_bind_poses", num_segments, 7), - _float_array("taxel_bind_poses", num_taxels, 7), - _int_array("taxel_segment_indices", (num_taxels, 4)), - _float_array("taxel_segment_weights", num_taxels, 4), - ], - ) - - class TactileSource(IDeviceIOSource): """Convert tactile frames and metadata into per-taxel forces and an LBS layout. diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py index 2454c5c44d..ff45b48b8f 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py @@ -24,6 +24,8 @@ RobotHandJoints, ) from .tactile_types import ( + TactileForces, + TactileLBSLayout, TactileVector, TactileHeatmap, FingerPowerVector, @@ -72,6 +74,8 @@ "NUM_SOMA_BODY_JOINTS", "RobotHandJoints", # Tactile / haptic types + "TactileForces", + "TactileLBSLayout", "TactileVector", "TactileHeatmap", "FingerPowerVector", diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/tactile_types.py b/src/python/isaaccapture/retargeting_engine/tensor_types/tactile_types.py index b64da00e5a..d4048f1709 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/tactile_types.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/tactile_types.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""TensorGroupType definitions for tactile feedback and haptic output. +"""TensorGroupType definitions for tactile input, feedback, and haptic output. Sim-side schemas (``TactileVector``, ``TactileHeatmap``) carry contact data into the retargeting pipeline; device-side schemas (``FingerPowerVector``, @@ -31,6 +31,40 @@ """Components in an :func:`EndEffectorForce`: ``[fx, fy, fz]``.""" +def _float_array(name: str, rows: int, width: int) -> NDArrayType: + return NDArrayType(name, shape=(rows, width), dtype=DLDataType.FLOAT, dtype_bits=32) + + +def _int_array(name: str, shape: tuple[int, ...]) -> NDArrayType: + return NDArrayType(name, shape=shape, dtype=DLDataType.INT, dtype_bits=32) + + +def TactileForces(num_taxels: int) -> TensorGroupType: + """Per-taxel force vectors: one ``(num_taxels, 3)`` float32 array.""" + return TensorGroupType( + f"tactile_forces_{num_taxels}", + [_float_array("taxel_forces", num_taxels, 3)], + ) + + +def TactileLBSLayout(num_segments: int, num_taxels: int) -> TensorGroupType: + """LBS and HandSkeletonRef arrays, ordered by :class:`TactileLBSLayoutIndex`. + + HandJoint identifiers and segment indices are int32. Pose rows contain + position XYZ followed by quaternion XYZW, all in hand-root space. + """ + return TensorGroupType( + f"tactile_lbs_layout_{num_segments}x{num_taxels}", + [ + _int_array("segment_joint_ids", (num_segments,)), + _float_array("segment_bind_poses", num_segments, 7), + _float_array("taxel_bind_poses", num_taxels, 7), + _int_array("taxel_segment_indices", (num_taxels, 4)), + _float_array("taxel_segment_weights", num_taxels, 4), + ], + ) + + # ============================================================================ # Sim-side types # ============================================================================ diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 575ee844fc..60deb44015 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -12,9 +12,9 @@ from isaaccapture.retargeting_engine.deviceio_source_nodes import ( TactileLBSLayoutIndex, - TactileLBSLayout, TactileSource, ) +from isaaccapture.retargeting_engine.tensor_types import TactileLBSLayout from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup from isaaccapture.schema import ( From 38e89dbe673cc22380bed64c45551b202582fa63 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 02:03:09 -0700 Subject: [PATCH 34/45] Move tactile layout indices into tensor_types.indices Signed-off-by: Lotus Li --- .../deviceio_source_nodes/__init__.py | 6 +----- .../deviceio_source_nodes/tactile_source.py | 11 ----------- .../retargeting_engine/tensor_types/__init__.py | 2 ++ .../retargeting_engine/tensor_types/indices.py | 10 ++++++++++ .../core/retargeting_engine/test_tactile_sources.py | 6 +++--- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index d0b472077a..968a94c718 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -42,10 +42,7 @@ messageChannelConfig, ) from .haptic_sink import HapticSink -from .tactile_source import ( - TactileLBSLayoutIndex, - TactileSource, -) +from .tactile_source import TactileSource from .deviceio_tensor_types import ( HeadPoseTrackedType, HandPoseTrackedType, @@ -118,7 +115,6 @@ "messageChannelConfig", "HapticSink", "TactileSource", - "TactileLBSLayoutIndex", "HeadPoseTrackedType", "HandPoseTrackedType", "ControllerSnapshotTrackedType", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index a5627c21b4..40a6a6b8c0 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -10,7 +10,6 @@ from __future__ import annotations -from enum import IntEnum from typing import Any, Optional, TYPE_CHECKING import numpy as np @@ -27,16 +26,6 @@ from isaaccapture.schema import TactileDataArray, TactileMetadata -class TactileLBSLayoutIndex(IntEnum): - """Positions of the arrays in a :func:`TactileLBSLayout` group.""" - - SEGMENT_JOINT_IDS = 0 - SEGMENT_BIND_POSES = 1 - TAXEL_BIND_POSES = 2 - TAXEL_SEGMENT_INDICES = 3 - TAXEL_SEGMENT_WEIGHTS = 4 - - class TactileSource(IDeviceIOSource): """Convert tactile frames and metadata into per-taxel forces and an LBS layout. diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py index ff45b48b8f..322e4554b3 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py @@ -44,6 +44,7 @@ Generic3AxisPedalInputIndex, FullBodyInputIndex, SomaBodyInputIndex, + TactileLBSLayoutIndex, HandJointIndex, BodyJointIndex, FingerIndex, @@ -93,6 +94,7 @@ "Generic3AxisPedalInputIndex", "FullBodyInputIndex", "SomaBodyInputIndex", + "TactileLBSLayoutIndex", "HandJointIndex", "BodyJointIndex", "FingerIndex", diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py index 32fa7abcea..26609c93ef 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py @@ -61,6 +61,16 @@ def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: ) +class TactileLBSLayoutIndex(IntEnum): + """Positions of the arrays in a :func:`TactileLBSLayout` group.""" + + SEGMENT_JOINT_IDS = 0 + SEGMENT_BIND_POSES = 1 + TAXEL_BIND_POSES = 2 + TAXEL_SEGMENT_INDICES = 3 + TAXEL_SEGMENT_WEIGHTS = 4 + + class HandJointIndex(IntEnum): """Indices for OpenXR hand joints (XR_HAND_JOINT_COUNT_EXT = 26).""" diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 60deb44015..12acaae131 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -10,11 +10,11 @@ import numpy as np import pytest -from isaaccapture.retargeting_engine.deviceio_source_nodes import ( +from isaaccapture.retargeting_engine.deviceio_source_nodes import TactileSource +from isaaccapture.retargeting_engine.tensor_types import ( + TactileLBSLayout, TactileLBSLayoutIndex, - TactileSource, ) -from isaaccapture.retargeting_engine.tensor_types import TactileLBSLayout from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup from isaaccapture.schema import ( From 27f2d04020f31b71700f246322fff17e3b1c50cb Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 02:04:11 -0700 Subject: [PATCH 35/45] Name tactile data and metadata input slots Signed-off-by: Lotus Li --- .../deviceio_source_nodes/tactile_source.py | 14 +++++++++----- .../retargeting_engine/tensor_types/__init__.py | 2 ++ .../retargeting_engine/tensor_types/indices.py | 7 +++++++ .../retargeting_engine/test_tactile_sources.py | 5 +++-- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 40a6a6b8c0..16f1aafe6a 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -18,7 +18,7 @@ from ..interface.retargeter_core_types import RetargeterIO, RetargeterIOType from ..interface.tensor_group import TensorGroup from ..interface.tensor_group_type import OptionalType -from ..tensor_types import TactileForces, TactileLBSLayout +from ..tensor_types import DeviceIOTactileIndex, TactileForces, TactileLBSLayout from .deviceio_tensor_types import DeviceIOTactileTracked if TYPE_CHECKING: @@ -79,8 +79,8 @@ def get_tracker(self) -> "ITracker": def poll_tracker(self, deviceio_session: Any) -> RetargeterIO: """Poll the tracker's frame and metadata for the compute step.""" tg = TensorGroup(DeviceIOTactileTracked()) - tg[0] = self._tracker.get_data(deviceio_session) - tg[1] = self._tracker.get_metadata(deviceio_session) + tg[DeviceIOTactileIndex.DATA] = self._tracker.get_data(deviceio_session) + tg[DeviceIOTactileIndex.METADATA] = self._tracker.get_metadata(deviceio_session) return {"deviceio_tactile": tg} def input_spec(self) -> RetargeterIOType: @@ -98,8 +98,12 @@ def output_spec(self) -> RetargeterIOType: def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: """Publish forces and a cached LBS layout; mark absent outputs with ``set_none()``.""" - data: "TactileDataArray | None" = inputs["deviceio_tactile"][0] - metadata: "TactileMetadata | None" = inputs["deviceio_tactile"][1] + data: "TactileDataArray | None" = inputs["deviceio_tactile"][ + DeviceIOTactileIndex.DATA + ] + metadata: "TactileMetadata | None" = inputs["deviceio_tactile"][ + DeviceIOTactileIndex.METADATA + ] forces_out = outputs[self.FORCES] if data is None: diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py index 322e4554b3..e77d869404 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py @@ -44,6 +44,7 @@ Generic3AxisPedalInputIndex, FullBodyInputIndex, SomaBodyInputIndex, + DeviceIOTactileIndex, TactileLBSLayoutIndex, HandJointIndex, BodyJointIndex, @@ -94,6 +95,7 @@ "Generic3AxisPedalInputIndex", "FullBodyInputIndex", "SomaBodyInputIndex", + "DeviceIOTactileIndex", "TactileLBSLayoutIndex", "HandJointIndex", "BodyJointIndex", diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py index 26609c93ef..dc675eac94 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py @@ -61,6 +61,13 @@ def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: ) +class DeviceIOTactileIndex(IntEnum): + """Data and metadata slots in a :func:`DeviceIOTactileTracked` group.""" + + DATA = 0 + METADATA = 1 + + class TactileLBSLayoutIndex(IntEnum): """Positions of the arrays in a :func:`TactileLBSLayout` group.""" diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 12acaae131..40deea8b5e 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -12,6 +12,7 @@ from isaaccapture.retargeting_engine.deviceio_source_nodes import TactileSource from isaaccapture.retargeting_engine.tensor_types import ( + DeviceIOTactileIndex, TactileLBSLayout, TactileLBSLayoutIndex, ) @@ -39,8 +40,8 @@ def _source(**kwargs) -> TactileSource: def _inputs(source, data, metadata) -> dict: tg = TensorGroup(source.input_spec()["deviceio_tactile"]) - tg[0] = data - tg[1] = metadata + tg[DeviceIOTactileIndex.DATA] = data + tg[DeviceIOTactileIndex.METADATA] = metadata return {"deviceio_tactile": tg} From 153532026930b8ead525b0310fd5ebf5ef8526c5 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 02:05:31 -0700 Subject: [PATCH 36/45] Key cached tactile layout arrays by named output slots Signed-off-by: Lotus Li --- .../deviceio_source_nodes/tactile_source.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 16f1aafe6a..29826904f4 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -18,7 +18,12 @@ from ..interface.retargeter_core_types import RetargeterIO, RetargeterIOType from ..interface.tensor_group import TensorGroup from ..interface.tensor_group_type import OptionalType -from ..tensor_types import DeviceIOTactileIndex, TactileForces, TactileLBSLayout +from ..tensor_types import ( + DeviceIOTactileIndex, + TactileForces, + TactileLBSLayout, + TactileLBSLayoutIndex, +) from .deviceio_tensor_types import DeviceIOTactileTracked if TYPE_CHECKING: @@ -68,7 +73,7 @@ def __init__( ) self._num_segments = int(num_segments) self._num_taxels = int(num_taxels) - self._layout: tuple[np.ndarray, ...] | None = None + self._layout: dict[TactileLBSLayoutIndex, np.ndarray] | None = None self._layout_bind_poses: np.ndarray | None = None super().__init__(name) @@ -142,22 +147,24 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N f"{self.name}: metadata has {num_segments} segments and {num_taxels} taxels, " f"source expects {self._num_segments} and {self._num_taxels}" ) - values = ( - np.asarray(segment_source.joint_ids, dtype=np.int32), - segment_bind_poses, - geometry_mapping.taxel_bind_poses, - geometry_mapping.taxel_segment_indices, - geometry_mapping.taxel_segment_weights, - ) - arrays = [] - for value in values: + values = { + TactileLBSLayoutIndex.SEGMENT_JOINT_IDS: np.asarray( + segment_source.joint_ids, dtype=np.int32 + ), + TactileLBSLayoutIndex.SEGMENT_BIND_POSES: segment_bind_poses, + TactileLBSLayoutIndex.TAXEL_BIND_POSES: geometry_mapping.taxel_bind_poses, + TactileLBSLayoutIndex.TAXEL_SEGMENT_INDICES: geometry_mapping.taxel_segment_indices, + TactileLBSLayoutIndex.TAXEL_SEGMENT_WEIGHTS: geometry_mapping.taxel_segment_weights, + } + arrays = {} + for index, value in values.items(): # Copy once: graph consumers must not alter the tracker's metadata baseline. array = np.asarray(value) - arrays.append( - array.astype(np.int32 if array.dtype == np.uint32 else array.dtype) + arrays[index] = array.astype( + np.int32 if array.dtype == np.uint32 else array.dtype ) - self._layout = tuple(arrays) + self._layout = arrays # Views keep the buffer alive and identify it across fresh Python wrappers. self._layout_bind_poses = segment_bind_poses - for index, array in enumerate(self._layout): + for index, array in self._layout.items(): layout_out[index] = array From 1b2b4a5561cbdfb9b686689e92b46feff83b6f27 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 14:35:12 -0700 Subject: [PATCH 37/45] Centralize metadata validation error context Signed-off-by: Lotus Li --- .../codegen/templates/pull/replay.cpp.template | 10 +--------- .../live_trackers/schema_metadata_tracker.hpp | 9 +-------- .../cpp/inc/schema/metadata_validation.hpp | 18 ++++++++++++++++++ .../cpp/core/schema/test_tactile_metadata.cpp | 3 +++ .../tracker_metadata/test_tracker_metadata.cpp | 5 ++--- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/core/codegen/templates/pull/replay.cpp.template b/src/core/codegen/templates/pull/replay.cpp.template index 1cd0d30bc5..eaf038dac8 100644 --- a/src/core/codegen/templates/pull/replay.cpp.template +++ b/src/core/codegen/templates/pull/replay.cpp.template @@ -10,7 +10,6 @@ #endif #include -#include #include #include #include @@ -30,14 +29,7 @@ Serialized<@METADATA@> @REPLAY_IMPL@::read_metadata(std::unique_ptr(sample.buffer.data())); - } - catch (const std::runtime_error& error) - { - throw std::runtime_error(collection_id + ": " + error.what()); - } + verify_metadata(*flatbuffers::GetRoot(sample.buffer.data()), collection_id); const int64_t log_time_ns = sample.timestamp.available_time_local_common_clock(); metadata_ = Serialized::adopt(std::move(sample.buffer)); diff --git a/src/core/schema/cpp/inc/schema/metadata_validation.hpp b/src/core/schema/cpp/inc/schema/metadata_validation.hpp index 9979d756c9..8cc6579df0 100644 --- a/src/core/schema/cpp/inc/schema/metadata_validation.hpp +++ b/src/core/schema/cpp/inc/schema/metadata_validation.hpp @@ -5,6 +5,10 @@ #include "tactile_metadata_validation.hpp" +#include +#include +#include + namespace core { @@ -20,4 +24,18 @@ inline void verify_metadata(const TactileMetadata& metadata) verify_tactile_metadata(metadata); } +//! Keep stream context here so live and replay report the same diagnostic. +template +inline void verify_metadata(const MetadataT& metadata, std::string_view name) +{ + try + { + verify_metadata(metadata); + } + catch (const std::runtime_error& error) + { + throw std::runtime_error(std::string(name) + ": " + error.what()); + } +} + } // namespace core diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index 5fac08e6be..a846b3abdb 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -87,6 +87,8 @@ TEST_CASE("TactileMetadata semantic verification accepts valid metadata", "[unit const auto metadata = core::pack(make_metadata()); CHECK_NOTHROW(core::verify_metadata(*metadata)); CHECK_NOTHROW(core::verify_metadata(*metadata)); + CHECK_NOTHROW(core::verify_metadata(*metadata, "tactile/right")); + CHECK_NOTHROW(core::verify_metadata(*metadata, "tactile/right")); } TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", "[unit][tactile_metadata]") @@ -118,6 +120,7 @@ TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", flatbuffers::Verifier verifier(metadata.buffer().data(), metadata.buffer().size()); REQUIRE(core::VerifyTactileMetadataBuffer(verifier)); CHECK_THROWS_WITH(core::verify_metadata(*metadata), "TactileMetadata." + reason); + CHECK_THROWS_WITH(core::verify_metadata(*metadata, "tactile/right"), "tactile/right: TactileMetadata." + reason); } TEST_CASE("TactileMetadata semantic verification rejects invalid or repeated joints", "[unit][tactile_metadata]") diff --git a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp index 4cd2b6f6d9..e14b81323a 100644 --- a/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp +++ b/tests/cpp/core/tracker_metadata/test_tracker_metadata.cpp @@ -322,8 +322,7 @@ TEST_CASE("Replay tactile metadata is semantically verified before force data is { REQUIRE_THROWS_WITH(core::verify_metadata(*metadata), Catch::Matchers::ContainsSubstring(reason)); CHECK_THROWS_WITH(core::ReplayTactileTrackerImpl::read_metadata(open_reader(path), base_name, recorded), - Catch::Matchers::ContainsSubstring("TactileMetadata." + reason) && - Catch::Matchers::ContainsSubstring(std::string(base_name))); + Catch::Matchers::StartsWith(std::string(base_name) + ": TactileMetadata." + reason)); return; } @@ -368,7 +367,7 @@ TEST_CASE("Live tactile metadata is semantically verified before adoption and re runtime.push(METADATA_COLLECTION, invalid.buffer(), 1); CHECK_THROWS_WITH(tracker.update(), - Catch::Matchers::ContainsSubstring(METADATA_COLLECTION + ": TactileMetadata.") && + Catch::Matchers::StartsWith(METADATA_COLLECTION + ": TactileMetadata.") && Catch::Matchers::ContainsSubstring("taxel_segment_weights[0]: weights must sum to one")); CHECK_FALSE(tracker.get()); From 999dcffa437a9df84d941df397613a5098b27bdc Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 14:49:37 -0700 Subject: [PATCH 38/45] Store tensor collection index and identity together Signed-off-by: Lotus Li --- .../inc/live_trackers/schema_tracker_base.hpp | 14 ++++++-- .../live_trackers/cpp/schema_tracker_base.cpp | 32 +++++++++---------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp b/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp index 75305b494d..4cf02b1204 100644 --- a/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp +++ b/src/core/live_trackers/cpp/inc/live_trackers/schema_tracker_base.hpp @@ -119,7 +119,11 @@ class SchemaTrackerBase //! Runtime identity of the discovered collection, distinct from its reusable list index. std::optional collection_instance_id() const { - return m_target_collection_index.has_value() ? m_target_collection_id : std::nullopt; + if (!m_target_collection) + { + return std::nullopt; + } + return m_target_collection->id; } private: @@ -143,8 +147,12 @@ class SchemaTrackerBase PFN_xrUpdateTensorListNV m_update_list_fn{ nullptr }; PFN_xrDestroyTensorListNV m_destroy_list_fn{ nullptr }; - std::optional m_target_collection_index; - std::optional m_target_collection_id; + struct TargetCollection + { + uint32_t index; + XrTensorCollectionIDNV id; + }; + std::optional m_target_collection; uint32_t m_sample_batch_stride{ 0 }; uint32_t m_sample_size{ 0 }; diff --git a/src/core/live_trackers/cpp/schema_tracker_base.cpp b/src/core/live_trackers/cpp/schema_tracker_base.cpp index bb688bd9ad..e7b1c75c7c 100644 --- a/src/core/live_trackers/cpp/schema_tracker_base.cpp +++ b/src/core/live_trackers/cpp/schema_tracker_base.cpp @@ -75,13 +75,13 @@ bool SchemaTrackerBase::ensure_collection() { if (is_known_non_fatal_tensor_unavailable(result)) { - m_target_collection_index.reset(); + m_target_collection.reset(); return false; } throw std::runtime_error("Failed to get latest generation, result=" + std::to_string(result)); } - if (latest_generation == m_cached_generation && m_target_collection_index.has_value()) + if (latest_generation == m_cached_generation && m_target_collection.has_value()) { return true; } @@ -93,7 +93,7 @@ bool SchemaTrackerBase::ensure_collection() { if (is_known_non_fatal_tensor_unavailable(result)) { - m_target_collection_index.reset(); + m_target_collection.reset(); return false; } throw std::runtime_error("Failed to update tensor list, result=" + std::to_string(result)); @@ -107,7 +107,7 @@ bool SchemaTrackerBase::ensure_collection() { if (is_known_non_fatal_tensor_unavailable(result)) { - m_target_collection_index.reset(); + m_target_collection.reset(); return false; } throw std::runtime_error("Failed to get list properties, result=" + std::to_string(result)); @@ -121,7 +121,7 @@ bool SchemaTrackerBase::ensure_collection() { if (is_known_non_fatal_tensor_unavailable(result)) { - m_target_collection_index.reset(); + m_target_collection.reset(); return false; } throw std::runtime_error("Failed to get collection properties, result=" + std::to_string(result)); @@ -144,17 +144,17 @@ bool SchemaTrackerBase::ensure_collection() m_sample_size = coll_props.data.totalSampleSize; // A recreated collection can reuse its list index, but never its runtime ID. - if (i != m_target_collection_index || coll_props.tensorCollectionId != m_target_collection_id) + if (!m_target_collection || i != m_target_collection->index || + coll_props.tensorCollectionId != m_target_collection->id) { m_last_sample_index.reset(); m_logger->info("Found target collection at index {}", i); } - m_target_collection_index = i; - m_target_collection_id = coll_props.tensorCollectionId; + m_target_collection = TargetCollection{ i, coll_props.tensorCollectionId }; return true; } - m_target_collection_index.reset(); + m_target_collection.reset(); return false; } @@ -171,9 +171,9 @@ bool SchemaTrackerBase::read_all_samples(std::vector& samples) samples.push_back(std::move(result)); } - // read_next_sample clears m_target_collection_index on API failure. + // read_next_sample clears m_target_collection on API failure. // If it's now empty we hit an error mid-drain, not a normal end-of-data. - if (!m_target_collection_index.has_value()) + if (!m_target_collection.has_value()) { return false; } @@ -218,7 +218,7 @@ void SchemaTrackerBase::create_tensor_list() bool SchemaTrackerBase::read_next_sample(SampleResult& out) { - if (!m_target_collection_index.has_value()) + if (!m_target_collection.has_value()) { return false; } @@ -226,7 +226,7 @@ bool SchemaTrackerBase::read_next_sample(SampleResult& out) // Prepare retrieval info XrTensorDataRetrievalInfoNV retrievalInfo{ XR_TYPE_TENSOR_DATA_RETRIEVAL_INFO_NV }; retrievalInfo.next = nullptr; - retrievalInfo.tensorCollectionIndex = m_target_collection_index.value(); + retrievalInfo.tensorCollectionIndex = m_target_collection->index; retrievalInfo.startSampleIndex = m_last_sample_index.has_value() ? m_last_sample_index.value() + 1 : 0; // Prepare output buffers (read one sample at a time for simplicity) @@ -250,14 +250,14 @@ bool SchemaTrackerBase::read_next_sample(SampleResult& out) // Policy: temporary collection loss is treated as non-fatal. m_logger->warn( "xrGetTensorDataNV: XR_ERROR_TENSOR_LOST_NV -- tensor/collection data lost; " - "clearing m_target_collection_index."); + "clearing target collection."); } else { - m_target_collection_index = std::nullopt; + m_target_collection.reset(); throw std::runtime_error("[SchemaTracker] xrGetTensorDataNV failed, result=" + std::to_string(result)); } - m_target_collection_index = std::nullopt; + m_target_collection.reset(); return false; } From dcb563ab752c3796c437aea0bacbcd681210f19b Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 14:57:38 -0700 Subject: [PATCH 39/45] Allow tactile records without a data payload Signed-off-by: Lotus Li --- src/core/schema/fbs/tactile_array.fbs | 3 ++- src/core/schema/python/schema_serialized.h | 22 ++++++---------- .../schema/python/tactile_array_bindings.h | 3 +-- tests/cpp/core/schema/test_tactile_array.cpp | 26 ++++++++++++------- .../python/core/schema/test_tactile_array.py | 9 ++++--- 5 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/core/schema/fbs/tactile_array.fbs b/src/core/schema/fbs/tactile_array.fbs index 9ff745e150..6e56f27fb1 100644 --- a/src/core/schema/fbs/tactile_array.fbs +++ b/src/core/schema/fbs/tactile_array.fbs @@ -18,7 +18,8 @@ table TactileDataArray { // MCAP recording wrapper for TactileDataArray. table TactileDataArrayRecord { - data: TactileDataArray (id: 0, required); + // Absent when tracking data is unavailable at this timestamp. + data: TactileDataArray (id: 0); timestamp: DeviceDataTimestamp (id: 1); } diff --git a/src/core/schema/python/schema_serialized.h b/src/core/schema/python/schema_serialized.h index 934e61642c..3fcb8a5855 100644 --- a/src/core/schema/python/schema_serialized.h +++ b/src/core/schema/python/schema_serialized.h @@ -156,17 +156,12 @@ std::vector> to_native_vector(c //! Binds a `Record` wrapper (an MCAP payload: data plus its capture timestamp). template -void bind_record(py::module& m, const char* name, const char* data_name, bool data_required = false) +void bind_record(py::module& m, const char* name, const char* data_name) { serialized_class(m, name, "Encoded MCAP record: a payload plus the timestamp it was captured at.") .def(py::init( - [data_required](const Serialized* data, const DeviceDataTimestamp& timestamp) + [](const Serialized* data, const DeviceDataTimestamp& timestamp) { - // Reject absent required data before the generated Pack asserts on it. - if (data_required && (data == nullptr || !*data)) - { - throw py::value_error("data: a payload is required"); - } typename RecordT::NativeTableType native; if (data != nullptr && *data) { @@ -176,14 +171,13 @@ void bind_record(py::module& m, const char* name, const char* data_name, bool da native.timestamp = std::make_shared(timestamp); return pack(native); }), - py::arg("data").none(!data_required), py::arg("timestamp"), - data_required ? "Encode a record from a required payload and its timestamp." : - "Encode a record from a payload and its timestamp. `data` may be None: MCAP " - "carries payload-less records, such as the message channel's frame sentinel.") + py::arg("data").none(true), py::arg("timestamp"), + "Encode a record from a payload and its timestamp. `data` may be None: MCAP " + "carries payload-less records, such as the message channel's frame sentinel.") // No no-arg form: `timestamp` is a struct with no meaningful default, and a record - // without one is not a thing MCAP stores. For optional data fields, a payload-less - // record -- the message channel's frame sentinel -- is spelled `Record(None, timestamp)`. + // without one is not a thing MCAP stores. A payload-less record -- the message + // channel's frame sentinel -- is spelled `Record(None, timestamp)`. .def_property_readonly( "data", [](const Serialized& self) -> py::object @@ -191,7 +185,7 @@ void bind_record(py::module& m, const char* name, const char* data_name, bool da const DataT* data = self->data(); return data != nullptr ? py::cast(self.narrow(data)) : py::none(); }, - data_required ? "The recorded payload." : "The recorded payload, or None when absent.") + "The recorded payload, or None when absent.") .def_property_readonly( "timestamp", [](const Serialized& self) { return self->timestamp(); }, py::return_value_policy::reference_internal, "Capture timestamp, or None when absent.") diff --git a/src/core/schema/python/tactile_array_bindings.h b/src/core/schema/python/tactile_array_bindings.h index a3f26a94bd..351947a0ba 100644 --- a/src/core/schema/python/tactile_array_bindings.h +++ b/src/core/schema/python/tactile_array_bindings.h @@ -49,8 +49,7 @@ inline void bind_tactile_array(py::module& m) return "TactileDataArray(taxels=" + std::to_string(forces != nullptr ? forces->size() : 0) + ")"; }); - bind_record( - m, "TactileDataArrayRecord", "TactileDataArray", /* data_required = */ true); + bind_record(m, "TactileDataArrayRecord", "TactileDataArray"); } } // namespace core diff --git a/tests/cpp/core/schema/test_tactile_array.cpp b/tests/cpp/core/schema/test_tactile_array.cpp index c480182fd5..2646e2cf48 100644 --- a/tests/cpp/core/schema/test_tactile_array.cpp +++ b/tests/cpp/core/schema/test_tactile_array.cpp @@ -57,21 +57,27 @@ TEST_CASE("TactileDataArray keeps its required vector present for an empty frame CHECK(verifier.VerifyBuffer()); } -TEST_CASE("Tactile verification rejects missing required fields", "[unit][tactile]") +TEST_CASE("TactileDataArray verification rejects a missing force vector", "[unit][tactile]") { flatbuffers::FlatBufferBuilder builder; // Build an unchecked empty table to test verification without required-field builder assertions. const auto start = builder.StartTable(); - builder.Finish(flatbuffers::Offset(builder.EndTable(start))); + builder.Finish(flatbuffers::Offset(builder.EndTable(start))); flatbuffers::Verifier verifier(builder.GetBufferPointer(), builder.GetSize()); - SECTION("taxel_forces") - { - CHECK_FALSE(verifier.VerifyBuffer()); - } - SECTION("record data") - { - CHECK_FALSE(core::VerifyTactileDataArrayRecordBuffer(verifier)); - } + CHECK_FALSE(verifier.VerifyBuffer()); +} + +TEST_CASE("TactileDataArrayRecord records unavailable data with its timestamp", "[unit][tactile]") +{ + core::TactileDataArrayRecordT native; + native.timestamp = std::make_shared(1, 2, 3); + const auto record = core::pack(native); + + flatbuffers::Verifier verifier(record.buffer().data(), record.buffer().size()); + REQUIRE(core::VerifyTactileDataArrayRecordBuffer(verifier)); + CHECK(record->data() == nullptr); + REQUIRE(record->timestamp() != nullptr); + CHECK(record->timestamp()->sample_time_local_common_clock() == 2); } TEST_CASE("TactileDataArrayRecord verifies and carries its timestamp", "[unit][tactile]") diff --git a/tests/python/core/schema/test_tactile_array.py b/tests/python/core/schema/test_tactile_array.py index 112badc6a5..0b3b235474 100644 --- a/tests/python/core/schema/test_tactile_array.py +++ b/tests/python/core/schema/test_tactile_array.py @@ -59,6 +59,9 @@ def test_record_carries_payload_and_timestamp(self): np.testing.assert_array_equal(record.data.taxel_forces, FORCES) assert record.timestamp.sample_time_raw_device_clock == 3 - def test_rejects_missing_payload(self): - with pytest.raises(TypeError, match="data"): - TactileDataArrayRecord(None, DeviceDataTimestamp(1, 2, 3)) + def test_record_without_payload_preserves_timestamp(self): + record = TactileDataArrayRecord(None, DeviceDataTimestamp(1, 2, 3)) + assert record.data is None + assert record.timestamp.available_time_local_common_clock == 1 + assert record.timestamp.sample_time_local_common_clock == 2 + assert record.timestamp.sample_time_raw_device_clock == 3 From 8f372cc049c7d49cd2c3aee2b0e944fda1c69d6d Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 15:01:25 -0700 Subject: [PATCH 40/45] Keep tactile layout cache state in one optional tuple Signed-off-by: Lotus Li --- .../deviceio_source_nodes/tactile_source.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 29826904f4..80094755f6 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -73,8 +73,9 @@ def __init__( ) self._num_segments = int(num_segments) self._num_taxels = int(num_taxels) - self._layout: dict[TactileLBSLayoutIndex, np.ndarray] | None = None - self._layout_bind_poses: np.ndarray | None = None + self._layout_cache: ( + tuple[dict[TactileLBSLayoutIndex, np.ndarray], np.ndarray] | None + ) = None super().__init__(name) def get_tracker(self) -> "ITracker": @@ -133,10 +134,8 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N ) # Normalize the schema stub's array type without copying its storage. segment_bind_poses = np.asarray(geometry_mapping.segment_bind_poses) - if ( - self._layout is None - or self._layout_bind_poses is None - or not np.shares_memory(segment_bind_poses, self._layout_bind_poses) + if self._layout_cache is None or not np.shares_memory( + segment_bind_poses, self._layout_cache[1] ): num_segments, num_taxels = ( len(segment_source.joint_ids), @@ -163,8 +162,8 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N arrays[index] = array.astype( np.int32 if array.dtype == np.uint32 else array.dtype ) - self._layout = arrays # Views keep the buffer alive and identify it across fresh Python wrappers. - self._layout_bind_poses = segment_bind_poses - for index, array in self._layout.items(): + self._layout_cache = (arrays, segment_bind_poses) + layout, _ = self._layout_cache + for index, array in layout.items(): layout_out[index] = array From 181a4b298987274afc59c0a548ec62aaac52e115 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 15:34:52 -0700 Subject: [PATCH 41/45] Remove redundant definition of TactileLBSLayoutIndex Signed-off-by: Lotus Li --- .../retargeting_engine/tensor_types/indices.py | 14 ++++---------- .../retargeting_engine/test_tactile_sources.py | 5 ----- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py index dc675eac94..70badad17b 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py @@ -25,6 +25,7 @@ FullBodyInput, SomaBodyInput, ) +from .tactile_types import TactileLBSLayout def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: @@ -59,6 +60,9 @@ def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: SomaBodyInputIndex: Any = _create_index_enum( "SomaBodyInputIndex", SomaBodyInput(), "soma_body_" ) +TactileLBSLayoutIndex: Any = _create_index_enum( + "TactileLBSLayoutIndex", TactileLBSLayout(0, 0) +) class DeviceIOTactileIndex(IntEnum): @@ -68,16 +72,6 @@ class DeviceIOTactileIndex(IntEnum): METADATA = 1 -class TactileLBSLayoutIndex(IntEnum): - """Positions of the arrays in a :func:`TactileLBSLayout` group.""" - - SEGMENT_JOINT_IDS = 0 - SEGMENT_BIND_POSES = 1 - TAXEL_BIND_POSES = 2 - TAXEL_SEGMENT_INDICES = 3 - TAXEL_SEGMENT_WEIGHTS = 4 - - class HandJointIndex(IntEnum): """Indices for OpenXR hand joints (XR_HAND_JOINT_COUNT_EXT = 26).""" diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 40deea8b5e..c91a05d191 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -13,7 +13,6 @@ from isaaccapture.retargeting_engine.deviceio_source_nodes import TactileSource from isaaccapture.retargeting_engine.tensor_types import ( DeviceIOTactileIndex, - TactileLBSLayout, TactileLBSLayoutIndex, ) from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group @@ -78,10 +77,6 @@ def test_specs(self): assert forces.inner_type.types[0].shape == (NUM_TAXELS, 3) assert src.output_spec()[TactileSource.LAYOUT].is_optional - def test_index_matches_layout_order(self): - names = [t.name for t in TactileLBSLayout(NUM_SEGMENTS, NUM_TAXELS).types] - assert names == [member.name.lower() for member in TactileLBSLayoutIndex] - def test_active_conversion(self): src = _source() forces = np.arange(9, dtype=np.float32).reshape(3, 3) From 163ea48576a03387432917b4c766085422e80cf6 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 15:38:59 -0700 Subject: [PATCH 42/45] Derive DeviceIO tactile indices from their tensor group Signed-off-by: Lotus Li --- .../retargeting_engine/deviceio_source_nodes/__init__.py | 2 ++ .../deviceio_source_nodes/deviceio_tensor_types.py | 6 ++++++ .../deviceio_source_nodes/tactile_source.py | 7 +++---- .../retargeting_engine/tensor_types/__init__.py | 2 -- .../retargeting_engine/tensor_types/indices.py | 7 ------- .../core/retargeting_engine/test_tactile_sources.py | 8 ++++---- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index 968a94c718..37a9c27205 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -73,6 +73,7 @@ DeviceIOSomaHandJointRotationsTracked, DeviceIOSomaHandJointPosesTracked, DeviceIOTactileTracked, + DeviceIOTactileIndex, MessageChannelMessagesTrackedType, MessageChannelConnectionStatus, MessageChannelStatusType, @@ -147,6 +148,7 @@ "DeviceIOSomaHandJointRotationsTracked", "DeviceIOSomaHandJointPosesTracked", "DeviceIOTactileTracked", + "DeviceIOTactileIndex", "DeviceIOMessageChannelMessagesTracked", "MessageChannelMessagesTrackedGroup", "MessageChannelStatusGroup", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py index 4516836d26..f37a98013f 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py @@ -14,6 +14,7 @@ from typing import Any from ..interface.tensor_type import TensorType from ..interface.tensor_group_type import TensorGroupType +from ..tensor_types.indices import _create_index_enum from isaaccapture.schema import ( HeadPose, HandPose, @@ -343,6 +344,11 @@ def DeviceIOTactileTracked() -> TensorGroupType: ) +DeviceIOTactileIndex: Any = _create_index_enum( + "DeviceIOTactileIndex", DeviceIOTactileTracked(), "tactile_" +) + + def DeviceIOMessageChannelMessagesTracked() -> TensorGroupType: """Message batch from DeviceIO MessageChannelTracker.""" return TensorGroupType( diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py index 80094755f6..95112c1086 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/tactile_source.py @@ -19,12 +19,11 @@ from ..interface.tensor_group import TensorGroup from ..interface.tensor_group_type import OptionalType from ..tensor_types import ( - DeviceIOTactileIndex, TactileForces, TactileLBSLayout, TactileLBSLayoutIndex, ) -from .deviceio_tensor_types import DeviceIOTactileTracked +from .deviceio_tensor_types import DeviceIOTactileIndex, DeviceIOTactileTracked if TYPE_CHECKING: from isaaccapture.deviceio_trackers import ITracker @@ -85,7 +84,7 @@ def get_tracker(self) -> "ITracker": def poll_tracker(self, deviceio_session: Any) -> RetargeterIO: """Poll the tracker's frame and metadata for the compute step.""" tg = TensorGroup(DeviceIOTactileTracked()) - tg[DeviceIOTactileIndex.DATA] = self._tracker.get_data(deviceio_session) + tg[DeviceIOTactileIndex.TRACKED] = self._tracker.get_data(deviceio_session) tg[DeviceIOTactileIndex.METADATA] = self._tracker.get_metadata(deviceio_session) return {"deviceio_tactile": tg} @@ -105,7 +104,7 @@ def output_spec(self) -> RetargeterIOType: def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: """Publish forces and a cached LBS layout; mark absent outputs with ``set_none()``.""" data: "TactileDataArray | None" = inputs["deviceio_tactile"][ - DeviceIOTactileIndex.DATA + DeviceIOTactileIndex.TRACKED ] metadata: "TactileMetadata | None" = inputs["deviceio_tactile"][ DeviceIOTactileIndex.METADATA diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py index e77d869404..322e4554b3 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py @@ -44,7 +44,6 @@ Generic3AxisPedalInputIndex, FullBodyInputIndex, SomaBodyInputIndex, - DeviceIOTactileIndex, TactileLBSLayoutIndex, HandJointIndex, BodyJointIndex, @@ -95,7 +94,6 @@ "Generic3AxisPedalInputIndex", "FullBodyInputIndex", "SomaBodyInputIndex", - "DeviceIOTactileIndex", "TactileLBSLayoutIndex", "HandJointIndex", "BodyJointIndex", diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py index 70badad17b..85eb57a502 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py @@ -65,13 +65,6 @@ def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: ) -class DeviceIOTactileIndex(IntEnum): - """Data and metadata slots in a :func:`DeviceIOTactileTracked` group.""" - - DATA = 0 - METADATA = 1 - - class HandJointIndex(IntEnum): """Indices for OpenXR hand joints (XR_HAND_JOINT_COUNT_EXT = 26).""" diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index c91a05d191..6379f0b738 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -10,11 +10,11 @@ import numpy as np import pytest -from isaaccapture.retargeting_engine.deviceio_source_nodes import TactileSource -from isaaccapture.retargeting_engine.tensor_types import ( +from isaaccapture.retargeting_engine.deviceio_source_nodes import ( DeviceIOTactileIndex, - TactileLBSLayoutIndex, + TactileSource, ) +from isaaccapture.retargeting_engine.tensor_types import TactileLBSLayoutIndex from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup from isaaccapture.schema import ( @@ -39,7 +39,7 @@ def _source(**kwargs) -> TactileSource: def _inputs(source, data, metadata) -> dict: tg = TensorGroup(source.input_spec()["deviceio_tactile"]) - tg[DeviceIOTactileIndex.DATA] = data + tg[DeviceIOTactileIndex.TRACKED] = data tg[DeviceIOTactileIndex.METADATA] = metadata return {"deviceio_tactile": tg} From 8fb5a118108d58851f8c2a7224220e696face9ee Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 15:46:10 -0700 Subject: [PATCH 43/45] Refine tactile schema test coverage and fixtures Signed-off-by: Lotus Li --- .../cpp/core/schema/test_tactile_metadata.cpp | 5 --- .../core/deviceio_trackers/pyproject.toml | 2 +- .../test_tactile_sources.py | 32 +++++++++++++++---- .../core/schema/test_tactile_metadata.py | 11 +------ 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/tests/cpp/core/schema/test_tactile_metadata.cpp b/tests/cpp/core/schema/test_tactile_metadata.cpp index a846b3abdb..ceca5e3169 100644 --- a/tests/cpp/core/schema/test_tactile_metadata.cpp +++ b/tests/cpp/core/schema/test_tactile_metadata.cpp @@ -86,9 +86,7 @@ TEST_CASE("TactileMetadata semantic verification accepts valid metadata", "[unit { const auto metadata = core::pack(make_metadata()); CHECK_NOTHROW(core::verify_metadata(*metadata)); - CHECK_NOTHROW(core::verify_metadata(*metadata)); CHECK_NOTHROW(core::verify_metadata(*metadata, "tactile/right")); - CHECK_NOTHROW(core::verify_metadata(*metadata, "tactile/right")); } TEST_CASE("TactileMetadata semantic verification rejects inconsistent lengths", "[unit][tactile_metadata]") @@ -221,14 +219,11 @@ TEST_CASE("TactileMetadata semantic verification checks each taxel's weights", " if (valid) { CHECK_NOTHROW(core::verify_metadata(*metadata)); - CHECK_NOTHROW(core::verify_metadata(*metadata)); } else { CHECK_THROWS_WITH( core::verify_metadata(*metadata), Catch::Matchers::ContainsSubstring("TactileMetadata." + reason)); - CHECK_THROWS_WITH(core::verify_metadata(*metadata), - Catch::Matchers::ContainsSubstring("TactileMetadata." + reason)); } } diff --git a/tests/python/core/deviceio_trackers/pyproject.toml b/tests/python/core/deviceio_trackers/pyproject.toml index eea42f28a9..deeaa91e92 100644 --- a/tests/python/core/deviceio_trackers/pyproject.toml +++ b/tests/python/core/deviceio_trackers/pyproject.toml @@ -7,7 +7,7 @@ version = "0.0.0" requires-python = ">=3.11,<3.14" [project.optional-dependencies] -dev = ["pytest", "numpy", "pyyaml"] +dev = ["pytest", "numpy"] [tool.pytest.ini_options] pythonpath = [".", "../.."] diff --git a/tests/python/core/retargeting_engine/test_tactile_sources.py b/tests/python/core/retargeting_engine/test_tactile_sources.py index 6379f0b738..4d6839dc05 100644 --- a/tests/python/core/retargeting_engine/test_tactile_sources.py +++ b/tests/python/core/retargeting_engine/test_tactile_sources.py @@ -3,10 +3,11 @@ """Tests for the TactileSource DeviceIO converter. -Payloads are built through the real schema bindings; no OpenXR device is involved. -Session-backed tracker reads are covered in tests/cpp/core/replay_deviceio_session. +Payloads use real schema bindings; polling uses a mock tracker. """ +from unittest.mock import MagicMock + import numpy as np import pytest @@ -49,7 +50,10 @@ def _outputs(source) -> dict: def _metadata( - num_segments: int = NUM_SEGMENTS, num_taxels: int = NUM_TAXELS + num_segments: int = NUM_SEGMENTS, + num_taxels: int = NUM_TAXELS, + *, + taxel_x: float = 0.01, ) -> TactileMetadata: indices = np.zeros((num_taxels, 4), np.uint32) weights = np.zeros((num_taxels, 4), np.float32) @@ -57,7 +61,7 @@ def _metadata( segment_poses = np.zeros((num_segments, 7), np.float32) segment_poses[:, 6] = 1 taxel_poses = np.zeros((num_taxels, 7), np.float32) - taxel_poses[:, :3] = 0.01 + taxel_poses[:, :3] = (taxel_x, 0.01, 0.01) taxel_poses[:, 6] = 1 return TactileMetadata( LBS(segment_poses, taxel_poses, indices, weights), @@ -69,6 +73,23 @@ class TestTactileSource: def test_creation_and_tracker(self): assert _source().get_tracker().get_name() == "TactileTracker" + def test_poll_reads_data_and_metadata_from_one_tracker(self): + src = _source() + data = TactileDataArray(np.zeros((NUM_TAXELS, 3), np.float32)) + metadata = _metadata() + tracker = MagicMock() + tracker.get_data.return_value = data + tracker.get_metadata.return_value = metadata + src._tracker = tracker + session = object() + + inputs = src.poll_tracker(session) + + tracker.get_data.assert_called_once_with(session) + tracker.get_metadata.assert_called_once_with(session) + assert inputs["deviceio_tactile"][DeviceIOTactileIndex.TRACKED] is data + assert inputs["deviceio_tactile"][DeviceIOTactileIndex.METADATA] is metadata + def test_specs(self): src = _source() assert list(src.input_spec()) == ["deviceio_tactile"] @@ -164,8 +185,7 @@ def test_layout_cache_follows_the_metadata_buffer(self): src = _source() metadata = _metadata() first = src(_inputs(src, None, metadata)) - replacement = _metadata() - replacement.geometry_mapping.taxel_bind_poses[:, 0] = 0.02 + replacement = _metadata(taxel_x=0.02) second = src(_inputs(src, None, replacement)) for index in TactileLBSLayoutIndex: diff --git a/tests/python/core/schema/test_tactile_metadata.py b/tests/python/core/schema/test_tactile_metadata.py index ef0ed38836..81d6915d97 100644 --- a/tests/python/core/schema/test_tactile_metadata.py +++ b/tests/python/core/schema/test_tactile_metadata.py @@ -73,15 +73,6 @@ def test_nested_views_and_arrays_retain_the_buffer(): np.testing.assert_array_equal(rows, lbs_arrays()["taxel_bind_poses"]) -def test_metadata_revalidates_existing_geometry_mapping(): - geometry_mapping = LBS(**lbs_arrays()) - geometry_mapping.taxel_segment_weights[0] = 0 - with pytest.raises(ValueError, match="taxel_segment_weights"): - TactileMetadata( - geometry_mapping, HandSkeletonRef([HandJoint.WRIST, HandJoint.INDEX_TIP]) - ) - - @pytest.mark.parametrize( "field", ["taxel_bind_poses", "taxel_segment_indices", "taxel_segment_weights"] ) @@ -110,7 +101,7 @@ def test_rejects_out_of_range_zero_weight_slot(): ) -@pytest.mark.parametrize("weight", [-1.0, np.inf, np.nan, 0.5]) +@pytest.mark.parametrize("weight", [-1.0, np.inf, np.nan, 0.5, 0.0]) def test_rejects_invalid_weights(weight): arrays = lbs_arrays() arrays["taxel_segment_weights"][0, 0] = weight From e94042e0dcf18cdb4e235c365a85c1a5c58578b0 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 15:48:22 -0700 Subject: [PATCH 44/45] Remove unreachable tactile feeder fallback paths Signed-off-by: Lotus Li --- .../tactile/python/isaaccapture_examples/tactile/feeder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py index e327e3c4fd..5b6841e8a0 100644 --- a/examples/tactile/python/isaaccapture_examples/tactile/feeder.py +++ b/examples/tactile/python/isaaccapture_examples/tactile/feeder.py @@ -66,8 +66,7 @@ def mock_metadata(joint_ids: list[HandJoint], num_taxels: int) -> TactileMetadat # LBS indices address joint_ids rows, not HandJoint ordinals. segment_by_joint = {int(joint): segment for segment, joint in enumerate(joint_ids)} positions = np.zeros((num_segments, 3), np.float32) - if int(HandJoint.PALM) in segment_by_joint: - positions[segment_by_joint[int(HandJoint.PALM)]] = (0.0, 0.05, 0.0) + positions[segment_by_joint[int(HandJoint.PALM)]] = (0.0, 0.05, 0.0) taxel_segments = [] for finger, first in enumerate(FINGER_FIRST_JOINTS): @@ -93,7 +92,7 @@ def mock_metadata(joint_ids: list[HandJoint], num_taxels: int) -> TactileMetadat taxel_segments.append(segment_by_joint[joint]) if not taxel_segments: - taxel_segments.append(segment_by_joint.get(WRIST, 0)) + taxel_segments.append(segment_by_joint[WRIST]) taxel_positions = np.zeros((num_taxels, 3), np.float32) taxel_segment_indices = np.zeros((num_taxels, 4), np.uint32) From 453e4ecd10a43d0bb03a6165ad5ecc6dc6b01010 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 23 Oct 2026 15:50:28 -0700 Subject: [PATCH 45/45] Clarify tactile consumer buffer sizing Signed-off-by: Lotus Li --- docs/source/device/tactile/index.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/device/tactile/index.rst b/docs/source/device/tactile/index.rst index 40cb4e4fa0..dfb386122a 100644 --- a/docs/source/device/tactile/index.rst +++ b/docs/source/device/tactile/index.rst @@ -196,7 +196,11 @@ and exposes it through ``get_metadata()``. It withholds force data until metadat is confirmed. Different or invalid metadata causes an error; after a producer restart, its metadata must be confirmed again. See :ref:`tracker-metadata`. -The force-buffer size is defined by ``max_flatbuffer_size``. Metadata has its own +The consumer's ``max_flatbuffer_size`` must be at least the producer's declared +sample size, including FlatBuffer overhead. The default is +``TactileTracker.DEFAULT_MAX_FLATBUFFER_SIZE``; pass a larger +``max_flatbuffer_size`` to ``TactileTracker`` when the producer uses larger force +samples. Metadata has its own ``TRACKER_METADATA_MAX_SIZE`` limit, including FlatBuffer overhead. The data array length must match the metadata's taxel count.