pyproject.toml

# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
# Copyright 2026 Tensor Auto Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[project.urls]
homepage = "https://github.com/TensorAuto/OpenTau"
issues = "https://github.com/TensorAuto/OpenTau/issues"
huggingface = "https://huggingface.co/TensorAuto"

[project]
name = "opentau"
version = "0.14.0"
description = "OpenTau: Tensor's VLA Training Infrastructure for Real-World Robotics in Pytorch"
authors = [
    { name = "Shuheng Liu", email = "wish1104@icloud.com" },
    { name = "William Yue", email = "williamyue37@gmail.com" },
    { name = "Akshay Shah", email = "akshayhitendrashah@gmail.com" },
]
readme = "README.md"
license = { text = "Apache-2.0" }
requires-python = "==3.10.*"
keywords = ["robotics", "deep learning", "pytorch"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Intended Audience :: Education",
    "Intended Audience :: Science/Research",
    "Topic :: Software Development :: Build Tools",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "License :: OSI Approved :: Apache Software License",
    "Programming Language :: Python :: 3.10",
]
dependencies = [
    "cmake>=3.29.0.1",
    "datasets>=2.19.0",
    "deepdiff>=7.0.1",
    "diffusers>=0.27.2",
    "draccus>=0.10.0",
    "einops>=0.8.0",
    "flask>=3.0.3",
    "gdown>=5.1.0",
    "h5py>=3.10.0",
    "huggingface-hub[hf-transfer,cli]>=0.27.1 ; python_version < '4.0'",
    "imageio[ffmpeg]>=2.34.0",
    "mediapipe>=0.10.0",
    "jsonlines>=4.0.0",
    "numba>=0.62.0",
    # flux3_action's video VAE imports `natten` at module scope with no fallback, and the
    # VAE is mandatory on the encode path -- so it is needed for inference, not just
    # decoding. NATTEN publishes no PyPI wheels (sdist only, which compiles CUDA kernels),
    # so the prebuilt wheel is pinned by URL in [tool.uv.sources] below. It exists only for
    # linux x86_64, hence the marker -- same shape as torchcodec / onnxruntime-gpu.
    # NOTE: the wheel is built against torch 2.10.0 / cu128 exactly, so a torch bump
    # requires re-pinning it here (see policies/flux3_action/VENDOR.md).
    "natten==0.21.6+torch2100cu128 ; sys_platform == 'linux' and platform_machine == 'x86_64'",
    "omegaconf>=2.3.0",
    "opencv-python-headless>=4.9.0",
    "packaging>=24.2",
    "av>=12.0.5",
    "pymunk>=6.6.0",
    "pynput>=1.7.7",
    "pyzmq>=26.2.1",
    "rerun-sdk>=0.21.0",
    "termcolor>=2.4.0",
    "torch>=2.7.1",
    "torchcodec>=0.10.0, <0.11.0; sys_platform != 'win32' and (sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')) and (sys_platform != 'darwin' or platform_machine != 'x86_64')",
    "torchvision>=0.22.1",
    "wandb>=0.27.0",
    "zarr>=2.17.0",
    "scikit-learn>=1.7.1",
    "onnx>=1.18.0",
    "onnxruntime>=1.22.1 ; sys_platform == 'darwin' or platform_machine == 'arm64' or platform_machine == 'aarch64'",
    "onnxruntime-gpu>=1.22.0 ; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and (platform_machine == 'AMD64' or platform_machine == 'x86_64'))) ",
    "onnxscript>=0.3.1",
    "onnx-ir>=0.1.4",
    "transformers>=4.57,<4.58",
    "scipy>=1.15.2",
    "pytest>=8.1.0",
    "pytest-cov>=5.0.0",
    "pyserial>=3.5",
    "pytest-xdist>=3.8.0",
    "scikit-image>=0.23.2",
    "pandas>=2.2.2",
    "accelerate>=1.4.0",
    "deepspeed>=0.17.1",
    "gymnasium[other]>=0.29",
    "grpcio>=1.60.0",
    "grpcio-tools>=1.60.0",
    "protobuf>=4.25.0",
    "deepspeed>=0.17.1",
    "rosbags>=0.10.4",
    "openai>=2.16.0",
    "python-dotenv>=1.2.1",
    "anthropic>=0.55.0",
    "google-genai>=1.0.0",
]

[project.scripts]
opentau-train = "opentau.scripts.launch:train"
opentau-eval = "opentau.scripts.launch:eval"
opentau-export = "opentau.scripts.launch:export"
opentau-dataset-viz = "opentau.scripts.launch:visualize"
opentau-so101-calibrate = "opentau.scripts.so101.calibrate:main"
opentau-so101-teleoperate = "opentau.scripts.so101.teleoperate:main"
opentau-so101-record = "opentau.scripts.so101.record:main"
opentau-so101-setup-motors = "opentau.scripts.so101.setup_motors:main"
opentau-so101-find-port = "opentau.scripts.so101.find_port:main"
opentau-so101-find-cameras = "opentau.scripts.so101.find_cameras:main"
opentau-accel-diagnose = "opentau.scripts.diagnose_accel:main"
opentau-accel-calibrate = "opentau.scripts.calibrate_accel:main"

[project.optional-dependencies]
dev = ["pre-commit>=3.7.0",
    "debugpy>=1.8.1",
    "py-spy>=0.4.0",
    "pytest>=8.1.0",
    "pytest-cov>=5.0.0",
    "pyserial>=3.5",
    "pytest-xdist>=3.8.0",
    "sphinx>=8.1.3",
    "sphinx-rtd-theme>=3.0.1",
    "sphinx-copybutton>=0.5.2",
    "sphinx-design>=0.6.1",
    "myst-parser>=4.0.0",
]
libero = [
    "ninja",
    "bddl==1.0.1",
    "easydict==1.9",
    "future==0.18.2",
    "matplotlib>=3.5.3",
    "egl-probe",
    "robomimic==0.2.0",
    # robosuite 1.5.2 (composite-controller framework) so LIBERO shares a venv with
    # RoboCasa. Sourced from a pinned robosuite *master* commit (see [tool.uv.sources])
    # because robocasa needs MujocoEnv(load_model_on_init=...), added on master after
    # the 1.5.2 PyPI release; master still self-reports "1.5.2" so this pin matches.
    # The LIBERO fork (1.5 controller API, see load_arm_controller_config) is validated
    # on this commit; robomimic 0.2.0 does not pin robosuite, so it co-installs unchanged.
    "robosuite==1.5.2",
    "thop==0.1.1.post2209072238",
    "mujoco>=3.3.5",
    # pyopengl-accelerate ships a Cython C-extension compiled against a specific
    # numpy ABI: the 3.1.7 wheels are numpy-1 only and crash with "numpy.dtype
    # size changed" under numpy 2.x (hit on the libero osmesa render path). 3.1.9+
    # ship wheels rebuilt against numpy 2; keep PyOpenGL pinned in lockstep.
    "PyOpenGL==3.1.10",
    "libero",
    "pyopengl-accelerate==3.1.10 ; sys_platform == 'linux'",
    "mujoco>=3.1.6 ; sys_platform == 'linux'",
    "pyopengl==3.1.10 ; sys_platform == 'linux'",
]
robocasa = [
    # RoboCasa365 kitchen sim for simulated eval, co-installed on the shared
    # robosuite-1.5 (master) stack from the `libero` extra. Uses the
    # shuheng-liu/robocasa packaging fork (see [tool.uv.sources]) that drops
    # upstream's lerobot==0.3.3 / tianshou / opencv-python / hidapi deps and loosens
    # its numpy/numba/scipy/mujoco pins so it resolves in the shared lock (upstream's
    # setup.py can't be `--no-deps`'d inside a uv lock). robosuite is not in robocasa's
    # deps (installed separately), so pin it here matching `libero`. Kitchen assets
    # (~5-10GB) are a separate runtime download:
    # `python -m robocasa.scripts.download_kitchen_assets`.
    "robocasa",
    "robosuite==1.5.2",
    "mujoco>=3.3.5",
]
urdf = [
    "rerun-sdk>=0.28.2",
]
trt = [
    "tensorrt>=10.15.1.29 ; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and (platform_machine == 'AMD64' or platform_machine == 'x86_64'))",
]
so101 = [
    "feetech-servo-sdk>=1.0.0",
]

[tool.uv.sources]
natten = [
    { url = "https://github.com/SHI-Labs/NATTEN/releases/download/v0.21.6/natten-0.21.6%2Btorch2100cu128-cp310-cp310-linux_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" },
]
libero = { git = "https://github.com/shuheng-liu/LIBERO" , branch = "master" }  # the official libero repo is misconfigured for pip install with git
# robomimic's egl_probe ships a CMakeLists with cmake_minimum_required(VERSION 2.8.12),
# which CMake >= 4 rejects. This fork raises the floor to 3.5 so it builds on CMake 4
# without pinning the build's cmake; the tag keeps the source reproducible.
egl-probe = { git = "https://github.com/shuheng-liu/egl_probe", tag = "v1.0.1-cmake4" }
# robocasa main needs robosuite *master* (MujocoEnv(load_model_on_init=...), added after
# the 1.5.2 PyPI release); pin the exact commit for reproducibility. Master self-reports
# version "1.5.2", which satisfies the `robosuite==1.5.2` pins in the extras above, and
# LIBERO is validated to construct+step on this commit (so the shared stack still works).
robosuite = { git = "https://github.com/ARISE-Initiative/robosuite", rev = "85abee228d1c43ab1939bce33028099945d453b4" }
# OpenTau packaging fork of robocasa: drops upstream's lerobot==0.3.3 / tianshou /
# opencv-python / hidapi deps and loosens its numpy/numba/scipy/mujoco pins + import-time
# version asserts so it co-installs on the shared robosuite-1.5 / numpy-2 stack.
robocasa = { git = "https://github.com/shuheng-liu/robocasa", rev = "f7db21c11f25408d3a59a5e878bb5c7ca9030c4d" }

[tool.uv]
# `extra-build-dependencies` injects a cmake wheel into egl-probe's PEP 517 build
# isolation so `uv sync --extra libero` builds the egl_probe sdist without a system
# cmake (a C/C++ toolchain is still required). egl_probe's CMakeLists floor was
# raised to 3.5 in the fork, so the injected cmake is free to be 4.x. This key is
# only honored by uv >= 0.8.4, which `required-version` enforces.
required-version = ">=0.8.4"
extra-build-dependencies = { egl-probe = ["cmake"] }
# robosuite 1.5.2 hard-pins mink==0.0.5, which caps numpy<2.0.0 and would block
# OpenTau's numpy-2 stack (core rerun-sdk, the urdf extra, and LIBERO itself all
# require numpy>=2). mink is only used by robosuite's GR1 whole-body-IK controller,
# imported under a try/except in robosuite/__init__.py — never on the OSC_POSE/Panda
# path LIBERO uses. mink 0.0.10+ drops the numpy<2 cap while still needing only
# mujoco>=3.1.6, so override robosuite's over-strict pin up to that line (no mujoco bump).
override-dependencies = ["mink>=0.0.10,<0.1"]

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools.package-data]
# Ship the custom flash-attention CUDA source so the kernel can be JIT-compiled
# from an installed wheel (not just an editable/source checkout).
"opentau.policies.flash_attn_cuda" = ["_csrc/*.cu", "_csrc/*.cuh", "_csrc/*.cpp", "_csrc/*.h"]
# Ship the planner prompt library so installed (non-editable) wheels can load it.
"opentau.planner" = ["prompts.yaml"]

[tool.ruff]
line-length = 110
target-version = "py310"
# Vendored third-party sources, kept byte-identical to upstream so re-syncing is a
# mechanical diff -- linting or formatting them would rewrite code we do not own.
# Mirrors the pre-commit global exclude; see policies/flux3_action/VENDOR.md.
extend-exclude = [
    "src/opentau/policies/flux3_action/config.py",
    "src/opentau/policies/flux3_action/hub.py",
    "src/opentau/policies/flux3_action/policy.py",
    "src/opentau/policies/flux3_action/models",
    "src/opentau/policies/flux3_action/processing",
    "src/opentau/policies/flux3_action/inference",
    "src/opentau/policies/flux3_action/checkpoints",
]

[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I", "N", "B", "C4", "SIM"]

[tool.ruff.lint.per-file-ignores]
# Server must implement gRPC interface with PascalCase method names
"src/opentau/scripts/grpc/server.py" = ["N802"]
# Uppercase names for rotation matrices follow standard math convention
"src/opentau/scripts/recordhuman_to_lerobot.py" = ["N803", "N806"]

[tool.bandit]
exclude_dirs = [
    "tests",
    "src/opentau/datasets/push_dataset_to_hub",
    "src/opentau/datasets/v2/convert_dataset_v1_to_v2"
]
skips = ["B101", "B311", "B404", "B603"]

[tool.typos]
default.extend-ignore-re = [
    "(?Rm)^.*(#|//)\\s*spellchecker:disable-line$", # spellchecker:disable-line
    "(?s)(#|//)\\s*spellchecker:off.*?\\n\\s*(#|//)\\s*spellchecker:on", # spellchecker:<on|off>
]
default.extend-ignore-identifiers-re = [
    # Add individual words here to ignore them
    "2nd",
    "pn",
    "ser",
    "ein",
    # Qwen3-VL grid shape names (temporal/height/width): image_grid_thw, video_grid_thw, grid_thw.
    ".*grid_thw.*",
    "thw",
    # Verbatim camera-column names from upstream community SO101 HF datasets
    # (configs/examples/refs/so101_community_mixture.json data_features_name_mapping
    # values) -- these must match the misspelled column exactly to resolve.
    "righ",
    "rigth",
    "sholder",
    "sence",
]

[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"

[tool.pytest.ini_options]
markers = [
    "slow: marks tests as slow (runtime > 1s).",
    "gpu: marks tests as gpu (uses gpu).",
    "network: marks tests that require live Hugging Face Hub access (excluded from the gating CPU run; run nightly in network_test.yml).",
]