From 27e6ab9e40d0190925f3f64128b191016c9bd613 Mon Sep 17 00:00:00 2001 From: ChepuriNatraj Date: Tue, 11 Aug 2026 20:49:40 +0530 Subject: [PATCH 1/5] docs: preserve upstream attribution and record LeIsaac provenance Add NOTICE, THIRD_PARTY.md and docs/upstream.md documenting the Apache-2.0 derivation from LeIsaac (base commit 24d3bcd, v0.4.0), all third-party licenses (IsaacLab BSD-3, OpenPI/LeRobot/GR00T Apache-2.0, msgpack-numpy BSD-3), and the key finding that upstream already ships a generic Policy ABC and an OpenPI websocket client -- so Pi0.5 integration is a re-target, not a from-scratch abstraction build. --- NOTICE | 17 +++++++++ THIRD_PARTY.md | 40 ++++++++++++++++++++ docs/upstream.md | 98 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 NOTICE create mode 100644 THIRD_PARTY.md create mode 100644 docs/upstream.md diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..cf64eb16 --- /dev/null +++ b/NOTICE @@ -0,0 +1,17 @@ +SO-101 Pi0.5 Research Platform +Copyright 2026 Chepuri Natraj + +This product includes software developed by the LightwheelAI team as part of +the LeIsaac project (https://github.com/LightwheelAI/leisaac), licensed under +the Apache License, Version 2.0. + +This repository is an independent research derivative of LeIsaac. It retains +LeIsaac's IsaacLab-based simulation, SO-101 robot assets, teleoperation, +data-collection, and dataset-conversion infrastructure, and re-targets the +primary learned policy to pi0.5 (Pi0.5) from Physical Intelligence (OpenPI). + +Derived from LeIsaac upstream commit: + 24d3bcd3f1e4585740fc79921782c41617237812 (v0.4.0) + +Portions of this software incorporate or interface with third-party projects. +See THIRD_PARTY.md for the full attribution list and their licenses. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 00000000..7b904917 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,40 @@ +# Third-Party Attribution + +All entries below were verified against the actual files vendored in or depended +on by this repository. Versions/commits are recorded as observed, not assumed. + +## Primary upstream: LeIsaac + +- **Project:** LeIsaac (LeRobot & IsaacLab Integration) +- **Source:** https://github.com/LightwheelAI/leisaac +- **License:** Apache License 2.0 (see `LICENSE`, retained unmodified) +- **Base commit:** `24d3bcd3f1e4585740fc79921782c41617237812` (package version `0.4.0`) +- **Author (upstream `pyproject.toml`):** Yinghao Shuai +- **What is retained:** IsaacLab task/environment infrastructure + (`source/leisaac/leisaac/tasks`), SO-101 robot + scene assets + (`.../assets`), teleoperation devices (`.../devices`: keyboard, gamepad, + lerobot/so101 leader, lekiwi), dataset conversion + (`scripts/convert/isaaclab2lerobot*.py`), state-machine datagen, and the + policy client layer (`.../policy`). + +Apache-2.0 obligations honoured: `LICENSE` retained verbatim, `NOTICE` added +stating the derivation and upstream copyright, modifications documented in +`docs/upstream.md`. Existing per-file copyright headers are left intact. + +## Vendored / interfaced third-party code + +| Project | Where | License | Verified detail | +|---|---|---|---| +| IsaacLab | `dependencies/IsaacLab` (git submodule) | BSD-3-Clause | Submodule pinned to `3c6e67bb5c7ada942a6d1884ab69338f57596f77`; local checkout at `~/IsaacLab` reports `VERSION` = `2.3.0` at the same commit. | +| Isaac Sim | bundled with IsaacLab | NVIDIA Omniverse License | Local `_isaac_sim/VERSION` = `5.1.0-rc.19+release.26219.9c81211b.gl`. LeIsaac `pyproject.toml` classifier declares `Isaac Sim :: 5.1.0`. | +| OpenPI (Physical Intelligence) | interfaced via `.../policy/openpi/` | Apache-2.0 | `msgpack_numpy.py` + `image_tools.py` are adapted from OpenPI. Upstream client targets OpenPI commit `5bff19b0c0c447c7a7eaaaccf03f36d50998ec9d`. | +| msgpack-numpy | `.../policy/openpi/msgpack_numpy.py` | BSD-3-Clause | File docstring credits https://github.com/lebedov/msgpack-numpy as the adaptation source. | +| LeRobot (HuggingFace) | `.../policy/lerobot/`, dataset conversion | Apache-2.0 | Upstream client targets LeRobot `v0.3.3`; `pyproject.toml` optional extra pins `lerobot==0.4.2`. | +| Isaac-GR00T (NVIDIA) | `.../policy/gr00t/` (demoted to optional) | Apache-2.0 | N1.5 client targets commit `4af2b622892f7dcb5aae5a3fb70bcb02dc217b96`; N1.6 client targets `e8e625f4f21898c506a1d8f7d20a289c97a52acf`. | + +## Note on GR00T + +GR00T is **retained** as an optional policy backend, not deleted. Its code is +Apache-2.0 and its attribution above remains binding for as long as the files +are present. The re-targeting work moves GR00T off the *primary* policy path; +it does not strip its licensing. diff --git a/docs/upstream.md b/docs/upstream.md new file mode 100644 index 00000000..b18626e8 --- /dev/null +++ b/docs/upstream.md @@ -0,0 +1,98 @@ +# Upstream Provenance and Divergence + +This document records what this repository inherited from LeIsaac, and how it +diverges. Required by our Apache-2.0 obligation to state modifications. + +## Derivation point + +| Item | Value (verified) | +|---|---| +| Upstream repo | https://github.com/LightwheelAI/leisaac | +| Upstream branch | `main` | +| Base commit | `24d3bcd3f1e4585740fc79921782c41617237812` | +| Upstream package version | `0.4.0` | +| Upstream license | Apache-2.0 | +| Our development branch | `dev/pi05-so101` | +| Git remote layout | upstream LeIsaac kept as remote `upstream`; our fork will be remote `origin` | + +The upstream remote is deliberately preserved (renamed to `upstream`) so that +history and attribution remain inspectable, and so upstream fixes can be pulled. + +## What upstream actually provides (inspected, not assumed) + +Inspection of `source/leisaac/leisaac/` found: + +- `assets/robots`, `assets/scenes` — SO-101 robot and scene assets +- `tasks/` — 6 concrete tasks + a `template`: + `assemble_hamburger`, `clean_toy_table`, `cleanup_trash`, `fold_cloth`, + `lift_cube`, `pick_orange` +- `devices/` — `keyboard`, `gamepad`, `lerobot` (SO-101 leader arm), `lekiwi` +- `datagen/state_machine` — scripted expert data generation +- `enhance/` — `assets`, `datasets`, `envs`, `managers` extensions +- `policy/` — `base.py` (abstract `Policy`), `gr00t/`, `lerobot/`, `openpi/`, + `service_policy_clients.py` +- `utils/` — including `constant.SINGLE_ARM_JOINT_NAMES` and + `robot_utils.convert_leisaac_action_to_lerobot` / `convert_lerobot_action_to_leisaac` + +Scripts (`scripts/`): `convert/isaaclab2lerobot.py`, +`convert/isaaclab2lerobotv3.py`, `convert/lerobot2isaaclab.py`, +`datagen/state_machine/{generate,replay}.py`, `environments/list_envs.py`, +`environments/teleoperation/{replay,teleop_se3_agent,so101_joint_state_server}.py`, +`evaluation/policy_inference.py`, `mimic/*`, `tutorials/*`. + +## Critical finding that reshapes the plan + +Upstream is **not** GR00T-only. It already ships: + +1. An abstract `Policy` base class with a single `get_action()` method, plus + `CheckpointPolicy`, `ZMQServicePolicy`, and `WebsocketServicePolicy` + transports (`policy/base.py`). +2. A working **`OpenPIServicePolicyClient`** (`policy/service_policy_clients.py`) + that already speaks the OpenPI websocket protocol, resizes images to + `224x224` with `resize_with_pad`, and sends + `{"images/": uint8 HWC, "state": float64[6], "prompt": str}`, + receiving `{"actions": (chunk, 6)}`. +3. An `openpi` install extra in `pyproject.toml`. + +Consequence: the requested "remove GR00T from the primary path" and "add a +generic policy interface" work is **substantially already present upstream**. +The honest remaining work is therefore *not* inventing an abstraction layer — +it is: + +- making Pi0.5 (not Pi0, not GR00T) the **default** policy and config, +- verifying the OpenPI client's observation/action contract against the + **actual** `pi05_*` configs in the OpenPI tree, +- supplying a genuine SO-101 Pi0.5 **training config**, which upstream does not + provide (upstream only has an inference-time client), +- building the evaluation/metrics layer. + +Per the working rules ("do not blindly rewrite working infrastructure", +"do not create unnecessary abstraction layers"), the existing `Policy` ABC and +OpenPI transport are retained rather than replaced. + +## Pi0.5 configs available in the local OpenPI checkout + +Local OpenPI checkout: `~/Desktop/pi0/openpi`, commit `15a9616`. +`grep` of `src/openpi/training/config.py` for real `pi05*` config names yields: + +``` +pi05_aloha +pi05_aloha_pen_uncap +pi05_droid +pi05_droid_finetune +pi05_full_droid_finetune +pi05_libero +``` + +There is **no** SO-101 Pi0.5 config upstream — one must be authored. Note also +that upstream's client class is named `OpenPIServicePolicyClient` and is +policy-agnostic at the transport level: Pi0 vs Pi0.5 is decided by which config +and checkpoint the **server** serves, not by the client. This is documented so +we do not conflate Pi0 and Pi0.5. + +## Modifications made by this repository + +Tracked incrementally in git history on `dev/pi05-so101`. Summary is maintained +here as work lands; see `git log --oneline` for the authoritative record. + +- Established independent project identity, `NOTICE`, `THIRD_PARTY.md`, this file. From f8a159118111f8bf2d969fde1cd6e3a5d9cd1906 Mon Sep 17 00:00:00 2001 From: ChepuriNatraj Date: Tue, 11 Aug 2026 21:14:26 +0530 Subject: [PATCH 2/5] feat(pi05): add Pi0.5 SO-101 training config and observation/action adapters - policies/pi05/config.py: repository-owned pi05_so101 TrainConfig mirroring OpenPI's TrainConfig schema (verified against local openpi commit 15a9616). Uses action_dim=6 (5 arm + gripper) and asset_id='so101' -- NOT borrowed from DROID(32)/Libero(7)/Aloha(7/14) to avoid silent incompatibility. to_openpi_config() lazily translates to a real OpenPI TrainConfig. - policies/pi05/adapters.py: deterministic SO-101<->Pi0.5 translation boundary wrapping leisaac.utils.robot_utils joint remapping (radians<->degrees, USD-limit<->motor-limit). Produces OpenPI obs {images/ uint8 224x224, state float64[6], prompt} and consumes action chunks (chunk,6) -> SO-101 radians tensor shape (chunk,1,6). Tests: tests/test_pi05_adapters.py (config + adapter shape/dtype/roundtrip). Sim-dependent tests skip cleanly when omni/leisaac unavailable. --- policies/pi05/adapters.py | 154 +++++++++++++++++++++++++++ policies/pi05/config.py | 212 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 policies/pi05/adapters.py create mode 100644 policies/pi05/config.py diff --git a/policies/pi05/adapters.py b/policies/pi05/adapters.py new file mode 100644 index 00000000..16032a50 --- /dev/null +++ b/policies/pi05/adapters.py @@ -0,0 +1,154 @@ +"""SO-101 <-> Pi0.5 observation and action adapters. + +These adapters translate between the *canonical SO-101 simulation representation* +(IsaacLab articulation joint positions/velocities in radians, gripper in +radians, camera RGB uint8 HWC) and the *Pi0.5 (OpenPI) representation* +(image dict "images/" uint8 224x224, "state" float64[6], "prompt" str, +and an "actions" array of shape (action_chunk, 6)). + +Grounded in (verified, not assumed): + - leisaac.assets.robots.lerobot.SO101_FOLLOWER_CFG joint names/order: + shoulder_pan, shoulder_lift, elbow_flex, wrist_flex, wrist_roll, gripper + - leisaac.utils.robot_utils.convert_leisaac_action_to_lerobot / + convert_lerobot_action_to_leisaac: radians<->degrees plus USD-joint-limit + <-> motor-limit remapping per joint. Pi0.5 is trained in the LeRobot + normalized joint space, so the gripper action is LeRobot-normalized [0,100]. + - leisaac.policy.service_policy_clients.OpenPIServicePolicyClient.get_action: + sends images resized to 224x224 with pad, state float64[6], prompt str; + receives {"actions": (chunk, 6)} and runs convert_lerobot_action_to_leisaac. + +Design rule (from the project spec): do NOT leak Pi0.5-specific formats into the +simulation. The simulation only ever sees the canonical SO-101 action tensor. +The adapters are the single translation boundary. +""" + +from __future__ import annotations + +import numpy as np +import torch + +# Canonical SO-101 joint order (radians for the 5 arm joints, radians for gripper). +SO101_JOINT_NAMES: tuple[str, ...] = ( + "shoulder_pan", + "shoulder_lift", + "elbow_flex", + "wrist_flex", + "wrist_roll", + "gripper", +) + +# Image size Pi0.5 (OpenPI) expects. +PI05_IMAGE_SIZE = 224 + + +def _to_uint8(img: np.ndarray) -> np.ndarray: + """Convert float [0,1] images to uint8, mirroring OpenPI image_tools.""" + if np.issubdtype(img.dtype, np.floating): + img = (255.0 * img).astype(np.uint8) + return img.astype(np.uint8) + + +def resize_with_pad(img: np.ndarray, height: int, width: int) -> np.ndarray: + """Resize HWC image to (height, width) preserving aspect ratio with zero pad. + + Mirrors leisaac.policy.openpi.image_tools.resize_with_pad (PIL-based). + Kept dependency-light here so adapters are unit-testable without PIL-heavy + imports; the production path can delegate to image_tools directly. + """ + from PIL import Image + + cur_h, cur_w = img.shape[:2] + if (cur_h, cur_w) == (height, width): + return img + ratio = max(cur_w / width, cur_h / height) + new_w, new_h = int(cur_w / ratio), int(cur_h / ratio) + resized = np.asarray(Image.fromarray(img).resize((new_w, new_h), Image.BILINEAR)) + out = np.zeros((height, width, img.shape[2]), dtype=img.dtype) + pad_h, pad_w = (height - new_h) // 2, (width - new_w) // 2 + out[pad_h : pad_h + new_h, pad_w : pad_w + new_w] = resized + return out + + +def build_pi05_observation( + camera_images: dict[str, np.ndarray], + joint_state: np.ndarray, + task_description: str, +) -> dict: + """Canonical SO-101 observation -> Pi0.5 (OpenPI) observation dict. + + Args: + camera_images: {camera_key: uint8 or float HWC image}. Keys are the + SO-101 camera names (e.g. "front", "wrist"). Order in the dict is + preserved into the OpenPI "images/" mapping. + joint_state: (6,) or (1,6) array in IsaacLab radians (canonical SO-101 + joint space). Converted to LeRobot-normalized [0,100] space. + task_description: language instruction string. + + Returns: + OpenPI observation dict: + {"images/": uint8 (224,224,3), "state": float64 (6,), + "prompt": str} + """ + from leisaac.utils import robot_utils + + obs: dict[str, object] = {} + for cam_key, img in camera_images.items(): + img_u8 = _to_uint8(img) + obs[f"images/{cam_key}"] = resize_with_pad(img_u8, PI05_IMAGE_SIZE, PI05_IMAGE_SIZE) + + js = np.asarray(joint_state).reshape(-1, 6) + # IsaacLab radians -> LeRobot normalized [0,100] (what Pi0.5 expects). + lerobot_state = robot_utils.convert_leisaac_action_to_lerobot(js) + obs["state"] = lerobot_state.astype(np.float64) + obs["prompt"] = task_description + return obs + + +def pi05_action_to_so101( + pi05_actions: np.ndarray, +) -> torch.Tensor: + """Pi0.5 action chunk -> canonical SO-101 action tensor. + + Args: + pi05_actions: array of shape (action_chunk, 6) in LeRobot-normalized + [0,100] space (5 arm + 1 gripper). + + Returns: + torch.Tensor shape (action_chunk, 1, 6) in IsaacLab radians + (canonical SO-101 action), matching the shape contract from + OpenPIServicePolicyClient.get_action. + """ + from leisaac.utils import robot_utils + + arr = np.asarray(pi05_actions).reshape(-1, 6) + # LeRobot normalized [0,100] -> IsaacLab radians. + so101 = robot_utils.convert_lerobot_action_to_leisaac(arr) + return torch.from_numpy(so101[:, None, :]) + + +class Pi05ObservationAdapter: + """Stateful wrapper around build_pi05_observation. + + Keeps the canonical camera key order so repeated calls produce a stable + layout for the policy server. + """ + + def __init__(self, camera_keys: tuple[str, ...] = ("front", "wrist")): + self.camera_keys = tuple(camera_keys) + + def __call__( + self, + camera_images: dict[str, np.ndarray], + joint_state: np.ndarray, + task_description: str, + ) -> dict: + # Re-order images to the canonical camera order. + ordered = {k: camera_images[k] for k in self.camera_keys if k in camera_images} + return build_pi05_observation(ordered, joint_state, task_description) + + +class Pi05ActionAdapter: + """Stateful wrapper around pi05_action_to_so101.""" + + def __call__(self, pi05_actions: np.ndarray) -> torch.Tensor: + return pi05_action_to_so101(pi05_actions) diff --git a/policies/pi05/config.py b/policies/pi05/config.py new file mode 100644 index 00000000..e0185a24 --- /dev/null +++ b/policies/pi05/config.py @@ -0,0 +1,212 @@ +"""SO-101 Pi0.5 training configuration. + +This module is the repository-owned training config for fine-tuning +Physical Intelligence's pi0.5 on SO-101 LeRobot datasets. + +Grounded in (verified against): + - Local OpenPI checkout ~/Desktop/pi0/openpi, commit 15a9616 + - openpi.training.config.TrainConfig schema (lines 465-557 of + src/openpi/training/config.py) + - openpi.training.config.pi0_config.Pi0Config(pi05=True, action_horizon=...) + +Key finding that drives this file +-------------------------------- +The upstream OpenPI configs that exist are: + pi05_aloha, pi05_aloha_pen_uncap, pi05_droid, pi05_droid_finetune, + pi05_full_droid_finetune, pi05_libero +NONE of them are SO-101. They use action spaces of 7/14 (Aloha), 32 (DROID), +or 7 (Libero). SO-101 has a 6-D action space (5 arm joints + 1 gripper) and +two cameras ("front", "wrist"). Therefore a dedicated config is REQUIRED and +must NOT reuse another dataset's norm stats or action_dim. + +The LeRobot <-> simulator joint remapping (radians/degrees + USD-limit to +motor-limit) is handled by leisaac.utils.robot_utils, which these adapters +wrap rather than replace (see ../adapters.py). + +This module is intentionally dependency-light: it defines plain dataclasses +that mirror OpenPI's schema so the repository is inspectable and the config +can be validated (pytest) WITHOUT installing openpi. At training time +(`training/train_pi05.py`) this config is translated into a real OpenPI +`TrainConfig` via `to_openpi_config()`. +""" + +from __future__ import annotations + +import dataclasses +from typing import Any + + +@dataclasses.dataclass +class Pi05ModelConfig: + """Mirror of openpi.training.config.pi0_config.Pi0Config, pi05 variant. + + Only the fields we actually set for SO-101 are modeled here. The + `to_openpi_config` helper maps these onto the real Pi0Config at runtime. + """ + + pi05: bool = True + # SO-101 action space: 5 arm joints + 1 gripper. + action_dim: int = 6 + # pi05 default action chunk length for single-arm; matches pi05_libero (10). + action_horizon: int = 16 + # SO-101 is a single-arm robot; no bi-arm. + discrete_state_input: bool = False + + +@dataclasses.dataclass +class Pi05DataConfig: + """SO-101 LeRobot dataset configuration for pi0.5 fine-tuning. + + Mirrors openpi.training.config.LeRobotDROIDDataConfig semantics but + targets an SO-101 repo_id with SO-101-specific repacking. + """ + + repo_id: str = "your_hf_username/so101_pi05_demos" + # Norm-stats asset id. Reuses pi05_base assets dir; the asset_id selects the + # normalization statistics bundle to load (must match the action_dim=6 + # SO-101 dataset, NOT a DROID/Libero dataset). + assets_dir: str = "gs://openpi-assets/checkpoints/pi05_base/assets" + asset_id: str = "so101" + prompt_from_task: bool = True + # Camera keys present in the SO-101 dataset. Order matters: OpenPI expects a + # stable, ordered image dict. + image_keys: tuple[str, ...] = ("front", "wrist") + # SO-101 state vector = joint_pos (5) concatenated with gripper (1). + state_dim: int = 6 + fps: int = 30 + + +@dataclasses.dataclass +class Pi05TrainConfig: + """Repository-owned SO-101 pi0.5 fine-tuning config. + + Field-for-field mirror of the relevant subset of + openpi.training.config.TrainConfig so it can be validated standalone. + """ + + name: str = "pi05_so101" + project_name: str = "so101-pi05" + exp_name: str = "lift_cube_v1" + + model: Pi05ModelConfig = dataclasses.field(default_factory=Pi05ModelConfig) + data: Pi05DataConfig = dataclasses.field(default_factory=Pi05DataConfig) + + # Start from the released pi05_base weights. + weight_loader: str = "gs://openpi-assets/checkpoints/pi05_base/params" + pytorch_weight_path: str | None = None + + pytorch_training_precision: str = "bfloat16" + lr_schedule_warmup_steps: int = 10_000 + lr_schedule_peak_lr: float = 5e-5 + lr_schedule_decay_steps: int = 1_000_000 + lr_schedule_decay_lr: float = 5e-5 + optimizer_clip_gradient_norm: float = 1.0 + ema_decay: float | None = 0.999 + + seed: int = 42 + batch_size: int = 64 + num_workers: int = 4 + num_train_steps: int = 20_000 + + log_interval: int = 100 + save_interval: int = 1_000 + keep_period: int | None = 5_000 + + overwrite: bool = False + resume: bool = False + wandb_enabled: bool = True + + assets_base_dir: str = "./assets" + checkpoint_base_dir: str = "./checkpoints" + policy_metadata: dict[str, Any] | None = dataclasses.field( + default_factory=lambda: {"reset_pose": [0.0, -1.0, 0.9, 0.5, 0.0, -0.1]} + ) + + @property + def assets_dirs(self) -> str: + return f"{self.assets_base_dir}/{self.name}" + + @property + def checkpoint_dir(self) -> str: + return f"{self.checkpoint_base_dir}/{self.name}/{self.exp_name}" + + +def default_pi05_so101_config() -> Pi05TrainConfig: + """Return a fresh, default SO-101 pi0.5 training config.""" + return Pi05TrainConfig() + + +def to_openpi_config(cfg: Pi05TrainConfig): + """Translate this repository config into a real OpenPI TrainConfig. + + Imports openpi lazily so this module stays importable for validation + tests even when openpi is not installed in the active environment. + + Args: + cfg: repository-owned SO-101 pi0.5 config. + + Returns: + An instance of openpi.training.config.TrainConfig. + """ + import pathlib + + from openpi.training import config as openpi_config + from openpi.training.config import ( + AdamW, + AssetsConfig, + CosineDecaySchedule, + LeRobotDROIDDataConfig, + Pi0Config, + ) + from openpi.training.weight_loaders import CheckpointWeightLoader + + model = Pi0Config( + pi05=True, + action_dim=cfg.model.action_dim, + action_horizon=cfg.model.action_horizon, + ) + if cfg.model.discrete_state_input: + model = dataclasses.replace(model, discrete_state_input=True) + + # LeRobotDROIDDataConfig is OpenPI's generic custom-LeRobot-dataset factory + # (the docstring explicitly says use it for your own converted dataset). + # It carries repo_id + norm-stats AssetsConfig; the SO-101-specific + # observation/action repacking transform is attached by the dataset layer. + data = LeRobotDROIDDataConfig( + repo_id=cfg.data.repo_id, + assets=AssetsConfig( + assets_dir=cfg.data.assets_dir, + asset_id=cfg.data.asset_id, + ), + base_config=openpi_config.DataConfig(prompt_from_task=cfg.data.prompt_from_task), + ) + + return openpi_config.TrainConfig( + name=cfg.name, + project_name=cfg.project_name, + exp_name=cfg.exp_name, + model=model, + data=data, + weight_loader=CheckpointWeightLoader(cfg.weight_loader), + pytorch_weight_path=cfg.pytorch_weight_path, + pytorch_training_precision=cfg.pytorch_training_precision, + lr_schedule=CosineDecaySchedule( + warmup_steps=cfg.lr_schedule_warmup_steps, + peak_lr=cfg.lr_schedule_peak_lr, + decay_steps=cfg.lr_schedule_decay_steps, + decay_lr=cfg.lr_schedule_decay_lr, + ), + optimizer=AdamW(clip_gradient_norm=cfg.optimizer_clip_gradient_norm), + ema_decay=cfg.ema_decay, + seed=cfg.seed, + batch_size=cfg.batch_size, + num_workers=cfg.num_workers, + num_train_steps=cfg.num_train_steps, + log_interval=cfg.log_interval, + save_interval=cfg.save_interval, + keep_period=cfg.keep_period, + overwrite=cfg.overwrite, + resume=cfg.resume, + wandb_enabled=cfg.wandb_enabled, + policy_metadata=cfg.policy_metadata, + ) From dec9a12b813c5d625ddb0f6983014e4107ed59f1 Mon Sep 17 00:00:00 2001 From: ChepuriNatraj Date: Tue, 11 Aug 2026 22:09:16 +0530 Subject: [PATCH 3/5] feat: add canonical observation/action contracts and generic policy/task interfaces Establishes the model-independent architecture boundaries: - robots/so101: SO-101 spec (6-DoF joint order, limits, rest pose, cameras) - environments: canonical Observation + RobotAction dataclasses with validation - policies/base: generic Policy ABC + PolicyWrapper (sim/eval depend ONLY on this) - tasks: framework-agnostic Task interface + TaskRegistry - policies/pi05/policy: Pi05Policy wrapper over the obs/action adapters (server seam clearly marked, no imaginary model calls) - data_collection / dataset / training / inference / evaluation: layer seams that KEEP existing LeIsaac infrastructure and document the integration points tests/test_architecture.py: 10 architecture-level tests (no Isaac Sim needed), all passing. Runtime-blocked pieces are marked, not faked. --- .gitignore | 5 +- data_collection/__init__.py | 70 ++++++++++++++ dataset/__init__.py | 67 ++++++++++++++ environments/action.py | 56 ++++++++++++ environments/observation.py | 56 ++++++++++++ evaluation/__init__.py | 177 ++++++++++++++++++++++++++++++++++++ inference/__init__.py | 71 +++++++++++++++ policies/base/__init__.py | 79 ++++++++++++++++ policies/pi05/policy.py | 75 +++++++++++++++ robots/so101/__init__.py | 115 +++++++++++++++++++++++ tasks/__init__.py | 81 +++++++++++++++++ tests/test_architecture.py | 110 ++++++++++++++++++++++ training/__init__.py | 52 +++++++++++ 13 files changed, 1013 insertions(+), 1 deletion(-) create mode 100644 data_collection/__init__.py create mode 100644 dataset/__init__.py create mode 100644 environments/action.py create mode 100644 environments/observation.py create mode 100644 evaluation/__init__.py create mode 100644 inference/__init__.py create mode 100644 policies/base/__init__.py create mode 100644 policies/pi05/policy.py create mode 100644 robots/so101/__init__.py create mode 100644 tasks/__init__.py create mode 100644 tests/test_architecture.py create mode 100644 training/__init__.py diff --git a/.gitignore b/.gitignore index ed4d05fe..5558411c 100644 --- a/.gitignore +++ b/.gitignore @@ -59,7 +59,10 @@ _build /datasets/ # Tests -tests/ +# NOTE: this repository commits tests/ (architecture + pipeline tests are part of +# the deliverable). Only ignore compiled/test caches, never the test sources. +**/__pycache__/ +**/.pytest_cache/ # Docker history .isaac-lab-docker-history diff --git a/data_collection/__init__.py b/data_collection/__init__.py new file mode 100644 index 00000000..2f2e89b9 --- /dev/null +++ b/data_collection/__init__.py @@ -0,0 +1,70 @@ +"""Data collection architecture. + +Control source -> Canonical Action -> Recorder +------------------------------------------------- +Teleoperator (keyboard / gamepad / SO-101 leader / lekiwi) +ScriptedPolicy (state-machine expert, e.g. pick_orange) +Policy (Pi05Policy / Gr00tPolicy for replay demos) + +The recorder does NOT care where the action came from. It consumes a canonical +RobotAction each step and appends it to a raw episode. + +Existing reusable LeIsaac infrastructure (KEEP): + - leisaac.enhance.managers.lerobot_recorder_manager.LeRobotRecorderManager + (IsaacLab RecorderManager subclass; writes LeRobot frames per step) + - leisaac.enhance.datasets.hdf5_dataset_file_handler.StreamingHDF5DatasetFileHandler + (raw HDF5 episode stream) + - leisaac.devices.* (keyboard, gamepad, lerobot SO-101 leader, lekiwi) + - scripts/datagen/state_machine/{generate,replay}.py (scripted expert) + +This module provides the canonical interface so new control sources plug in +without touching the recorder. + +Runtime status: + - Interface + recorder seam : IMPLEMENTED (pure Python) + - LeIsaac recorder/manager : VERIFIED in source (needs Isaac Sim to run) + - Teleoperation devices : VERIFIED in source (SO-101 leader keyboard/gamepad) + - Live demo recording : BLOCKED by Isaac Sim runtime (omni.isaac.ml_archive) +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Iterator + +from environments.action import RobotAction +from environments.observation import Observation + + +class ControlSource(ABC): + """Produces canonical RobotActions given the current observation. + + Implemented by teleoperators and scripted/scripted-expert policies. + """ + + @abstractmethod + def get_action(self, observation: Observation) -> RobotAction: + ... + + def reset(self) -> None: + return None + + +class EpisodeRecorder(ABC): + """Consumes canonical actions + observations and stores a raw episode.""" + + @abstractmethod + def reset(self) -> None: + ... + + @abstractmethod + def record_step(self, observation: Observation, action: RobotAction) -> None: + ... + + @abstractmethod + def finalize(self) -> dict: + """Return episode metadata (id, length, instruction, paths).""" + + def __iter__(self) -> Iterator[tuple[Observation, RobotAction]]: + # Optional: default recorders may not support replay. + raise NotImplementedError diff --git a/dataset/__init__.py b/dataset/__init__.py new file mode 100644 index 00000000..f9fcdcf2 --- /dev/null +++ b/dataset/__init__.py @@ -0,0 +1,67 @@ +"""Dataset pipeline architecture. + +Raw Simulation Episodes (HDF5 / IsaacLab EpisodeData) + ↓ +Raw Dataset Schema + ↓ +Converter (IsaacLab HDF5 -> LeRobot Dataset v3) + ↓ +LeRobot Dataset + ↓ +Dataset Validator + +Existing reusable LeIsaac infrastructure (KEEP, do not rewrite): + - scripts/convert/isaaclab2lerobotv3.py : HDF5 -> LeRobot v3 (lerobot==0.4.2) + - scripts/convert/isaaclab2lerobot.py : earlier HDF5 -> LeRobot converter + - scripts/convert/lerobot2isaaclab.py : reverse direction + - leisaac.enhance.datasets.lerobot_dataset_handler.LeRobotDatasetHandler + (builds the LeRobot feature schema from an IsaacLab env) + - leisaac.enhance.datasets.hdf5_dataset_file_handler + (streaming HDF5 raw-episode writer) + +This module defines the schema/validation seam so the canonical representation +(raw episode) is distinct from the LeRobot representation. + +Mapping documented for SO-101 (verified against LeIsaac source): + camera "front" -> observation.images.front (uint8 HWC, 640x480) + camera "wrist" -> observation.images.wrist (uint8 HWC, 640x480) + robot state -> observation.state (6,) radians + action -> action (6,) radians + instruction -> task_description string + timestamps -> per-frame sim time + episodes -> HDF5 group per demo; LeRobot episode index + +Runtime status: + - Schema/validation seam : IMPLEMENTED (pure Python) + - LeIsaac HDF5 converters : VERIFIED in source (need Isaac Sim + lerobot==0.4.2) + - Live conversion : BLOCKED by Isaac Sim runtime +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class RawEpisodeSchema: + """Canonical raw-episode description (before LeRobot conversion).""" + + camera_keys: tuple[str, ...] = ("front", "wrist") + image_shape: tuple[int, int, int] = (480, 640, 3) # (h, w, c) + state_dim: int = 6 + action_dim: int = 6 + fps: int = 30 + has_instruction: bool = True + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class LerobotDatasetRef: + """Pointer to a produced LeRobot dataset.""" + + repo_id: str + root: str + fps: int = 30 + episode_count: int = 0 + features: dict[str, Any] = field(default_factory=dict) diff --git a/environments/action.py b/environments/action.py new file mode 100644 index 00000000..8dd072e7 --- /dev/null +++ b/environments/action.py @@ -0,0 +1,56 @@ +"""Canonical action contract. + +Policy output -> Policy Action Adapter -> Canonical SO-101 Action -> Environment +-> Robot Controller. + +Keeping a robot-side action representation lets future policies (Pi0.5, GR00T, +scripted, human) be swapped without touching the controller or environment. + +Runtime status: data-structure layer is pure-Python and unit-tested. Live +application to the IsaacLab articulation requires the Isaac Sim runtime. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np + + +@dataclass +class RobotAction: + """Canonical SO-101 action. + + Attributes: + joints: (5,) or (action_horizon, 5) arm joint commands in canonical + SO-101 space (IsaacLab radians). A 2-D array represents an action + *chunk* (multiple future steps), matching Pi0.5's chunked output. + gripper: (1,) or (action_horizon, 1) gripper command. + timestamp: when produced. + metadata: free-form (policy name, chunk index, ...). + """ + + joints: np.ndarray = field(default_factory=lambda: np.zeros((1, 5), dtype=np.float32)) + gripper: np.ndarray = field(default_factory=lambda: np.zeros((1, 1), dtype=np.float32)) + timestamp: float = 0.0 + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def action_horizon(self) -> int: + return max(self.joints.shape[0], self.gripper.shape[0]) + + @property + def as_vector(self) -> np.ndarray: + """Concatenate into the canonical (horizon, 6) SO-101 vector.""" + j = np.asarray(self.joints).reshape(self.action_horizon, -1) + g = np.asarray(self.gripper).reshape(self.action_horizon, -1) + return np.concatenate([j, g], axis=-1) + + def validate(self) -> None: + if self.joints.shape[-1] != 5: + raise ValueError(f"joints must be (H,5), got {self.joints.shape}") + if self.gripper.shape[-1] != 1: + raise ValueError(f"gripper must be (H,1), got {self.gripper.shape}") + if self.joints.shape[0] != self.gripper.shape[0]: + raise ValueError("joints and gripper horizons must match") diff --git a/environments/observation.py b/environments/observation.py new file mode 100644 index 00000000..5a0ef875 --- /dev/null +++ b/environments/observation.py @@ -0,0 +1,56 @@ +"""Canonical observation contract. + +The simulation produces a *model-independent* observation. Policy-specific +adapters (see policies/pi05/adapters.py) translate this into whatever the +policy (Pi0.5, GR00T, scripted) expects. + +Architectural rule (from the project spec): + + IsaacLab + ↓ + Canonical Observation <-- defined here + ↓ + Policy-specific Adapter <-- policies/*/adapters.py + +NOT: IsaacLab -> Pi0.5 tensors everywhere. + +Runtime status: data-structure layer is pure-Python and unit-tested. Live +population from IsaacLab sensors requires the Isaac Sim runtime. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np + + +@dataclass +class Observation: + """Canonical SO-101 observation. + + Attributes: + images: {camera_name: uint8 HWC (h, w, 3)}. Keys match + robots.so101.SO101_CAMERA_KEYS by convention ("front", "wrist"). + robot_state: (state_dim,) joint positions + gripper in the canonical + SO-101 representation (IsaacLab radians). + timestamp: seconds (float). May be simulation time. + metadata: free-form dict (task name, episode id, env id, ...). + """ + + images: dict[str, np.ndarray] = field(default_factory=dict) + robot_state: np.ndarray = field(default_factory=lambda: np.zeros(6, dtype=np.float32)) + timestamp: float = 0.0 + metadata: dict[str, Any] = field(default_factory=dict) + + def image_keys(self) -> list[str]: + return sorted(self.images.keys()) + + def validate(self) -> None: + """Raise if the contract is violated (used by tests and recorders).""" + if self.robot_state.shape[-1] != 6: + raise ValueError(f"robot_state must be 6-dim, got {self.robot_state.shape}") + for key, img in self.images.items(): + if img.ndim != 3 or img.shape[-1] != 3: + raise ValueError(f"image {key!r} must be HWC uint8, got {img.shape}") diff --git a/evaluation/__init__.py b/evaluation/__init__.py new file mode 100644 index 00000000..715c3a55 --- /dev/null +++ b/evaluation/__init__.py @@ -0,0 +1,177 @@ +"""Evaluation engine. + +Evaluation Runner + ↓ +Environment Reset + ↓ +Task Instruction + ↓ +Policy Inference (generic Policy interface) + ↓ +SO-101 Execution + ↓ +Success / Failure Check (Task contract) + ↓ +Metrics + +The evaluator is model-independent: it drives any ``Policy`` over any registered +``Task``. It does not know about Pi0.5, GR00T, or training internals. + +Metrics recorded per episode: + episode_id, task, checkpoint, seed, + success (bool), failure_reason, termination_reason, + episode_length, completion_time, inference_latency (mean/last). + +Failure reasons are structured where detectable: + timeout | object_dropped | out_of_workspace | collision | + joint_limit | incorrect_target | policy_divergence | sim_instability | unknown + +Runtime status: + - Evaluator + metrics : IMPLEMENTED (pure Python, unit-tested) + - Live multi-episode eval : BLOCKED by Isaac Sim runtime +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict, dataclass, field +from typing import Any, Callable + +from environments.action import RobotAction +from environments.observation import Observation +from policies.base import Policy +from tasks import Task + + +@dataclass +class EpisodeMetric: + episode_id: int + task: str + checkpoint: str + seed: int + success: bool + failure_reason: str = "none" + termination_reason: str = "unknown" + episode_length: int = 0 + completion_time_s: float = 0.0 + mean_inference_latency_ms: float = 0.0 + + def as_dict(self) -> dict: + return asdict(self) + + +FAILURE_REASONS = ( + "timeout", + "object_dropped", + "out_of_workspace", + "collision", + "joint_limit", + "incorrect_target", + "policy_divergence", + "sim_instability", + "unknown", +) + + +@dataclass +class EvaluationReport: + task: str + checkpoint: str + episodes: list[EpisodeMetric] = field(default_factory=list) + + @property + def success_rate(self) -> float: + if not self.episodes: + return 0.0 + return sum(1 for e in self.episodes if e.success) / len(self.episodes) + + def to_json(self) -> str: + return json.dumps( + { + "task": self.task, + "checkpoint": self.checkpoint, + "success_rate": self.success_rate, + "n_episodes": len(self.episodes), + "episodes": [e.as_dict() for e in self.episodes], + }, + indent=2, + ) + + def to_csv(self) -> str: + header = "episode_id,task,checkpoint,seed,success,failure_reason,termination_reason,episode_length,completion_time_s,mean_inference_latency_ms" + rows = [header] + for e in self.episodes: + d = e.as_dict() + rows.append( + ",".join( + str(d[k]) + for k in ( + "episode_id", + "task", + "checkpoint", + "seed", + "success", + "failure_reason", + "termination_reason", + "episode_length", + "completion_time_s", + "mean_inference_latency_ms", + ) + ) + ) + return "\n".join(rows) + + +def evaluate_episode( + policy: Policy, + task: Task, + reset_env: Callable[[], None], + get_observation: Callable[[], Observation], + step: Callable[[RobotAction], None], + episode_id: int, + checkpoint: str, + seed: int, + max_steps: int = 300, +) -> EpisodeMetric: + """Run one evaluation episode and return its metric.""" + reset_env() + task.reset(None) # task may reconfigure object/target poses + instruction = task.get_instruction() + policy.reset() + + start = time.time() + latencies: list[float] = [] + done = False + step_i = 0 + termination = "max_steps" + while step_i < max_steps and not done: + t0 = time.time() + obs = get_observation() + obs.validate() + action = policy.predict_action(obs, instruction) + action.validate() + latencies.append((time.time() - t0) * 1000.0) + step(action) + step_i += action.action_horizon + if task.is_success(None): + done = True + termination = "success" + elif task.is_failure(None): + done = True + termination = "failure" + elapsed = time.time() - start + + success = termination == "success" + return EpisodeMetric( + episode_id=episode_id, + task=task.__class__.__name__, + checkpoint=checkpoint, + seed=seed, + success=success, + failure_reason="none" if success else "unknown", + termination_reason=termination, + episode_length=step_i, + completion_time_s=elapsed, + mean_inference_latency_ms=(sum(latencies) / len(latencies) if latencies else 0.0), + ) diff --git a/inference/__init__.py b/inference/__init__.py new file mode 100644 index 00000000..75ef5ea8 --- /dev/null +++ b/inference/__init__.py @@ -0,0 +1,71 @@ +"""Closed-loop inference architecture. + +while not done: + observation = env.get_observation() # -> canonical Observation + action = policy.predict_action( # -> canonical RobotAction + observation, instruction) + env.step(action) # apply chunk, advance sim + +The inference runner is independent of training code and of any specific policy: +it only depends on the generic ``Policy`` interface and the environment's +``get_observation`` / ``step`` contract. + +Two closed-loop execution patterns (documented, choose per task): + (1) Predict N actions, execute all N, replan. + (2) Predict N actions, execute K < N, replan. +The action chunk is consumed incrementally; K is the execution horizon. + +Existing reusable LeIsaac infrastructure (KEEP): + - scripts/evaluation/policy_inference.py (drives a policy against an env) + - leisaac.policy.service_policy_clients.* (transport clients; Pi0.5 via + OpenPIServicePolicyClient) + - leisaac.datagen.state_machine.{generate,replay}.py (expert replay baseline) + +Runtime status: + - Inference runner seam : IMPLEMENTED (pure Python) + - Policy adapters : IMPLEMENTED (Pi0.5 obs/action adapters) + - LeIsaac inference script: VERIFIED in source (needs Isaac Sim to run) + - Live closed-loop run : BLOCKED by Isaac Sim runtime +""" + +from __future__ import annotations + +from environments.action import RobotAction +from environments.observation import Observation +from policies.base import Policy + + +def run_closed_loop( + policy: Policy, + get_observation, + step, + instruction: str, + max_episode_steps: int = 300, + execution_horizon: int | None = None, +) -> dict: + """Generic closed-loop rollout. + + Args: + policy: any ``Policy`` (Pi05Policy, scripted, ...). + get_observation: callable -> Observation. + step: callable(RobotAction) -> None (advances sim by the chunk). + instruction: language task instruction. + max_episode_steps: safety cap on outer loop iterations. + execution_horizon: K in "execute K None: + """Clear any hidden state / action-chunk history.""" + + @abstractmethod + def predict_action(self, observation: Observation, instruction: str) -> RobotAction: + """Predict the next SO-101 action chunk for ``observation``. + + Args: + observation: canonical Observation from the environment. + instruction: language task instruction for this episode. + + Returns: + A RobotAction (may be an action chunk of length >= 1). + """ + + def step(self, observation: Observation, instruction: str) -> RobotAction: + """Convenience: alias for ``predict_action`` (kept for readability).""" + return self.predict_action(observation, instruction) + + +class PolicyWrapper(Policy): + """Base class for adapters that wrap a backend policy client. + + Concrete subclasses (Pi05Policy, Gr00tPolicy) translate the canonical + Observation -> backend-specific request and the backend response -> + canonical RobotAction. + """ + + kind = "wrapper" + + def __init__(self, backend: Any | None = None): + self._backend = backend + + @property + def backend(self) -> Any: + if self._backend is None: + raise RuntimeError( + f"{self.__class__.__name__} has no backend configured. " + "See docs/pipeline.md for setup." + ) + return self._backend + + def reset(self) -> None: + # Default: stateless policies need no reset. Subclasses may override. + return None diff --git a/policies/pi05/policy.py b/policies/pi05/policy.py new file mode 100644 index 00000000..695fb858 --- /dev/null +++ b/policies/pi05/policy.py @@ -0,0 +1,75 @@ +"""Pi0.5 policy integration boundary. + +Pi05Policy adapts the generic :class:`policies.base.Policy` contract to a +Physical-Intelligence OpenPI pi0.5 backend. + +Two deployment shapes are supported by the OpenPI reference design (and by +LeIsaac's existing ``OpenPIServicePolicyClient``): + + A) Remote policy server (OpenPI serves pi0.5 over websocket). Pi05Policy sends + a canonical Observation (via the observation adapter) and receives an action + chunk. This is the shape LeIsaac already implements and what we reuse. + + B) In-process fine-tuning/inference (JAX/PyTorch). Used for training and for + headless eval without a separate server. Wired in training/ and inference/. + +This module deliberately does NOT contain imaginary model calls. Where the +exact OpenPI server message contract has not been re-verified against the +deployed server, the call is guarded and documented. + +Integration status: + - Observation adapter : IMPLEMENTED (verified against LeIsaac joint math) + - Action adapter : IMPLEMENTED (verified against LeIsaac joint math) + - Training config : IMPLEMENTED (pi05_so101, action_dim=6) + - Policy wrapper class : IMPLEMENTED (structure + server-client seam) + - Live server call : BLOCKED -- requires running OpenPI pi0.5 server + - In-process load : BLOCKED -- requires OpenPI installed in runtime +""" + +from __future__ import annotations + +from typing import Any + +from environments.action import RobotAction +from environments.observation import Observation +from policies.base import PolicyWrapper +from policies.pi05.adapters import Pi05ActionAdapter, Pi05ObservationAdapter +from policies.pi05.config import Pi05TrainConfig, default_pi05_so101_config + + +class Pi05Policy(PolicyWrapper): + """Generic-policy wrapper around a pi0.5 backend (server or in-process).""" + + kind = "pi05" + + def __init__( + self, + backend: Any | None = None, + camera_keys: tuple[str, ...] = ("front", "wrist"), + config: Pi05TrainConfig | None = None, + ): + super().__init__(backend=backend) + self.config = config or default_pi05_so101_config() + self._obs_adapter = Pi05ObservationAdapter(camera_keys=camera_keys) + self._act_adapter = Pi05ActionAdapter() + + def predict_action(self, observation: Observation, instruction: str) -> RobotAction: + observation.validate() + # Translate canonical Observation -> OpenPI request dict. + pi05_obs = self._obs_adapter( + camera_images=observation.images, + joint_state=observation.robot_state, + task_description=instruction, + ) + # TODO(openpi-integration): verify the exact server request/response + # contract against the deployed OpenPI pi0.5 server before enabling. + # LeIsaac's OpenPIServicePolicyClient sends {"images/": uint8 + # 224x224, "state": float64[6], "prompt": str} and receives + # {"actions": (chunk, 6)}. Reuse that mapping. + if self._backend is None: + raise RuntimeError( + "Pi05Policy has no backend. Start an OpenPI pi0.5 server or " + "load an in-process model. See docs/pipeline.md > Pi0.5 Integration." + ) + pi05_actions = self._backend.infer(pi05_obs)["actions"] # (chunk, 6) + return self._act_adapter(pi05_actions) diff --git a/robots/so101/__init__.py b/robots/so101/__init__.py new file mode 100644 index 00000000..44b362ce --- /dev/null +++ b/robots/so101/__init__.py @@ -0,0 +1,115 @@ +"""SO-101 digital twin boundary. + +This module is the single entry point for SO-101 robot-specific knowledge: +joint names/order, joint limits, actuator configuration, gripper, default pose, +and robot-side observation extraction. + +It MUST NOT contain any Pi0.5 / GR00T / LeRobot-training specific code. Those +layers depend on this one, never the reverse. + +The authoritative joint configuration lives in the upstream LeIsaac module +``leisaac.assets.robots.lerobot`` (ArticulationCfg SO101_FOLLOWER_CFG) and the +joint-limit tables in ``leisaac.assets.robots.lerobot``. This module re-exports +them so the rest of the repository has one stable import surface. + +Runtime status: + SO-101 asset / joint config / actuators / gripper -> VERIFIED in source + Spawn / actuation in live sim -> BLOCKED by Isaac Sim + omni.isaac.ml_archive + extension defect + (see docs/upstream.md / README) +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Final + +# --------------------------------------------------------------------------- +# Canonical SO-101 joint order (MUST match leisaac.assets.robots.lerobot). +# 5 arm joints + 1 gripper = 6-DoF action/state vector. +# --------------------------------------------------------------------------- +SO101_JOINT_NAMES: Final[tuple[str, ...]] = ( + "shoulder_pan", + "shoulder_lift", + "elbow_flex", + "wrist_flex", + "wrist_roll", + "gripper", +) + +# Degrees (written in the SO-101 follower USD). +SO101_USD_JOINT_LIMITS_DEG: Final[dict[str, tuple[float, float]]] = { + "shoulder_pan": (-110.0, 110.0), + "shoulder_lift": (-100.0, 100.0), + "elbow_flex": (-100.0, 90.0), + "wrist_flex": (-95.0, 95.0), + "wrist_roll": (-160.0, 160.0), + "gripper": (-10.0, 100.0), +} + +# Motor limits in the normalized LeRobot space (what Pi0.5 is trained on). +SO101_MOTOR_LIMITS: Final[dict[str, tuple[float, float]]] = { + "shoulder_pan": (-100.0, 100.0), + "shoulder_lift": (-100.0, 100.0), + "elbow_flex": (-100.0, 100.0), + "wrist_flex": (-100.0, 100.0), + "wrist_roll": (-100.0, 100.0), + "gripper": (0.0, 100.0), +} + +SO101_ACTION_DIM: Final[int] = 6 +SO101_STATE_DIM: Final[int] = 6 + +# Default rest pose in motor-normalized degrees (from +# SO101_FOLLOWER_REST_POSE_RANGE midpoint in leisaac source). +SO101_REST_POSE_DEG: Final[dict[str, float]] = { + "shoulder_pan": 0.0, + "shoulder_lift": -100.0, + "elbow_flex": 90.0, + "wrist_flex": 50.0, + "wrist_roll": 0.0, + "gripper": -10.0, +} + +# Default camera keys used by the SO-101 follower rig. +# (These match the LiftCube / template camera prims in LeIsaac.) +SO101_CAMERA_KEYS: Final[tuple[str, ...]] = ("front", "wrist") + + +@dataclass(frozen=True) +class SO101RobotSpec: + """Immutable description of the SO-101 follower digital twin.""" + + joint_names: tuple[str, ...] = SO101_JOINT_NAMES + action_dim: int = SO101_ACTION_DIM + state_dim: int = SO101_STATE_DIM + usd_joint_limits_deg: dict[str, tuple[float, float]] = field( + default_factory=lambda: dict(SO101_USD_JOINT_LIMITS_DEG) + ) + motor_limits: dict[str, tuple[float, float]] = field( + default_factory=lambda: dict(SO101_MOTOR_LIMITS) + ) + rest_pose_deg: dict[str, float] = field( + default_factory=lambda: dict(SO101_REST_POSE_DEG) + ) + camera_keys: tuple[str, ...] = SO101_CAMERA_KEYS + + @property + def joint_index(self) -> dict[str, int]: + return {name: i for i, name in enumerate(self.joint_names)} + + +# A ready-to-use singleton spec. +SO101_SPEC = SO101RobotSpec() + + +def load_isaac_articulation_cfg(): + """Return the authoritative IsaacLab ArticulationCfg for SO-101. + + Delegates to upstream LeIsaac so there is one source of truth. Requires the + Isaac Sim runtime (``omni``) to import; call only inside a launched app. + """ + from leisaac.assets.robots.lerobot import SO101_FOLLOWER_CFG + + return SO101_FOLLOWER_CFG diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 00000000..83786571 --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,81 @@ +"""Task interface. + +A Task owns task-specific behaviour: initial scene/object configuration, +the language instruction, and success/failure detection. It does NOT own the +robot, the scene rig, or the physics -- those come from the environment layer. +This keeps tasks pluggable without rewriting the robot or environment. + +Conceptually (adapted to the IsaacLab ManagerBasedEnv the repo already uses): + + class Task: + def reset(self, env): ... + def get_instruction(self) -> str: ... + def is_success(self, env) -> bool: ... + def is_failure(self, env) -> bool: ... + +The canonical upstream tasks (lift_cube, pick_orange, ...) implement this shape +via IsaacLab termination/observation terms and a ``task_description`` field on +the env config. We provide a thin, framework-agnostic TaskProtocol so new tasks +can be authored against a stable contract and the evaluator can drive any task. + +Runtime status: interface is pure-Python and unit-tested. Live success/failure +checks run inside the Isaac Sim environment. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class TaskContext: + """Lightweight, serializable description of a task instance.""" + + name: str + instruction: str + # Optional structured config (object poses, target poses, seeds, ...). + params: dict[str, Any] = field(default_factory=dict) + + +class Task(ABC): + """Framework-agnostic task contract.""" + + @abstractmethod + def reset(self, env: Any) -> None: + """Configure the scene/objects for a new episode in ``env``.""" + + @abstractmethod + def get_instruction(self) -> str: + """Return the language instruction for this task.""" + + @abstractmethod + def is_success(self, env: Any) -> bool: + """Return True when the task success condition is met.""" + + def is_failure(self, env: Any) -> bool: + """Return True on a terminal failure (optional; default never fails).""" + return False + + +class TaskRegistry: + """Registry so tasks can be referenced by name (string) in configs/scripts.""" + + def __init__(self) -> None: + self._tasks: dict[str, type[Task]] = {} + + def register(self, name: str, task_cls: type[Task]) -> None: + if name in self._tasks: + raise ValueError(f"Task {name!r} already registered") + self._tasks[name] = task_cls + + def get(self, name: str) -> type[Task]: + return self._tasks[name] + + def names(self) -> list[str]: + return sorted(self._tasks) + + +# Global registry used across the repo. +REGISTRY = TaskRegistry() diff --git a/tests/test_architecture.py b/tests/test_architecture.py new file mode 100644 index 00000000..1ae98c5d --- /dev/null +++ b/tests/test_architecture.py @@ -0,0 +1,110 @@ +"""Architecture-level tests that do NOT require Isaac Sim / OpenPI. + +These validate the structural contracts (data classes, interfaces, registries, +adapters, config) so the repo is testable in lightweight CI. Sim/OpenPI-bound +paths are exercised separately once the runtimes are available. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from environments.action import RobotAction +from environments.observation import Observation +from policies.base import Policy +from policies.pi05.config import default_pi05_so101_config +from tasks import REGISTRY, Task, TaskContext +from robots.so101 import SO101_SPEC, SO101_JOINT_NAMES + + +# --- Robot layer ----------------------------------------------------------- +def test_so101_spec_is_6dof_and_consistent(): + assert SO101_SPEC.action_dim == 6 + assert SO101_SPEC.state_dim == 6 + assert len(SO101_SPEC.joint_names) == 6 + assert set(SO101_SPEC.joint_names) == set(SO101_JOINT_NAMES) + assert len(SO101_SPEC.joint_index) == 6 + + +# --- Observation / Action contracts ---------------------------------------- +def test_observation_validate_rejects_bad_state(): + obs = Observation(robot_state=np.zeros(5)) + with pytest.raises(ValueError): + obs.validate() + + +def test_observation_validate_ok(): + obs = Observation( + images={"front": np.zeros((480, 640, 3), np.uint8)}, + robot_state=np.zeros(6), + ) + obs.validate() + assert obs.image_keys() == ["front"] + + +def test_action_as_vector_concatenates_joints_gripper(): + a = RobotAction( + joints=np.zeros((10, 5)), + gripper=np.ones((10, 1)), + ) + vec = a.as_vector + assert vec.shape == (10, 6) + assert np.allclose(vec[:, 5], 1.0) + assert a.action_horizon == 10 + a.validate() + + +def test_action_validate_rejects_mismatched_horizon(): + a = RobotAction(joints=np.zeros((3, 5)), gripper=np.zeros((2, 1))) + with pytest.raises(ValueError): + a.validate() + + +# --- Generic policy interface ---------------------------------------------- +def test_policy_is_abstract(): + with pytest.raises(TypeError): + Policy() + + +def test_policy_wrapper_guards_missing_backend(): + class Dummy(Policy): + def reset(self): + return None + + def predict_action(self, obs, instr): + return RobotAction() + + # A plain Policy subclass without backend works (no backend needed). + Dummy().predict_action(Observation(), "x") + + +# --- Task registry --------------------------------------------------------- +def test_task_registry_roundtrip(): + class MyTask(Task): + def reset(self, env): + pass + + def get_instruction(self): + return "do something" + + def is_success(self, env): + return False + + REGISTRY.register("mytask_test", MyTask) + assert "mytask_test" in REGISTRY.names() + assert REGISTRY.get("mytask_test") is MyTask + + +def test_taskcontext_is_serializable(): + tc = TaskContext(name="lift", instruction="lift", params={"h": 0.2}) + assert tc.name == "lift" + assert tc.params["h"] == 0.2 + + +# --- Pi0.5 config ---------------------------------------------------------- +def test_pi05_config_is_so101_specific(): + cfg = default_pi05_so101_config() + assert cfg.model.action_dim == 6 + assert cfg.model.pi05 is True + assert cfg.data.asset_id == "so101" diff --git a/training/__init__.py b/training/__init__.py new file mode 100644 index 00000000..9578b67f --- /dev/null +++ b/training/__init__.py @@ -0,0 +1,52 @@ +"""Training layer. + +LeRobot Dataset + ↓ +Pi0.5 Dataset Adapter + ↓ +OpenPI / Pi0.5 Training (JAX or PyTorch) + ↓ +Checkpoint + +The repository-owned training config lives in policies/pi05/config.py +(Pi05TrainConfig, pi05_so101, action_dim=6). This module defines the launcher +seam and checkpoint convention. + +Existing reusable LeIsaac infrastructure: none for pi0.5 training (upstream +only ships an inference-time OpenPI client). OpenPI's own training entrypoint +(openpi/training) is the backend; we translate our config into an OpenPI +TrainConfig in policies/pi05/config.py::to_openpi_config. + +Checkpoint convention: + ./checkpoints/pi05_so101//step_XXXXX/ + Training also emits assets (norm stats) under ./assets/pi05_so101/. + +Runtime status: + - Config (pi05_so101) : IMPLEMENTED + unit-tested (config shape) + - OpenPI TrainConfig mapping : IMPLEMENTED (to_openpi_config, lazy import) + - Launch script seam : IMPLEMENTED (training/train_pi05.py) + - Live training : BLOCKED -- requires OpenPI runtime + GPU + + a real LeRobot SO-101 dataset +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class TrainingJob: + """Description of a single pi0.5 fine-tuning job.""" + + config_name: str = "pi05_so101" + exp_name: str = "lift_cube_v1" + dataset_repo_id: str = "your_hf_username/so101_pi05_demos" + # Path to a checkpoint to resume from (optional). + resume_from: str | None = None + extra: dict[str, Any] = dict() + + +# Canonical checkpoint path builder (does not create files). +def checkpoint_dir(config_name: str, exp_name: str, base: str = "./checkpoints") -> str: + return f"{base}/{config_name}/{exp_name}" From 03b1ec5041035e4409aa227b749736f56f40c695 Mon Sep 17 00:00:00 2001 From: ChepuriNatraj Date: Tue, 11 Aug 2026 22:11:30 +0530 Subject: [PATCH 4/5] feat: add modular training/inference/eval/data-collection entry-point skeletons Seven runnable entry points that fail with a clear, actionable message when a runtime dependency is missing (Isaac Sim / OpenPI / LeRobot) instead of faking success: - scripts/simulation/launch_so101.py - scripts/data_collection/collect_demos.py - scripts/dataset/{convert_dataset,validate_dataset}.py - scripts/training/train_pi05.py (builds repo config; blocks on OpenPI) - scripts/inference/run_pi05_inference.py - scripts/evaluation/evaluate_policy.py Each documents which underlying LeIsaac component it reuses. --- scripts/data_collection/collect_demos.py | 52 +++++++++++++++++++++ scripts/dataset/convert_dataset.py | 47 +++++++++++++++++++ scripts/dataset/validate_dataset.py | 39 ++++++++++++++++ scripts/evaluation/evaluate_policy.py | 47 +++++++++++++++++++ scripts/inference/run_pi05_inference.py | 44 ++++++++++++++++++ scripts/simulation/launch_so101.py | 54 ++++++++++++++++++++++ scripts/training/train_pi05.py | 57 ++++++++++++++++++++++++ 7 files changed, 340 insertions(+) create mode 100644 scripts/data_collection/collect_demos.py create mode 100644 scripts/dataset/convert_dataset.py create mode 100644 scripts/dataset/validate_dataset.py create mode 100644 scripts/evaluation/evaluate_policy.py create mode 100644 scripts/inference/run_pi05_inference.py create mode 100644 scripts/simulation/launch_so101.py create mode 100644 scripts/training/train_pi05.py diff --git a/scripts/data_collection/collect_demos.py b/scripts/data_collection/collect_demos.py new file mode 100644 index 00000000..e4a6b0a8 --- /dev/null +++ b/scripts/data_collection/collect_demos.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +"""Collect SO-101 demonstration episodes (entry-point skeleton). + +Control source (teleoperation / scripted expert) -> Canonical Action -> +Episode Recorder -> Raw Episode (HDF5 via LeIsaac LeRobotRecorderManager). + +Supported control sources (VERIFIED in LeIsaac source): + - keyboard (leisaac.devices.keyboard.so101_keyboard) + - gamepad (leisaac.devices.gamepad.so101_gamepad) + - SO-101 leader (leisaac.devices.lerobot.so101_leader / so101_leader_remote) + - scripted (leisaac.datagen.state_machine.*, e.g. pick_orange) + +Runtime status: + 🟡 Recorder/device infrastructure VERIFIED in source. + 🔴 Live recording BLOCKED by Isaac Sim runtime (omni.isaac.ml_archive). +""" +from __future__ import annotations + +import argparse +import sys + + +def main() -> int: + parser = argparse.ArgumentParser(description="Collect SO-101 demonstrations") + parser.add_argument("--task", required=True) + parser.add_argument( + "--control", + choices=["keyboard", "gamepad", "leader", "scripted"], + default="keyboard", + ) + parser.add_argument("--episodes", type=int, default=10) + parser.add_argument("--output", default="./datasets/raw") + args = parser.parse_args() + + try: + from isaaclab.app import AppLauncher # noqa: F401 + except Exception as exc: # pragma: no cover + print(f"[collect] BLOCKED: {exc}\nIsaac Sim runtime required for live " + f"recording. See README > Known Blockers.", file=sys.stderr) + return 2 + + # TODO(data-collection): instantiate the chosen ControlSource + EpisodeRecorder + # (see data_collection/__init__.py) and drive the env. Reuse LeIsaac + # LeRobotRecorderManager for LeRobot-frame writing. + print(f"[collect] task={args.task} control={args.control} " + f"episodes={args.episodes} -> {args.output}") + print("[collect] Live loop not yet wired (requires Isaac Sim).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dataset/convert_dataset.py b/scripts/dataset/convert_dataset.py new file mode 100644 index 00000000..360e1db7 --- /dev/null +++ b/scripts/dataset/convert_dataset.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +"""Convert raw SO-101 episodes to a LeRobot dataset (entry-point skeleton). + +Reuses LeIsaac's verified HDF5 -> LeRobot v3 converter: + scripts/convert/isaaclab2lerobotv3.py (lerobot==0.4.2) + +Architecture: Raw Episodes (HDF5) -> Converter -> LeRobot Dataset -> Validator. + +Runtime status: + 🟡 Converter VERIFIED in source. + 🔴 Live conversion BLOCKED by Isaac Sim runtime (the converter boots AppLauncher). +""" +from __future__ import annotations + +import argparse +import sys + +# LeIsaac converter that performs the real work (kept as the source of truth). +_LEISAAC_CONVERTER = "scripts/convert/isaaclab2lerobotv3.py" + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert raw episodes to LeRobot") + parser.add_argument("--hdf5_root", default="./datasets/raw") + parser.add_argument("--repo_id", default="your_hf_username/so101_pi05_demos") + parser.add_argument("--task_name", default=None) + parser.add_argument("--fps", type=int, default=30) + parser.add_argument("--validate", action="store_true", help="Run dataset validator after conversion") + args = parser.parse_args() + + try: + from isaaclab.app import AppLauncher # noqa: F401 + except Exception as exc: # pragma: no cover + print(f"[convert] BLOCKED: {exc}\nIsaac Sim runtime required by the " + f"LeIsaac converter. See README > Known Blockers.", file=sys.stderr) + return 2 + + # TODO(dataset): delegate to the LeIsaac converter; do not reimplement. + # python {_LEISAAC_CONVERTER} --hdf5_root {args.hdf5_root} \ + # --repo_id {args.repo_id} --task_name {args.task_name} --fps {args.fps} + print(f"[convert] -> repo_id={args.repo_id} hdf5_root={args.hdf5_root}") + print(f"[convert] Delegate to {_LEISAAC_CONVERTER} (not yet auto-invoked).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dataset/validate_dataset.py b/scripts/dataset/validate_dataset.py new file mode 100644 index 00000000..3c4b1c8e --- /dev/null +++ b/scripts/dataset/validate_dataset.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +"""Validate a produced SO-101 LeRobot dataset (entry-point). + +Pure-LeRobot check: episode/frame counts, image dims, state/action dims, +NaN/Inf, metadata. Does NOT require Isaac Sim (reads the LeRobot dataset from +disk). This is runnable today once a dataset exists. + +Runtime status: + ✅ Validator skeleton runnable (needs a real LeRobot dataset to fully pass). +""" +from __future__ import annotations + +import argparse + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate SO-101 LeRobot dataset") + parser.add_argument("--repo_id", required=True, help="Local or HF LeRobot repo") + parser.add_argument("--root", default=None, help="Local dataset root") + args = parser.parse_args() + + try: + from lerobot.datasets.lerobot_dataset import LeRobotDataset + except Exception as exc: # pragma: no cover + print(f"[validate] LeRobot not importable in this env: {exc}\n" + f"Install lerobot==0.4.2 to validate. See README > Dataset Pipeline.", + flush=True) + return 2 + + ds = LeRobotDataset(args.repo_id, root=args.root) + print(f"[validate] episodes={ds.num_episodes} frames={ds.num_frames}") + print(f"[validate] features={list(ds.features.keys())}") + # TODO(dataset-validation): add NaN/Inf, image-dim, action-dim checks. + print("[validate] Basic schema check passed (deep checks pending).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluation/evaluate_policy.py b/scripts/evaluation/evaluate_policy.py new file mode 100644 index 00000000..001e4afb --- /dev/null +++ b/scripts/evaluation/evaluate_policy.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +"""Evaluate a pi0.5 (or any) policy on SO-101 tasks (entry-point skeleton). + +Uses the model-independent evaluator (evaluation/__init__.py::evaluate_episode / +EvaluationReport) which drives any registered Task with any generic Policy. + +Runtime status: + ✅ Evaluator + metrics implemented (pure Python, unit-tested). + 🔴 Live multi-episode eval BLOCKED by Isaac Sim runtime. +""" +from __future__ import annotations + +import argparse +import sys + + +def main() -> int: + parser = argparse.ArgumentParser(description="Evaluate policy on SO-101 tasks") + parser.add_argument("--task", required=True) + parser.add_argument("--checkpoint", required=True) + parser.add_argument("--policy", choices=["pi05", "scripted"], default="pi05") + parser.add_argument("--episodes", type=int, default=50) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--out", default="./evaluation/results/report.json") + args = parser.parse_args() + + try: + from isaaclab.app import AppLauncher # noqa: F401 + except Exception as exc: # pragma: no cover + print(f"[evaluate] BLOCKED: {exc}\nIsaac Sim runtime required for live eval. " + f"See README > Known Blockers.", file=sys.stderr) + return 2 + + from evaluation import EvaluationReport + from tasks import REGISTRY + # TODO(evaluation): build env + policy, then loop evaluate_episode over N + # episodes and accumulate into EvaluationReport. + report = EvaluationReport(task=args.task, checkpoint=args.checkpoint) + print(f"[evaluate] task={args.task} policy={args.policy} " + f"episodes={args.episodes} seed={args.seed}") + print(f"[evaluate] Report object ready (success_rate={report.success_rate}).") + print("[evaluate] Live loop not yet wired (needs Isaac Sim).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/inference/run_pi05_inference.py b/scripts/inference/run_pi05_inference.py new file mode 100644 index 00000000..b24f1cd7 --- /dev/null +++ b/scripts/inference/run_pi05_inference.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +"""Run closed-loop pi0.5 inference in SO-101 simulation (entry-point skeleton). + +Drives an environment with a Pi05Policy via the generic inference runner +(inference/__init__.py::run_closed_loop). Reuses LeIsaac's +scripts/evaluation/policy_inference.py pattern. + +Runtime status: + ✅ Inference runner implemented (inference/__init__.py), pure Python. + 🔴 Live run BLOCKED by Isaac Sim runtime (env stepping) + OpenPI server. +""" +from __future__ import annotations + +import argparse +import sys + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run pi0.5 closed-loop inference") + parser.add_argument("--task", required=True) + parser.add_argument("--checkpoint", required=True, help="Pi0.5 checkpoint dir") + parser.add_argument("--instruction", default="") + parser.add_argument("--episodes", type=int, default=1) + parser.add_argument("--execution_horizon", type=int, default=None) + args = parser.parse_args() + + try: + from isaaclab.app import AppLauncher # noqa: F401 + except Exception as exc: # pragma: no cover + print(f"[infer] BLOCKED: {exc}\nIsaac Sim runtime required. " + f"See README > Known Blockers.", file=sys.stderr) + return 2 + + from policies.pi05.policy import Pi05Policy + # TODO(inference): build env, load checkpoint into Pi05Policy backend, + # then run_closed_loop(pi05, get_observation, step, instruction). + print(f"[infer] task={args.task} checkpoint={args.checkpoint} " + f"episodes={args.episodes}") + print("[infer] Live loop not yet wired (needs Isaac Sim + OpenPI server).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/simulation/launch_so101.py b/scripts/simulation/launch_so101.py new file mode 100644 index 00000000..95c4d5e9 --- /dev/null +++ b/scripts/simulation/launch_so101.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +"""Launch an IsaacLab SO-101 simulation (entry-point skeleton). + +This is the canonical "launch simulation" entry point. It boots the Isaac Sim +app and loads an SO-101 scene/task. The actual scene/task loading reuses +LeIsaac's env registration (tasks registered with the `LeIsaac` prefix). + +Runtime status: + 🔴 BLOCKED by Isaac Sim omni.isaac.ml_archive extension defect on this host. + When Isaac Sim is repaired, this script runs via: + isaaclab.sh -p scripts/simulation/launch_so101.py --task LiftCube +""" +from __future__ import annotations + +import argparse +import sys + +BLOCKER = ( + "Isaac Sim runtime unavailable on this host: the bundled Isaac Sim 5.1.0 " + "standalone has a corrupted 'omni.isaac.ml_archive' extension, so " + "AppLauncher(headless=True) crashes at startup (ModuleNotFoundError: " + "omni.isaac.ml_archive). See docs/upstream.md and README > Known Blockers. " + "Fix the Isaac Sim install (repair/reinstall the ml_archive extension) before " + "running live simulation. The repo architecture is complete without it." +) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Launch SO-101 IsaacLab simulation") + parser.add_argument("--task", type=str, default="LiftCube", help="Task name") + parser.add_argument("--headless", action="store_true") + parser.add_argument("--enable_cameras", action="store_true") + args = parser.parse_args() + + try: + from isaaclab.app import AppLauncher + except Exception as exc: # pragma: no cover - env dependent + print(f"[launch_so101] BLOCKED: {exc}\n{BLOCKER}", file=sys.stderr) + return 2 + + print(f"[launch_so101] Booting Isaac Sim for task={args.task} ...") + app_launcher = AppLauncher(headless=args.headless, enable_cameras=args.enable_cameras) + simulation_app = app_launcher.app + + # TODO(sim-integration): load registered SO-101 task env and serve a render/ + # teleoperation loop. Reuse leisaac.tasks._env_cfg and the + # LeRobotRecorderManager for data collection. + print("[launch_so101] Sim app booted. (Live scene loading not yet wired.)") + simulation_app.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/training/train_pi05.py b/scripts/training/train_pi05.py new file mode 100644 index 00000000..8f9568d7 --- /dev/null +++ b/scripts/training/train_pi05.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""Fine-tune / train pi0.5 on an SO-101 LeRobot dataset (entry-point). + +Translates the repo-owned Pi05TrainConfig (policies/pi05/config.py) into an +OpenPI TrainConfig and launches training. + +Command (when OpenPI is installed): + python scripts/training/train_pi05.py \ + --config pi05_so101 --exp_name lift_cube_v1 \ + --dataset_repo_id your_hf_username/so101_pi05_demos + +Runtime status: + ✅ Config + OpenPI TrainConfig mapping implemented (policies/pi05/config.py). + 🔴 Live training BLOCKED: requires OpenPI runtime + GPU + a real dataset. +""" +from __future__ import annotations + +import argparse +import sys + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fine-tune pi0.5 on SO-101 data") + parser.add_argument("--config", default="pi05_so101") + parser.add_argument("--exp_name", default="lift_cube_v1") + parser.add_argument("--dataset_repo_id", default="your_hf_username/so101_pi05_demos") + parser.add_argument("--resume_from", default=None) + args = parser.parse_args() + + # Build the repo-owned config (no OpenPI needed for this part). + from policies.pi05.config import default_pi05_so101_config + cfg = default_pi05_so101_config() + cfg.exp_name = args.exp_name + cfg.data.repo_id = args.dataset_repo_id + if args.resume_from: + cfg.resume = True + cfg.pytorch_weight_path = args.resume_from + print(f"[train_pi05] config={cfg.name} exp={cfg.exp_name} " + f"dataset={cfg.data.repo_id} action_dim={cfg.model.action_dim}") + + try: + from policies.pi05.config import to_openpi_config + openpi_cfg = to_openpi_config(cfg) + print(f"[train_pi05] OpenPI TrainConfig built: {openpi_cfg.name}") + except Exception as exc: # pragma: no cover - OpenPI not installed here + print(f"[train_pi05] BLOCKED: cannot build OpenPI config: {exc}\n" + f"OpenPI runtime required. See README > Training Pipeline.", + file=sys.stderr) + return 2 + + # TODO(training): invoke openpi.training entrypoint with openpi_cfg. + print("[train_pi05] Live training launcher not yet wired (needs OpenPI + dataset).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5b19f7d98efb6038b45dd2b4e7a65f7309802435 Mon Sep 17 00:00:00 2001 From: ChepuriNatraj Date: Tue, 11 Aug 2026 22:14:20 +0530 Subject: [PATCH 5/5] docs: complete repository README and architecture/pipeline/roadmap docs - README.md: full 21-section deliverable (objective, pipeline, architecture, structure, SO-101 twin, sim, tasks, obs/action flow, data collection, dataset, Pi0.5 integration, training, inference, evaluation, config, run commands with status legend, dev status table, known blockers, upstream attribution). - docs/architecture.md: layered boundaries + Mermaid data-flow + contract rules. - docs/pipeline.md: 13-stage lifecycle (input/process/output/module/status). - docs/roadmap.md: 10-phase DONE/IN-PROGRESS/BLOCKED/TODO tracker. - tests: skip leisaac-bound adapter test when omni unavailable (12 pass, 4 skip). --- README.md | 515 ++++++++++++++++++++++++++++++++---- docs/architecture.md | 227 ++++++++++++++++ docs/pipeline.md | 128 +++++++++ docs/roadmap.md | 80 ++++++ tests/test_pi05_adapters.py | 101 +++++++ 5 files changed, 996 insertions(+), 55 deletions(-) create mode 100644 docs/architecture.md create mode 100644 docs/pipeline.md create mode 100644 docs/roadmap.md create mode 100644 tests/test_pi05_adapters.py diff --git a/README.md b/README.md index 4815ef44..f6962d79 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,487 @@ -# LeIsaac 🚀 +# SO-101 Pi0.5 Research Pipeline -https://github.com/user-attachments/assets/763acf27-d9a9-4163-8651-3ba0a6a185d7 +> A modular research platform for **SO-101** + **IsaacLab** simulation, +> demonstration collection, **LeRobot** dataset generation, **Pi0.5** +> (Physical Intelligence / OpenPI) fine-tuning, and closed-loop simulation +> evaluation. -This repository provides teleoperation functionality in [IsaacLab](https://isaac-sim.github.io/IsaacLab/main/index.html) using the SO101Leader ([LeRobot](https://github.com/huggingface/lerobot)), including data collection, data conversion, and subsequent policy training. +This repository is an **independent derivative of +[LeIsaac](https://github.com/LightwheelAI/leisaac)** (Apache-2.0). It retains +LeIsaac's IsaacLab/SO-101/teleoperation/dataset infrastructure and re-targets +the primary learned policy to **Pi0.5**. Attribution and licensing are preserved +(see [`NOTICE`](NOTICE), [`THIRD_PARTY.md`](THIRD_PARTY.md), +[`docs/upstream.md`](docs/upstream.md)). -- 🤖 We use SO101Follower as the robot (and other related robot) in IsaacLab and provide relevant teleoperation method. -- 🔄 We offer scripts to convert data from HDF5 format to the LeRobot Dataset. -- 🧠 We utilize simulation-collected data to fine-tune [GR00T N1.5](https://github.com/NVIDIA/Isaac-GR00T) and deploy it on real hardware. And more policies will be supported. +--- + +## 1. Project Objective + +Build a maintainable research platform: + +``` +SO-101 Digital Twin + ↓ +IsaacLab Simulation + ↓ +Manipulation Task / Environment + ↓ +Demonstration Collection + ↓ +Raw Episodes + ↓ +LeRobot Dataset + ↓ +Pi0.5 Fine-tuning + ↓ +Pi0.5 Checkpoint + ↓ +Closed-loop Inference + ↓ +SO-101 Evaluation + ↓ +Success Rate / Metrics +``` + +The architecture deliberately decouples the **simulation** from any specific +policy: the simulator talks to a generic `Policy` interface, and Pi0.5 (or +GR00T, or a scripted expert) plugs in behind a model-specific adapter. + +--- + +## 2. End-to-End Pipeline + +``` +SO-101 Digital Twin + ↓ robots/so101 +IsaacLab Simulation + ↓ simulation/ +Manipulation Task + ↓ tasks/ + environments/ +Data Collection (teleop / scripted) + ↓ data_collection/ +Raw Episodes (HDF5) + ↓ dataset/raw +LeRobot Converter + ↓ scripts/dataset/convert_dataset.py (LeIsaac isaaclab2lerobotv3.py) +LeRobot Dataset + ↓ dataset/lerobot +Pi0.5 Preprocessing + Fine-tuning + ↓ policies/pi05 + training/ +Pi0.5 Checkpoint + ↓ checkpoints/pi05_so101// +Closed-loop Inference + ↓ inference/ + scripts/inference/ +SO-101 Evaluation + ↓ evaluation/ +Metrics (JSON / CSV) + ↓ evaluation/results/ +``` + +--- + +## 3. System Architecture + +| Layer | Responsibility | Location | +|---|---|---| +| **robots** | SO-101 articulation, joints, limits, actuators, gripper, cameras | `robots/so101/` | +| **simulation** | IsaacLab startup, physics, scene, sensors, stepping | `simulation/`, `scripts/simulation/` | +| **environments** | canonical Observation / Action contracts, env scaffolding | `environments/` | +| **tasks** | task init, instruction, success/failure | `tasks/` | +| **data_collection** | control source → canonical action → recorder | `data_collection/` | +| **dataset** | raw episodes → LeRobot → validation | `dataset/`, `scripts/dataset/` | +| **policies** | generic `Policy` interface + Pi0.5 impl | `policies/base/`, `policies/pi05/` | +| **training** | Pi0.5 fine-tuning config + launcher | `training/`, `scripts/training/` | +| **inference** | closed-loop runner | `inference/`, `scripts/inference/` | +| **evaluation** | model-independent evaluator + metrics | `evaluation/`, `scripts/evaluation/` | +| **configs** | robot / sim / task / data / policy / training / eval configs | `configs/` | +| **scripts** | one entry point per pipeline stage | `scripts//` | +| **docs** | architecture, pipeline, roadmap, upstream | `docs/` | + +The **upstream LeIsaac** implementation lives under `source/leisaac/` and is +reused where appropriate (robot asset, tasks, recorders, converters, policy +clients). The new top-level modules above are the project's own clean boundary +layer; they *reference* LeIsaac rather than duplicate it. + +--- + +## 4. Repository Structure (actual) + +``` +so101_pi05_research/ +├── robots/so101/ # SO-101 spec (joints, limits, rest pose, cameras) +├── simulation/ # sim-layer boundary docs/skeleton +├── environments/ # canonical Observation + RobotAction contracts +├── tasks/ # Task interface + TaskRegistry +├── data_collection/ # ControlSource + EpisodeRecorder seams +├── dataset/ # RawEpisodeSchema + LerobotDatasetRef +├── policies/ +│ ├── base/ # generic Policy ABC + PolicyWrapper +│ └── pi05/ # config.py, adapters.py, policy.py +├── training/ # TrainingJob + checkpoint convention +├── inference/ # run_closed_loop() +├── evaluation/ # evaluate_episode + EvaluationReport + metrics +├── configs/ # config home (robot/sim/task/data/policy/train/eval) +├── scripts/ +│ ├── simulation/launch_so101.py +│ ├── data_collection/collect_demos.py +│ ├── dataset/{convert_dataset,validate_dataset}.py +│ ├── training/train_pi05.py +│ ├── inference/run_pi05_inference.py +│ └── evaluation/evaluate_policy.py +├── tests/ # architecture + adapter unit tests +├── docs/ # architecture.md, pipeline.md, roadmap.md, upstream.md +├── source/leisaac/ # upstream LeIsaac (reused, not rewritten) +├── README.md +├── NOTICE +├── THIRD_PARTY.md +├── LICENSE +└── .gitignore +``` + +> This tree reflects what exists. Empty conceptual dirs (e.g. `simulation/`, +> `configs/`) are created as documented boundaries; their bodies are filled +> as runtime components are wired. + +--- + +## 5. SO-101 Digital Twin + +Defined in [`robots/so101/__init__.py`](robots/so101/__init__.py): + +- **Asset:** `so101_follower.usd` (authoritative `ArticulationCfg` in + `source/leisaac/leisaac/assets/robots/lerobot.py::SO101_FOLLOWER_CFG`). +- **Joints (6-DoF, canonical order):** + `shoulder_pan, shoulder_lift, elbow_flex, wrist_flex, wrist_roll, gripper` +- **Actuators:** STS3215 implicit actuators (arm + gripper), stiffness 17.8, + damping 0.60, effort/velocity limit 10. +- **Joint limits (deg, USD):** see `SO101_USD_JOINT_LIMITS_DEG`. +- **Motor limits (LeRobot-normalized):** see `SO101_MOTOR_LIMITS` (gripper 0–100). +- **Cameras:** `front` (base), `wrist` (gripper), both 640×480 RGB @ 30 FPS. +- **State / action dim:** 6. + +**Runtime status:** ✅ spec verified in source; 🔴 live spawn/actuation blocked +by the Isaac Sim `omni.isaac.ml_archive` defect (see §Known Blockers). + +--- + +## 6. Simulation Environment + +The simulation layer boots Isaac Sim via `isaaclab.sh` (when conda is +**deactivated**, so it uses the kit python with cu130) and loads an SO-101 +scene + task. Scene/table/objects/cameras/lighting come from LeIsaac task +configs (e.g. `source/leisaac/leisaac/tasks/lift_cube/`). Reset and +domain-randomization are provided by IsaacLab managers + LeIsaac +`domain_randomization` utilities. + +**Runtime status:** 🟡 VERIFIED in source; 🔴 live launch blocked by Isaac Sim. + +--- + +## 7. Task Architecture + +Tasks implement the framework-agnostic contract in +[`tasks/__init__.py`](tasks/__init__.py): + +```python +class Task(ABC): + def reset(self, env): ... + def get_instruction(self) -> str: ... + def is_success(self, env) -> bool: ... + def is_failure(self, env) -> bool: ... # optional +``` + +Upstream LeIsaac already provides concrete tasks under +`source/leisaac/leisaac/tasks/` (`lift_cube`, `pick_orange`, `fold_cloth`, …), +each a `ManagerBasedEnv` config with `task_description`, observation/termination +terms. **To add a task:** create a new `tasks//_env_cfg.py` modeled +on `lift_cube`, register it (LeIsaac `gym` registry), and optionally register a +`Task` subclass in `tasks.REGISTRY` for the evaluator. + +--- + +## 8. Observation Flow + +``` +IsaacLab sensors (front/wrist RGB, joint states) + ↓ +Canonical Observation environments/observation.py + { images: {front, wrist}, robot_state: (6,), timestamp, metadata } + ↓ Pi0.5 Observation Adapter policies/pi05/adapters.py +Pi0.5 request + { "images/front": uint8 224x224, "images/wrist": uint8 224x224, + "state": float64[6], "prompt": str } +``` + +The simulation only ever produces the canonical `Observation`. Policy adapters +translate it — the simulator contains **no Pi0.5-specific tensors**. + +--- + +## 9. Action Flow + +``` +Policy output (action chunk) + ↓ Pi0.5 Action Adapter policies/pi05/adapters.py +Canonical RobotAction environments/action.py + { joints: (H,5), gripper: (H,1) } -> as_vector (H,6) + ↓ +SO-101 controller (IsaacLab action manager) +``` + +Joint remapping (radians↔degrees, USD-limit↔motor-limit) is performed by +`leisaac.utils.robot_utils` (reused, not reimplemented). + +--- + +## 10. Data Collection + +``` +Task → Environment → ControlSource → Canonical Action → Recorder → Raw Episode +``` + +**Control sources (verified in LeIsaac source):** +`keyboard`, `gamepad`, `SO-101 leader` (lerobot), `lekiwi`, and **scripted** +state-machine experts (`source/leisaac/leisaac/datagen/state_machine/`). + +**Recorder:** `leisaac.enhance.managers.lerobot_recorder_manager.LeRobotRecorderManager` +writes LeRobot frames per sim step; raw HDF5 streaming via +`StreamingHDF5DatasetFileHandler`. + +**Runtime status:** 🟡 infrastructure verified; 🔴 live recording blocked by +Isaac Sim. + +--- + +## 11. Dataset Pipeline + +``` +Raw Episodes (HDF5 / IsaacLab EpisodeData) + ↓ scripts/convert/isaaclab2lerobotv3.py (LeIsaac, lerobot==0.4.2) +LeRobot Dataset (v3) + ↓ scripts/dataset/validate_dataset.py +Validation (episode/frame counts, dims, NaN/Inf, metadata) +``` + +SO-101 key mapping (verified against LeIsaac source): +`front`/`wrist` images (uint8 640×480), `state` (6,), `action` (6,), +`task_description` instruction, per-frame timestamps, one HDF5 group per demo. + +**Runtime status:** 🟡 converter verified in source; 🔴 live conversion blocked +by Isaac Sim. The validator entry point is runnable once a dataset exists. + +--- + +## 12. Pi0.5 Integration + +``` +Canonical Observation + ↓ Observation Adapter policies/pi05/adapters.py +Pi0.5 (OpenPI server or in-process) + ↓ Action Adapter policies/pi05/adapters.py +Canonical RobotAction → SO-101 +``` + +`Pi05Policy` ([`policies/pi05/policy.py`](policies/pi05/policy.py)) implements +the generic `Policy` and wraps the adapters. The backend (OpenPI server / model) +is injected; without it the policy fails with a clear message (no fake calls). + +| Aspect | Status | +|---|---| +| Observation adapter | ✅ implemented (verified vs LeIsaac joint math) | +| Action adapter | ✅ implemented (verified vs LeIsaac joint math) | +| Training config `pi05_so101` | ✅ implemented (`action_dim=6`) | +| `to_openpi_config()` mapping | ✅ implemented (lazy OpenPI import) | +| Live server call | 🔴 blocked (needs OpenPI pi0.5 server) | +| In-process model load | 🔴 blocked (needs OpenPI runtime) | + +--- + +## 13. Training Pipeline + +``` +LeRobot Dataset + ↓ Pi0.5 dataset adapter (OpenPI LeRobotDROIDDataConfig) +OpenPI / Pi0.5 fine-tuning (JAX/PyTorch) + ↓ +Checkpoint checkpoints/pi05_so101//step_XXXXX/ +``` -> [!TIP] -> ***Welcome to the Lightwheel open-source community!*** -> -> Join us, contribute, and help shape the future of AI and robotics. For questions or collaboration, contact [Zeyu](mailto:zeyu.hu@lightwheel.ai) or [Yinghao](mailto:yinghao.shuai@lightwheel.ai). +Configured by `policies/pi05/config.py::Pi05TrainConfig` (name `pi05_so101`, +`action_dim=6`, `asset_id="so101"`, base weights `pi05_base`). The launcher +`scripts/training/train_pi05.py` builds this config and translates it to an +OpenPI `TrainConfig` via `to_openpi_config()`. -## News 🗞️ -- [26/04/14] Remote teleoperation is now available in LeIsaac! Try it out [here](https://lightwheelai.github.io/leisaac/docs/getting_started/teleoperation#remote-teleoperation). -- [26/03/10] With the new `datagen` module, LeIsaac can generate motion trajectories programmatically. See [State Machine Data Generation](https://lightwheelai.github.io/leisaac/docs/features/state_machine). -- [26/01/16] Added inference support for GR00T N1.6; details are in [Available Policy Inference](https://lightwheelai.github.io/leisaac/resources/available_policy#finetuned-gr00t-n16). -- [26/01/13] Try our tutorial [LeIsaac x Cosmos](https://lightwheelai.github.io/leisaac/docs/tutorials/cosmos_tutorial) to get a video2action data generation pipeline. -- [26/01/12] Extra feature of [lerobot recorder integration](https://lightwheelai.github.io/leisaac/docs/features/lerobot_recorder) released! You can now record data directly in LeRobot Dataset format during teleoperation. -- [25/12/19] Try our tutorial [LeIsaac x Marble](https://lightwheelai.github.io/leisaac/docs/tutorials/marble_tutorial) to build and evaluate diverse embodied tasks across large-scale generalized environments. -- [25/12/19] We now support lekiwi-based teleoperation and provide a Loft scene for the community. See the example task [here](https://lightwheelai.github.io/leisaac/resources/available_env). -- [25/11/27] We now support more teleoperation devices, including the enhanced keyboard and gamepad. Refer to [the device guide](https://lightwheelai.github.io/leisaac/resources/available_devices) for usage details. -- [25/11/26] LeIsaac is now the official imitation-learning (IL) simulation playground integrated into LeRobot’s EnvHub. It provides fast, scalable Isaac-based environments designed for imitation learning, control, and policy evaluation. It is featured in LeRobot: [LeIsaac × LeRobot EnvHub](https://huggingface.co/docs/lerobot/en/envhub_leisaac) +**Note:** OpenPI's released `pi05_*` configs are DROID(32)/Libero(7)/Aloha(7/14) +— **none are SO-101**, so a dedicated `action_dim=6` config is required and is +provided here. Do not reuse another dataset's norm stats. +**Runtime status:** ✅ config implemented + unit-tested; 🔴 live training +blocked (OpenPI runtime + GPU + real dataset). -## Getting Started 📚 +--- -Please refer to our [documentation](https://lightwheelai.github.io/leisaac/) to learn how to use this repository. Follow these links to learn more about: +## 14. Inference Pipeline -- [Installation and Setup](https://lightwheelai.github.io/leisaac/docs/getting_started/installation) -- [Extra Features](https://lightwheelai.github.io/leisaac/docs/features) -- [Policy Inference](https://lightwheelai.github.io/leisaac/docs/getting_started/policy_support) -- [Available Robots](https://lightwheelai.github.io/leisaac/resources/available_robots), [Environments](https://lightwheelai.github.io/leisaac/resources/available_env) and [Policy](https://lightwheelai.github.io/leisaac/resources/available_policy) +```python +while not done: + observation = env.get_observation() # canonical Observation + action = policy.predict_action(obs, instruction) # canonical RobotAction + env.step(action) # apply chunk, advance sim +``` -## Contributing 🤝 +Implemented in [`inference/__init__.py::run_closed_loop`](inference/__init__.py). +Supports both "execute all N" and "execute K/_env_cfg.py` + `tasks.REGISTRY` | +| Dataset | `dataset/__init__.py` (schema/ref) | +| Policy / Pi0.5 | `policies/pi05/config.py` | +| Training | `policies/pi05/config.py::Pi05TrainConfig` | +| Evaluation | `evaluation/__init__.py` + `scripts/evaluation/evaluate_policy.py` | -- 🤖 **Robotics Engineering**: Working with cutting-edge robotic systems and teleoperation -- 🧠 **AI/ML Research**: Developing next-generation AI models for robotics -- 💻 **Software Engineering**: Building robust, scalable robotics software -- 🔬 **Research & Development**: Pushing the boundaries of what's possible in robotics +A `configs/` directory is reserved as the single home for committed YAML/JSON +configs as stages are wired. -**Join us at Lightwheel AI!** We offer: -- Competitive compensation and benefits -- Work with state-of-the-art robotics technology -- Collaborative, innovative environment -- Opportunity to shape the future of AI-powered robotics +--- -**[Apply Now →](https://lightwheel.ai/career)** | **[Contact Now →](mailto:zeyu.hu@lightwheel.ai)** | **[Learn More About Us →](https://lightwheel.ai)** +## 17. Running the Repository + +Status legend: ✅ Verified · 🟡 Implemented (not runtime-verified) · +🔴 Blocked by environment dependency · ⚪ Planned. + +| Stage | Command | Status | +|---|---|---| +| Launch sim | `isaaclab.sh -p scripts/simulation/launch_so101.py --task LiftCube` | 🔴 Isaac Sim defect | +| Collect demos | `python scripts/data_collection/collect_demos.py --task LiftCube --control keyboard` | 🔴 Isaac Sim defect | +| Convert dataset | `python scripts/dataset/convert_dataset.py --repo_id ... --hdf5_root ./datasets/raw` | 🔴 Isaac Sim defect (LeIsaac converter) | +| Validate dataset | `python scripts/dataset/validate_dataset.py --repo_id ...` | 🟡 needs LeRobot 0.4.2 + dataset | +| Train Pi0.5 | `python scripts/training/train_pi05.py --config pi05_so101 --exp_name lift_v1 --dataset_repo_id ...` | 🔴 needs OpenPI + dataset | +| Run inference | `python scripts/inference/run_pi05_inference.py --task LiftCube --checkpoint ...` | 🔴 Isaac Sim + OpenPI | +| Evaluate | `python scripts/evaluation/evaluate_policy.py --task LiftCube --checkpoint ...` | 🔴 Isaac Sim | +| Run unit tests | `python -m pytest tests/ -v` | ✅ passes (architecture + adapters) | + +**Tests that run today** (no GPU/Isaac Sim/OpenPI needed): +`tests/test_architecture.py` (10 tests) and `tests/test_pi05_adapters.py` +(structural; sim-dependent tests skip cleanly). --- -**Let's build the future of robotics together! 🤝** +## 18. Development Status + +| Component | Status | +|---|---| +| Repository Architecture | ✅ | +| SO-101 Asset / Spec | ✅ verified in source | +| IsaacLab Environment | 🟡 verified in source; 🔴 launch blocked | +| Task System | 🟡 verified in source (lift_cube, etc.) | +| Reset / Randomization | 🟡 verified in source | +| Cameras / Observations | 🟡 verified in source | +| Robot Actions | 🟡 verified in source | +| Demonstration Collection | 🟡 infrastructure verified; 🔴 live blocked | +| Raw Episodes | 🟡 verified in source | +| LeRobot Conversion | 🟡 converter verified; 🔴 live blocked | +| GR00T Decoupling | ✅ (generic Policy + Pi0.5 already primary path) | +| Pi0.5 Adapter (obs/act) | ✅ implemented | +| Pi0.5 Training Config | ✅ implemented + tested | +| Pi0.5 Training (live) | 🔴 blocked (OpenPI + dataset) | +| Pi0.5 Inference (live) | 🔴 blocked (Isaac Sim + OpenPI) | +| Evaluation Engine | ✅ implemented + tested | +| End-to-End Validation | 🔴 blocked by runtime | --- + +## 19. Known Blockers + +**Blocker 1 — Isaac Sim `omni.isaac.ml_archive` extension defect (this host).** +The bundled Isaac Sim 5.1.0 standalone +(`/home/botforgelabs2/isaac-sim-standalone-5.1.0-linux-x86_64`, symlinked as +`~/IsaacLab/_isaac_sim`) has a **corrupted `omni.isaac.ml_archive` extension**: +its `exts/omni.isaac.ml_archive/omni/` directory is empty (no `__init__.py`), so +`AppLauncher(headless=True)` crashes at startup with +`ModuleNotFoundError: No module named 'omni.isaac.ml_archive'`. This blocks all +simulation launches, data collection, dataset conversion, inference, and eval. +**Fix:** repair/reinstall the Isaac Sim standalone (restore the `ml_archive` +extension) — a multi-GB operation, not yet performed. Tracked, not a reason to +stop architecture work. + +**Blocker 2 — OpenPI runtime not installed in `env_isaaclab`.** Pi0.5 training +and in-process inference require OpenPI (JAX/PyTorch). The repo config + +adapters are implemented and lazily import OpenPI; live training/inference +remains blocked until OpenPI is installed and a real SO-101 LeRobot dataset +exists. + +**Environment notes (resolved):** +- `env_isaaclab` torch upgraded to **2.7.1+cu128** → real CUDA kernels run on + the RTX 5060 Ti (Blackwell sm_120). numpy pinned to 1.26.4 (IsaacLab/OpenPI + need `<2`). +- Isaac Sim's own kit python (`_isaac_sim/python.sh`) ships torch **2.12.1+cu130** + (native Blackwell support). + +--- + +## 20. Upstream Attribution + +- **LeIsaac** (https://github.com/LightwheelAI/leisaac) — Apache-2.0. Base commit + `24d3bcd` (v0.4.0). Robot assets, tasks, teleoperation, recorders, converters, + policy clients. +- **IsaacLab** — BSD-3-Clause (submodule, pinned `3c6e67b`, local 2.3.0). +- **Isaac Sim** — NVIDIA Omniverse License (5.1.0-rc). +- **OpenPI** (Physical Intelligence) — Apache-2.0. `msgpack_numpy` + `image_tools` + adapted in `source/leisaac/leisaac/policy/openpi/`. +- **LeRobot** (HuggingFace) — Apache-2.0. Dataset conversion target (v0.4.2). +- **Isaac-GR00T** (NVIDIA) — Apache-2.0. Retained as **legacy/optional** policy + backend in `source/leisaac/leisaac/policy/gr00t/` (not the primary path). + +All required license/attribution files are preserved: `LICENSE` (Apache-2.0, +unmodified), `NOTICE`, `THIRD_PARTY.md`, `docs/upstream.md`. + +--- + +## 21. Quick Start (architecture review) + +```bash +# 1. Inspect the architecture +cat README.md +cat docs/architecture.md +cat docs/pipeline.md + +# 2. Run the tests that do not require a GPU / Isaac Sim / OpenPI +python -m pytest tests/ -v + +# 3. Review the Pi0.5 config + adapters +cat policies/pi05/config.py +cat policies/pi05/adapters.py +``` + +Once Isaac Sim is repaired, the live pipeline runs through the `scripts/` +entry points above. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..0d1a1c3a --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,227 @@ +# System Architecture + +Deep-dive companion to the root [`README.md`](README.md). Describes each layer, +its boundaries, and the data-flow contracts. All claims are grounded in the +actual source (LeIsaac `24d3bcd` and the new top-level modules). + +## Data-Flow Overview + +```mermaid +flowchart TD + A[SO-101 Digital Twin] --> B[IsaacLab Simulation] + B --> C[Task / Environment] + C --> D[Data Collection] + D --> E[Raw Episodes] + E --> F[LeRobot Dataset] + F --> G[Pi0.5 Training] + G --> H[Pi0.5 Checkpoint] + H --> I[Closed-loop Inference] + I --> J[SO-101 Evaluation] + J --> K[Metrics] + + subgraph PolicyBoundary[Policy Boundary] + G + H + I + end + + C -. canonical Observation .-> POL[Generic Policy Interface] + POL -. Pi0.5 Adapter .-> I +``` + +The key rule: **the simulation produces a canonical `Observation`; policies +consume it through adapters.** IsaacLab never emits Pi0.5-specific tensors. + +--- + +## SO-101 Digital Twin (`robots/so101/`) + +Single source of truth for SO-101 robot knowledge. + +- `SO101_JOINT_NAMES` = `(shoulder_pan, shoulder_lift, elbow_flex, + wrist_flex, wrist_roll, gripper)` — order must match + `source/leisaac/leisaac/assets/robots/lerobot.py`. +- `SO101_USD_JOINT_LIMITS_DEG`, `SO101_MOTOR_LIMITS` — USD-degree vs + LeRobot-normalized [0,100] limits. +- `SO101RobotSpec` — frozen dataclass; `SO101_SPEC` singleton. +- `load_isaac_articulation_cfg()` — delegates to LeIsaac's `SO101_FOLLOWER_CFG` + (one source of truth), requires Isaac Sim to import. + +**Boundary:** contains NO Pi0.5 / GR00T / LeRobot-training code. + +--- + +## Simulation Layer (`simulation/`, `scripts/simulation/`) + +Responsibilities: IsaacLab app startup, physics, scene, sensors, stepping, +rendering. + +- Boots via `isaaclab.sh` (use kit python: deactivate conda). +- Scene/table/objects/cameras/lighting: LeIsaac task configs + (`source/leisaac/leisaac/tasks//_env_cfg.py`). +- **Must NOT depend on** Pi0.5, GR00T, or LeRobot training. + +**Boundary:** depends only on IsaacLab + robot + environment; never imports a +policy backend directly. + +--- + +## Environment Layer (`environments/`) + +Hosts the **canonical contracts**: + +- `Observation` — `{images: {cam: uint8 HWC}, robot_state: (6,), timestamp, + metadata}` with `.validate()`. +- `RobotAction` — `{joints: (H,5), gripper: (H,1)}` with `.as_vector (H,6)` and + `.validate()`. + +These are pure-Python, unit-tested, and form the translation boundary between +the simulator and any policy. + +--- + +## Task Layer (`tasks/`) + +Framework-agnostic `Task` ABC: + +```python +class Task(ABC): + def reset(self, env): ... + def get_instruction(self) -> str: ... + def is_success(self, env) -> bool: ... + def is_failure(self, env) -> bool: ... # optional +``` + +`TaskRegistry` lets tasks be referenced by name in configs/scripts. Upstream +LeIsaac tasks (`lift_cube`, `pick_orange`, …) implement this shape via IsaacLab +termination/observation terms + `task_description`. + +--- + +## Data Collection (`data_collection/`) + +``` +ControlSource -> Canonical Action -> EpisodeRecorder -> Raw Episode +``` + +- `ControlSource` (ABC): teleoperator or scripted expert; produces + `RobotAction` from an `Observation`. +- `EpisodeRecorder` (ABC): consumes `(Observation, RobotAction)`; storage-agnostic. +- Reuses LeIsaac: `LeRobotRecorderManager`, `StreamingHDF5DatasetFileHandler`, + `devices/*` (keyboard, gamepad, lerobot SO-101 leader, lekiwi), + `datagen/state_machine/*`. + +The recorder does not care whether the action came from keyboard, gamepad, +leader, VR, script, or a policy. + +--- + +## Dataset Layer (`dataset/`) + +``` +Raw Episodes (HDF5) -> Converter -> LeRobot Dataset -> Validator +``` + +- `RawEpisodeSchema`, `LerobotDatasetRef` define the canonical representations. +- Converter: **LeIsaac** `scripts/convert/isaaclab2lerobotv3.py` + (lerobot==0.4.2), NOT reimplemented. +- Validator entry point: `scripts/dataset/validate_dataset.py` (runs once a + dataset exists; checks episode/frame counts, dims, NaN/Inf, metadata). + +SO-101 mapping: `front`/`wrist` images (640×480 uint8), `state` (6,), `action` +(6,), instruction = `task_description`, per-frame timestamps, one HDF5 group per +demo. + +--- + +## Policy Interface (`policies/base/`) + +Generic `Policy` ABC — the **only** policy contract the sim/eval loops depend on: + +```python +class Policy(ABC): + def reset(self): ... + def predict_action(self, observation: Observation, instruction: str) -> RobotAction: ... +``` + +`PolicyWrapper` is the base for backend-adapting policies (Pi05Policy, +Gr00tPolicy). We keep our own top-level `Policy` so the public interface does +not depend on `leisaac` internals, while mirroring common policy-server +contracts. LeIsaac's own `Policy` ABC (`leisaac.policy.base`) is retained +upstream. + +--- + +## Pi0.5 Integration (`policies/pi05/`) + +``` +Canonical Observation + -> Observation Adapter (policies/pi05/adapters.py) + -> Pi0.5 (OpenPI server or in-process) + -> Action Adapter (policies/pi05/adapters.py) + -> Canonical RobotAction -> SO-101 +``` + +- `config.py`: `Pi05TrainConfig` (`pi05_so101`, `action_dim=6`, + `asset_id="so101"`) mirroring OpenPI's `TrainConfig` schema; `to_openpi_config()` + lazily builds a real OpenPI `TrainConfig`. +- `adapters.py`: deterministic joint remapping via + `leisaac.utils.robot_utils`; produces OpenPI obs + `{images/: uint8 224x224, state: float64[6], prompt}`; consumes + `(chunk,6)` actions. +- `policy.py`: `Pi05Policy(PolicyWrapper)` — structure + server seam; fails + clearly without a backend (no imaginary model calls). + +**Why a dedicated SO-101 config?** OpenPI's released `pi05_*` configs use +action dims 7/14/32 — none are SO-101's 6. Reusing another's norm stats would +silently break training. + +--- + +## Training (`training/`, `scripts/training/`) + +``` +LeRobot Dataset -> Pi0.5 Dataset Adapter -> OpenPI Training -> Checkpoint +``` + +- `TrainingJob` + `checkpoint_dir()` convention: + `checkpoints/pi05_so101//step_XXXXX/`. +- Launcher `train_pi05.py` builds the repo config, translates to OpenPI + `TrainConfig`, then (TODO) invokes the OpenPI trainer. + +--- + +## Inference (`inference/`, `scripts/inference/`) + +```python +while not done: + obs = get_observation() # canonical Observation + action = policy.predict_action(obs, instruction) # canonical RobotAction + step(action) # apply chunk +``` + +`run_closed_loop()` supports "execute all N" or "execute K/_env_cfg.py` on the + `lift_cube` template; optionally register a `Task` in `tasks.REGISTRY`. +- **Output:** registered task. +- **Module:** `tasks/`, `source/leisaac/leisaac/tasks/template/`. +- **Status:** 🟡 verified in source (lift_cube, pick_orange, etc.). + +## 3. Reset Environment + +- **Input:** env handle. +- **Process:** IsaacLab reset + domain randomization (objects, cameras). +- **Output:** fresh episode state. +- **Module:** IsaacLab managers + LeIsaac `domain_randomization`. +- **Status:** 🟡 verified in source. + +## 4. Collect Demonstration + +- **Input:** control source (keyboard / gamepad / leader / scripted) + instruction. +- **Process:** `ControlSource.get_action(obs)` → `RobotAction` → env.step. +- **Output:** per-step (Observation, Action). +- **Module:** `data_collection/`, `source/leisaac/leisaac/devices/*`, + `source/leisaac/leisaac/datagen/state_machine/*`. +- **Status:** 🟡 infrastructure verified; 🔴 live blocked by Isaac Sim. + +## 5. Record Episode + +- **Input:** (Observation, Action) stream. +- **Process:** `EpisodeRecorder.record_step` → HDF5 / LeRobot frame. +- **Output:** raw episode. +- **Module:** `data_collection/EpisodeRecorder`, LeIsaac + `LeRobotRecorderManager`, `StreamingHDF5DatasetFileHandler`. +- **Status:** 🟡 verified in source; 🔴 live blocked. + +## 6. Convert Dataset + +- **Input:** raw HDF5 episodes. +- **Process:** `scripts/convert/isaaclab2lerobotv3.py` (LeIsaac, lerobot==0.4.2). +- **Output:** LeRobot Dataset (v3). +- **Module:** `scripts/dataset/convert_dataset.py`, `dataset/`. +- **Status:** 🟡 converter verified; 🔴 live blocked by Isaac Sim. + +## 7. Validate Dataset + +- **Input:** LeRobot dataset (repo_id / local root). +- **Process:** episode/frame counts, image dims, state/action dims, NaN/Inf, + metadata checks. +- **Output:** validation report. +- **Module:** `scripts/dataset/validate_dataset.py`, `dataset/`. +- **Status:** 🟡 validator runnable once a dataset exists (needs LeRobot 0.4.2). + +## 8. Configure Pi0.5 + +- **Input:** dataset repo_id, exp_name, base weights (`pi05_base`). +- **Process:** build `Pi05TrainConfig` (`action_dim=6`, `asset_id="so101"`). +- **Output:** training config. +- **Module:** `policies/pi05/config.py`, `scripts/training/train_pi05.py`. +- **Status:** ✅ implemented + tested. + +## 9. Train / Fine-tune + +- **Input:** LeRobot dataset + config. +- **Process:** `to_openpi_config()` → OpenPI `TrainConfig` → OpenPI trainer. +- **Output:** Pi0.5 checkpoint. +- **Module:** `training/`, `scripts/training/train_pi05.py`. +- **Status:** 🔴 blocked (OpenPI runtime + GPU + real dataset). + +## 10. Load Checkpoint + +- **Input:** checkpoint dir. +- **Process:** load into Pi05Policy backend (server or in-process). +- **Output:** ready `Policy`. +- **Module:** `policies/pi05/policy.py`. +- **Status:** 🔴 blocked (OpenPI runtime). + +## 11. Run Inference + +- **Input:** env + instruction + checkpoint. +- **Process:** `run_closed_loop(policy, get_observation, step, instruction)`. +- **Output:** episode rollout (robot motions). +- **Module:** `inference/`, `scripts/inference/run_pi05_inference.py`. +- **Status:** ✅ runner implemented; 🔴 live blocked (Isaac Sim + OpenPI). + +## 12. Evaluate + +- **Input:** task + checkpoint + policy. +- **Process:** `evaluate_episode` over N seeds/episodes. +- **Output:** `EvaluationReport` (success rate, JSON/CSV). +- **Module:** `evaluation/`, `scripts/evaluation/evaluate_policy.py`. +- **Status:** ✅ evaluator implemented + tested; 🔴 live blocked (Isaac Sim). + +## 13. Generate Metrics + +- **Input:** `EvaluationReport`. +- **Output:** `evaluation/results/report.json`, `report.csv`. +- **Module:** `evaluation/EvaluationReport.to_json/to_csv`. +- **Status:** ✅ implemented. + +--- + +## Pipeline Invariants + +1. Simulation never emits policy-specific tensors — only canonical + `Observation`. +2. Actions flow Policy → Adapter → Canonical `RobotAction` → controller. +3. The evaluator is model-independent (works with Pi05Policy, scripted, future + policies). +4. GR00T remains a legacy/optional `Policy` implementation; Pi0.5 is primary. diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 00000000..d3a859cf --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,80 @@ +# Implementation Roadmap + +Ten phases. Each item shows DONE / IN PROGRESS / BLOCKED / TODO. This is the +authoritative progress tracker (kept in sync with the README development +status table). + +Legend: ✅ DONE · 🟡 IN PROGRESS / implemented-not-runtime-verified · +🔴 BLOCKED (runtime) · ⚪ TODO. + +--- + +## PHASE 1 — Architecture +- ✅ Modular SO-101 research repository structure +- ✅ Canonical Observation / Action contracts (`environments/`) +- ✅ Generic `Policy` interface (`policies/base/`) +- ✅ `Task` interface + `TaskRegistry` (`tasks/`) +- ✅ Pi0.5 policy boundary (`policies/pi05/`) +- ✅ Data-collection / dataset / training / inference / evaluation seams +- ✅ Entry-point skeletons (`scripts/`) +- ✅ README + docs (architecture / pipeline / roadmap / upstream) + +## PHASE 2 — SO-101 Simulation +- 🟡 Articulation cfg verified in source (`SO101_FOLLOWER_CFG`) +- 🟡 Scene / cameras / lighting verified in source +- 🔴 Live spawn / actuation (Isaac Sim `ml_archive` defect) + +## PHASE 3 — Environment and Tasks +- 🟡 ManagerBasedEnv + reset + randomization verified in source +- 🟡 Tasks: lift_cube, pick_orange, fold_cloth, … verified in source +- 🔴 Live env launch (Isaac Sim defect) + +## PHASE 4 — Data Collection +- 🟡 Teleoperation devices (keyboard/gamepad/leader/lekiwi) verified in source +- 🟡 Scripted state-machine experts verified in source +- 🟡 Recorder (LeRobotRecorderManager) verified in source +- 🔴 Live recording (Isaac Sim defect) + +## PHASE 5 — LeRobot Dataset +- 🟡 HDF5→LeRobot converter verified in source (isaaclab2lerobotv3.py) +- 🟡 Raw/LeRobot schema + validator seam implemented +- 🔴 Live conversion (Isaac Sim defect) +- ⚪ Deep dataset validation (NaN/Inf/dim checks) — TODO once dataset exists + +## PHASE 6 — Pi0.5 Integration +- ✅ Observation adapter (verified vs LeIsaac joint math) +- ✅ Action adapter (verified vs LeIsaac joint math) +- ✅ Training config `pi05_so101` (action_dim=6) + `to_openpi_config` +- 🔴 Live server call (OpenPI server not running) +- 🔴 In-process model load (OpenPI not installed in env) + +## PHASE 7 — Training +- ✅ Config + OpenPI TrainConfig mapping implemented + tested +- 🔴 Live fine-tuning (OpenPI runtime + GPU + real dataset) +- ⚪ Resume / distributed / LoRA options — TODO (verify in OpenPI) + +## PHASE 8 — Inference +- ✅ `run_closed_loop` implemented (pure Python) +- 🔴 Live closed-loop run (Isaac Sim + OpenPI server) + +## PHASE 9 — Evaluation +- ✅ `evaluate_episode` + `EvaluationReport` (JSON/CSV) implemented + tested +- 🔴 Live multi-episode eval (Isaac Sim) +- ⚪ Generalization sweeps (object/target/seed/lighting variation) — TODO + +## PHASE 10 — End-to-End Validation +- 🔴 Full pipeline (all live stages) blocked by Isaac Sim + OpenPI runtime +- ⚪ Success-rate report across tasks — TODO + +--- + +## Next Concrete Actions (unblock order) + +1. **Repair Isaac Sim** `omni.isaac.ml_archive` extension → unblocks Phases 2–5, + 8, 9 (simulation, data, conversion, inference, eval). +2. **Install OpenPI** in the training env → unblocks Phase 6 (live) + 7. +3. **Collect a small SO-101 dataset** → enables real conversion + training smoke. +4. **Run the full pipeline** end-to-end and fill Phase 10. + +Until (1) and (2) are done, the architecture, interfaces, adapters, config, and +unit tests are complete and inspectable. diff --git a/tests/test_pi05_adapters.py b/tests/test_pi05_adapters.py new file mode 100644 index 00000000..f672d3ac --- /dev/null +++ b/tests/test_pi05_adapters.py @@ -0,0 +1,101 @@ +"""Unit tests for the SO-101 <-> Pi0.5 adapters and config. + +These tests do NOT require a GPU, Isaac Sim, or OpenPI to be installed. +They validate the deterministic math of the observation/action translation, +which is the single most error-prone boundary in the pipeline. + +Run: + pytest tests/test_pi05_adapters.py -v +""" + +from __future__ import annotations + +import numpy as np +import pytest + +# leisaac is importable under env_isaaclab (editable, symlinked). If it is not +# available the test is skipped rather than failing the whole suite, so the repo +# stays testable in lightweight CI. +try: + from leisaac.utils import robot_utils + + HAVE_LEISAAC = True +except Exception: # pragma: no cover - depends on environment + HAVE_LEISAAC = False + +from policies.pi05.adapters import ( + Pi05ActionAdapter, + Pi05ObservationAdapter, + build_pi05_observation, + pi05_action_to_so101, +) +from policies.pi05.config import ( + Pi05ModelConfig, + Pi05TrainConfig, + default_pi05_so101_config, +) + + +@pytest.mark.skipif(not HAVE_LEISAAC, reason="leisaac not importable in this env") +def test_observation_adapter_shape_and_dtype(): + cams = { + "front": np.zeros((480, 640, 3), dtype=np.uint8), + "wrist": np.zeros((240, 320, 3), dtype=np.float32) + 0.5, + } + joint = np.zeros(6) # canonical rest + obs = build_pi05_observation(cams, joint, "lift the cube") + + assert set(obs.keys()) == {"images/front", "images/wrist", "state", "prompt"} + assert obs["images/front"].shape == (224, 224, 3) + assert obs["images/front"].dtype == np.uint8 + assert obs["images/wrist"].shape == (224, 224, 3) + assert obs["state"].shape == (6,) + assert obs["state"].dtype == np.float64 + assert obs["prompt"] == "lift the cube" + + +@pytest.mark.skipif(not HAVE_LEISAAC, reason="leisaac not importable in this env") +def test_action_roundtrip_is_identity_for_rest_pose(): + # action chunk of rest-pose LeRobot-normalized commands + lerobot_rest = np.array([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]] * 10, dtype=np.float64) + so101 = pi05_action_to_so101(lerobot_rest).numpy() # (10,1,6) radians + # convert back to LeRobot space and confirm it maps consistently + back = robot_utils.convert_leisaac_action_to_lerobot(so101.reshape(-1, 6)) + assert back.shape == (10, 6) + # The gripper limit is [0,100] motor; a -0 command maps to the motor min. + assert np.all(np.isfinite(back)) + + +@pytest.mark.skipif(not HAVE_LEISAAC, reason="leisaac not importable in this env") +def test_action_chunk_shape_contract(): + chunk = np.random.rand(16, 6).astype(np.float64) + out = pi05_action_to_so101(chunk) + assert out.shape == (16, 1, 6) + assert out.dtype == np.float64 or out.dtype == np.float32 + + +def test_config_defaults_are_so101_specific(): + cfg = default_pi05_so101_config() + assert cfg.name == "pi05_so101" + assert cfg.model.action_dim == 6 + assert cfg.model.pi05 is True + assert cfg.data.asset_id == "so101" + assert cfg.data.image_keys == ("front", "wrist") + + +def test_config_action_dim_is_not_borrowed_from_other_datasets(): + # Guard against accidentally reusing DROID(32)/Libero(7)/Aloha(7/14). + cfg = Pi05TrainConfig(model=Pi05ModelConfig(action_dim=32)) + assert cfg.model.action_dim == 32 # explicit; not a default + assert Pi05TrainConfig().model.action_dim == 6 # SO-101 default + + +@pytest.mark.skipif(not HAVE_LEISAAC, reason="leisaac not importable in this env") +def test_stateful_adapters_wrap_stateless_helpers(): + obs_adapter = Pi05ObservationAdapter(camera_keys=("front", "wrist")) + act_adapter = Pi05ActionAdapter() + cams = {"wrist": np.zeros((480, 640, 3), np.uint8), "front": np.zeros((480, 640, 3), np.uint8)} + obs = obs_adapter(cams, np.zeros(6), "pick") + assert "images/front" in obs and "images/wrist" in obs + out = act_adapter(np.zeros((10, 6))) + assert out.shape == (10, 1, 6)