From e08b8709f8e8e31f121e4688318444fba1a6062c Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Wed, 9 Sep 2026 18:32:42 +0300 Subject: [PATCH 1/7] Preserve DROID checkpoint defaults for N1.7 fine-tuning and serving --- .github/workflows/docker-build.yml | 19 +++ README.md | 2 +- docker/Dockerfile | 6 + docker/Makefile | 16 +++ docker/README.md | 16 ++- gr00t/configs/finetune_config.py | 7 +- gr00t/experiment/launch_finetune.py | 53 +++++-- gr00t/policy/gr00t_policy.py | 12 +- tests/fixtures/droid/config.json | 87 ++++++++++++ tests/fixtures/droid/processor_config.json | 134 ++++++++++++++++++ .../experiment/test_finetune_checkpoint.py | 70 +++++++++ tests/gr00t/policy/test_gr00t_policy.py | 18 ++- 12 files changed, 417 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/docker-build.yml create mode 100644 docker/Makefile create mode 100644 tests/fixtures/droid/config.json create mode 100644 tests/fixtures/droid/processor_config.json create mode 100644 tests/gr00t/experiment/test_finetune_checkpoint.py diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..3af3c3dc4 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,19 @@ +name: Build and Push Positronic GR00T Base + +on: + push: + branches: [main-positronic] + workflow_dispatch: + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push branch and commit tags + working-directory: docker + run: make push diff --git a/README.md b/README.md index 6d86ea8af..eb935ad8e 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ See the [Orin setup guide](scripts/deployment/README.md#jetson-orin-setup) for D > per-platform Docker and bare-metal setup. -For a containerized setup that avoids system-level dependency conflicts, see our [Docker Setup Guide](docker/README.md). The recommended container workflow is to start the image first, then clone or pull the repo inside the running container so your checkout uses the image's prebuilt dependency environment. +For a containerized setup that avoids system-level dependency conflicts, see our [Docker Setup Guide](docker/README.md). The image includes this fork at `/gr00t` with its prebuilt dependency environment. --- diff --git a/docker/Dockerfile b/docker/Dockerfile index 34fe6d6f1..1d7f7a47a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -225,4 +225,10 @@ ENV MUJOCO_GL="egl" \ # bootstrap_wheels.sh before sync (then committed back) if missing. flash-attn # comes from official release URLs — no source build needed. +# Install the fork alongside its dependency environment for Positronic subprocesses. +WORKDIR /gr00t +COPY . /gr00t +RUN uv pip install --python /opt/gr00t-venv/bin/python --no-deps -e /gr00t +ENV PYTHONPATH=/gr00t + CMD ["/bin/bash"] diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 000000000..c47f779c1 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,16 @@ +.PHONY: build tag push + +IMAGE_NAME := positro/gr00t-base +GIT_SHA := $(shell git rev-parse --short HEAD) +IMAGE_TAG ?= $(shell git branch --show-current | tr '/' '-') + +build: + docker build --platform linux/amd64 -f Dockerfile -t $(IMAGE_NAME):local .. + +tag: build + docker tag $(IMAGE_NAME):local $(IMAGE_NAME):$(IMAGE_TAG) + docker tag $(IMAGE_NAME):local $(IMAGE_NAME):$(GIT_SHA) + +push: tag + docker push $(IMAGE_NAME):$(IMAGE_TAG) + docker push $(IMAGE_NAME):$(GIT_SHA) diff --git a/docker/README.md b/docker/README.md index e4c62d7c7..bc4797651 100644 --- a/docker/README.md +++ b/docker/README.md @@ -18,7 +18,21 @@ From the repository root: bash docker/build.sh ``` -This builds from `nvidia/cuda:12.8.0-devel-ubuntu24.04` and installs all dependencies into `/opt/gr00t-venv`. The image does not include a working source checkout; for normal use, start the image and then clone or pull the repo you want to run inside the container. +This builds from `nvidia/cuda:12.8.0-devel-ubuntu24.04` and installs all dependencies into `/opt/gr00t-venv`. The image includes this fork at `/gr00t`, installed into `/opt/gr00t-venv`. Positronic launches that environment directly. + +## Positronic base image + +```bash +make -C docker build +make -C docker push IMAGE_TAG=my-branch +``` + +The Makefile publishes `positro/gr00t-base` with branch and commit tags. It does not replace +`latest`. Positronic's `GROOT_BASE_IMAGE` selects this image for its adapter build. + +Fine-tuning defaults to the base checkpoint's saved model and modality configuration. +`--video-keys` selects the fine-tuning camera layout; omitted, it retains the checkpoint's views. +The policy server accepts `--model-path hf://nvidia/GR00T-N1.7-DROID` and downloads that snapshot. ## Running the Container diff --git a/gr00t/configs/finetune_config.py b/gr00t/configs/finetune_config.py index 15918e206..6b1d3ce66 100644 --- a/gr00t/configs/finetune_config.py +++ b/gr00t/configs/finetune_config.py @@ -41,10 +41,13 @@ class FinetuneConfig: modality_config_path: str | None = None """ - Path to a Python file defining the modality configuration for the given embodiment. - If None, use the pre-registered modality config in `gr00t/configs/data/embodiment_configs.py`. + Path to a Python file defining the modality configuration for the given embodiment. + If None, retain the modality configuration saved in the base checkpoint. """ + video_keys: list[str] | None = None + """Explicit camera keys for the fine-tuning dataset; omission retains the checkpoint's cameras.""" + # --- Model Tuning Flags --- tune_llm: bool = False """If True, fine-tune the language model (LLM) backbone during training.""" diff --git a/gr00t/experiment/launch_finetune.py b/gr00t/experiment/launch_finetune.py index 00cb3ef1f..3260c2618 100644 --- a/gr00t/experiment/launch_finetune.py +++ b/gr00t/experiment/launch_finetune.py @@ -16,15 +16,19 @@ # Launch finetuning for N1.7 on "single node". # This script tries to provide a similar user experience as current OSS. +import copy import json import os from pathlib import Path +from transformers.utils import cached_file import tyro from gr00t.configs.base_config import get_default_config from gr00t.configs.finetune_config import FinetuneConfig -from gr00t.experiment.experiment import run +from gr00t.configs.model.gr00t_n1d7 import Gr00tN1d7Config +from gr00t.data.embodiment_tags import EmbodimentTag +from gr00t.data.types import ModalityConfig # Make sure the user provided modality config is registered. @@ -41,16 +45,9 @@ def load_modality_config(modality_config_path: str): raise FileNotFoundError(f"Modality config path does not exist: {modality_config_path}") -if __name__ == "__main__": - # Set LOGURU_LEVEL environment variable if not already set (default: INFO) - if "LOGURU_LEVEL" not in os.environ: - os.environ["LOGURU_LEVEL"] = "INFO" - # Use tyro for clean CLI - ft_config = tyro.cli(FinetuneConfig, description=__doc__) - from gr00t.data.embodiment_tags import EmbodimentTag - - ft_config.embodiment_tag = EmbodimentTag.resolve(ft_config.embodiment_tag) - embodiment_tag = ft_config.embodiment_tag.value +def build_config(ft_config: FinetuneConfig): + """Inherit model and modality contracts from the checkpoint before applying training overrides.""" + embodiment_tag = EmbodimentTag.resolve(ft_config.embodiment_tag).value # all rank workers should register for the modality config if ft_config.modality_config_path is not None: @@ -73,6 +70,24 @@ def load_modality_config(modality_config_path: str): } ) config.load_config_path = None + config.model = Gr00tN1d7Config.from_pretrained(ft_config.base_model_path) + checkpoint = Path(ft_config.base_model_path) + processor_root = checkpoint / "processor" if (checkpoint / "processor").is_dir() else checkpoint + processor_file = cached_file(str(processor_root), "processor_config.json") + with open(processor_file) as f: + processor_kwargs = json.load(f)["processor_kwargs"] + config.model.use_relative_action = processor_kwargs["use_relative_action"] + if ft_config.modality_config_path is None: + modalities = processor_kwargs["modality_configs"][embodiment_tag] + config.data.modality_configs = { + embodiment_tag: {name: ModalityConfig(**value) for name, value in modalities.items()} + } + else: + config.data.modality_configs = copy.deepcopy(config.data.modality_configs) + if ft_config.video_keys is not None: + if not ft_config.video_keys or len(set(ft_config.video_keys)) != len(ft_config.video_keys): + raise ValueError("video_keys must be nonempty and unique") + config.data.modality_configs[embodiment_tag]["video"].modality_keys = ft_config.video_keys # overwrite with finetune config supplied by the user config.model.tune_llm = ft_config.tune_llm @@ -80,8 +95,10 @@ def load_modality_config(modality_config_path: str): config.model.tune_projector = ft_config.tune_projector config.model.tune_diffusion_model = ft_config.tune_diffusion_model config.model.state_dropout_prob = ft_config.state_dropout_prob - config.model.random_rotation_angle = ft_config.random_rotation_angle - config.model.color_jitter_params = ft_config.color_jitter_params + if ft_config.random_rotation_angle is not None: + config.model.random_rotation_angle = ft_config.random_rotation_angle + if ft_config.color_jitter_params is not None: + config.model.color_jitter_params = ft_config.color_jitter_params config.model.use_percentiles = ft_config.use_percentiles if (ft_config.shortest_image_edge is None) != (ft_config.crop_fraction is None): raise ValueError("shortest_image_edge and crop_fraction must be set together") @@ -99,7 +116,6 @@ def load_modality_config(modality_config_path: str): config.model.reproject_vision = False config.model.model_name = "nvidia/Cosmos-Reason2-2B" config.model.backbone_trainable_params_fp32 = True - config.model.use_relative_action = True config.training.experiment_name = ft_config.experiment_name config.training.start_from_checkpoint = ft_config.base_model_path @@ -127,4 +143,11 @@ def load_modality_config(modality_config_path: str): config.training.resume_from_checkpoint = ft_config.resume_from_checkpoint config.training.skip_weight_loading = ft_config.skip_weight_loading - run(config) + return config + + +if __name__ == "__main__": + from gr00t.experiment.experiment import run + + os.environ.setdefault("LOGURU_LEVEL", "INFO") + run(build_config(tyro.cli(FinetuneConfig, description=__doc__))) diff --git a/gr00t/policy/gr00t_policy.py b/gr00t/policy/gr00t_policy.py index 6f5a46b10..73289f6fc 100644 --- a/gr00t/policy/gr00t_policy.py +++ b/gr00t/policy/gr00t_policy.py @@ -23,6 +23,7 @@ from pathlib import Path from typing import Any +from huggingface_hub import snapshot_download import numpy as np import torch from transformers import AutoModel, AutoProcessor @@ -103,7 +104,10 @@ def __init__( super().__init__(strict=strict) if isinstance(embodiment_tag, str): embodiment_tag = EmbodimentTag.resolve(embodiment_tag) - model_dir = Path(model_path) + if str(model_path).startswith("hf://"): + model_dir = Path(snapshot_download(str(model_path).removeprefix("hf://"))) + else: + model_dir = Path(model_path) # Load the pretrained model and move to target device with bfloat16 precision model = AutoModel.from_pretrained(model_dir) @@ -251,6 +255,12 @@ def check_observation(self, observation: dict[str, Any]) -> None: bs = -1 # ===== VIDEO VALIDATION ===== + expected_cameras = set(self.modality_configs["video"].modality_keys) + if set(observation["video"]) - expected_cameras: + raise ValueError( + f"Checkpoint cameras {sorted(expected_cameras)} do not match " + f"observation cameras {sorted(observation['video'])}" + ) # Validate each video stream defined in the modality config for video_key in self.modality_configs["video"].modality_keys: assert video_key in observation["video"], ( diff --git a/tests/fixtures/droid/config.json b/tests/fixtures/droid/config.json new file mode 100644 index 000000000..11a846e5a --- /dev/null +++ b/tests/fixtures/droid/config.json @@ -0,0 +1,87 @@ +{ + "action_horizon": 40, + "add_pos_embed": true, + "apply_sincos_state_encoding": false, + "architectures": [ + "Gr00tN1d7" + ], + "attn_dropout": 0.2, + "attn_implementation": null, + "backbone_embedding_dim": 2048, + "color_jitter_params": { + "brightness": 0.3, + "contrast": 0.4, + "hue": 0.08, + "saturation": 0.5 + }, + "crop_fraction": 0.95, + "diffusion_model_cfg": { + "attention_head_dim": 48, + "dropout": 0.2, + "final_dropout": true, + "interleave_self_attention": true, + "norm_type": "ada_norm", + "num_attention_heads": 32, + "num_layers": 32, + "output_dim": 1024, + "positional_embeddings": null + }, + "dtype": "bfloat16", + "exclude_state": false, + "formalize_language": true, + "hidden_size": 1024, + "image_crop_size": [ + 230, + 230 + ], + "image_target_size": [ + 256, + 256 + ], + "letter_box_transform": false, + "load_bf16": true, + "max_action_dim": 132, + "max_num_embodiments": 32, + "max_seq_len": 1024, + "max_state_dim": 132, + "model_dtype": "bfloat16", + "model_type": "Gr00tN1d7", + "noise_beta_alpha": 1.5, + "noise_beta_beta": 1.0, + "noise_s": 0.999, + "num_inference_timesteps": 4, + "num_timestep_buckets": 1000, + "random_history_crop": true, + "random_rotation_angle": 0, + "reproject_vision": false, + "rtc_ramp_rate": 6.0, + "select_layer": 16, + "shortest_image_edge": 256, + "state_dropout_prob": 0.2, + "state_gaussian_noise_std": 0.0, + "transformers_version": "4.57.1", + "tune_diffusion_model": true, + "tune_linear": true, + "tune_llm": true, + "tune_projector": true, + "tune_top_llm_layers": 0, + "tune_visual": true, + "tune_vlln": true, + "use_albumentations": true, + "use_alternate_vl_dit": true, + "use_flash_attention": true, + "use_future_tokens": false, + "use_mean_std": false, + "use_percentiles": true, + "use_vl_self_attention": true, + "use_vlln": true, + "vl_self_attention_cfg": { + "attention_head_dim": 64, + "dropout": 0.2, + "final_dropout": true, + "num_attention_heads": 32, + "num_layers": 4, + "positional_embeddings": null + }, + "model_name": "nvidia/Cosmos-Reason2-2B" +} diff --git a/tests/fixtures/droid/processor_config.json b/tests/fixtures/droid/processor_config.json new file mode 100644 index 000000000..55b4d74b3 --- /dev/null +++ b/tests/fixtures/droid/processor_config.json @@ -0,0 +1,134 @@ +{ + "processor_class": "Gr00tN1d7Processor", + "processor_kwargs": { + "modality_configs": { + "oxe_droid_relative_eef_relative_joint": { + "video": { + "delta_indices": [ + 0 + ], + "modality_keys": [ + "exterior_image_1_left", + "wrist_image_left" + ] + }, + "state": { + "delta_indices": [ + 0 + ], + "modality_keys": [ + "eef_9d", + "gripper_position", + "joint_position" + ] + }, + "action": { + "delta_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ], + "modality_keys": [ + "eef_9d", + "gripper_position", + "joint_position" + ], + "action_configs": [ + { + "rep": "RELATIVE", + "type": "EEF", + "format": "XYZ_ROT6D", + "state_key": "eef_9d" + }, + { + "rep": "ABSOLUTE", + "type": "NON_EEF", + "format": "DEFAULT", + "state_key": "gripper_position" + }, + { + "rep": "RELATIVE", + "type": "NON_EEF", + "format": "DEFAULT", + "state_key": "joint_position" + } + ] + }, + "language": { + "delta_indices": [ + 0 + ], + "modality_keys": [ + "annotation.language.language_instruction" + ] + } + } + }, + "use_percentiles": true, + "use_mean_std": false, + "image_crop_size": [ + 230, + 230 + ], + "image_target_size": [ + 256, + 256 + ], + "formalize_language": true, + "max_state_dim": 132, + "max_action_dim": 132, + "apply_sincos_state_encoding": false, + "color_jitter_params": { + "brightness": 0.3, + "contrast": 0.4, + "saturation": 0.5, + "hue": 0.08 + }, + "random_rotation_angle": 0, + "letter_box_transform": false, + "exclude_state": false, + "state_dropout_prob": 0.2, + "use_albumentations": true, + "shortest_image_edge": 256, + "crop_fraction": 0.95, + "max_action_horizon": 40, + "use_relative_action": true + } +} diff --git a/tests/gr00t/experiment/test_finetune_checkpoint.py b/tests/gr00t/experiment/test_finetune_checkpoint.py new file mode 100644 index 000000000..3ac2bc0a4 --- /dev/null +++ b/tests/gr00t/experiment/test_finetune_checkpoint.py @@ -0,0 +1,70 @@ +"""DROID checkpoint contracts survive fine-tuning configuration. + +Fixtures are from nvidia/GR00T-N1.7-DROID revision 05e7cc9 on Hugging Face. +""" + +import json +from pathlib import Path + +from gr00t.configs.finetune_config import FinetuneConfig +from gr00t.data.types import ActionFormat, ActionRepresentation, ActionType +from gr00t.experiment.launch_finetune import build_config +import pytest + + +CHECKPOINT = Path(__file__).parents[2] / "fixtures" / "droid" +EMBODIMENT = "oxe_droid_relative_eef_relative_joint" + + +@pytest.mark.parametrize( + "cameras", [None, ["exterior_image_1_left", "exterior_image_2_left", "wrist_image_left"]] +) +def test_finetuning_retains_droid_checkpoint_contract(cameras): + config = build_config( + FinetuneConfig( + base_model_path=str(CHECKPOINT), + dataset_path="/dataset", + embodiment_tag=EMBODIMENT, + video_keys=cameras, + resume_from_checkpoint=True, + ) + ) + saved = json.loads((CHECKPOINT / "processor_config.json").read_text())["processor_kwargs"] + modalities = config.data.modality_configs[EMBODIMENT] + assert modalities["video"].delta_indices == [0] + assert modalities["video"].modality_keys == ( + cameras or saved["modality_configs"][EMBODIMENT]["video"]["modality_keys"] + ) + assert modalities["state"].delta_indices == [0] + assert modalities["action"].delta_indices == list(range(40)) + eef, grip, joints = modalities["action"].action_configs + assert (eef.rep, eef.type, eef.format) == ( + ActionRepresentation.RELATIVE, + ActionType.EEF, + ActionFormat.XYZ_ROT6D, + ) + assert grip.rep is ActionRepresentation.ABSOLUTE + assert joints.rep is ActionRepresentation.RELATIVE + assert config.model.use_relative_action + assert config.model.diffusion_model_cfg["num_layers"] == 32 + assert config.model.select_layer == 16 + assert config.model.action_horizon == 40 + assert config.model.shortest_image_edge == saved["shortest_image_edge"] == 256 + assert config.model.crop_fraction == saved["crop_fraction"] == 0.95 + assert config.model.color_jitter_params == saved["color_jitter_params"] + assert not config.model.letter_box_transform + assert not config.model.tune_llm + assert not config.model.tune_visual + assert config.training.resume_from_checkpoint + + +def test_camera_override_does_not_change_a_subsequent_run(): + arguments = dict( + base_model_path=str(CHECKPOINT), dataset_path="/dataset", embodiment_tag=EMBODIMENT + ) + build_config(FinetuneConfig(**arguments, video_keys=["custom_wrist", "custom_external"])) + config = build_config(FinetuneConfig(**arguments)) + assert config.data.modality_configs[EMBODIMENT]["video"].modality_keys == [ + "exterior_image_1_left", + "wrist_image_left", + ] diff --git a/tests/gr00t/policy/test_gr00t_policy.py b/tests/gr00t/policy/test_gr00t_policy.py index 54de946f8..89a99000f 100644 --- a/tests/gr00t/policy/test_gr00t_policy.py +++ b/tests/gr00t/policy/test_gr00t_policy.py @@ -49,8 +49,8 @@ def _build_modality_configs(): } -@pytest.fixture -def policy(): +@pytest.fixture(params=["/fake/path", "hf://owner/model"]) +def policy(request): mock_model = MagicMock() mock_model.eval = MagicMock() mock_model.to = MagicMock(return_value=mock_model) @@ -96,6 +96,7 @@ def fake_decode_action(action, embodiment_tag, state=None): with ( patch("gr00t.policy.gr00t_policy.AutoModel") as MockAutoModel, patch("gr00t.policy.gr00t_policy.AutoProcessor") as MockAutoProcessor, + patch("gr00t.policy.gr00t_policy.snapshot_download", return_value="/fake/path") as download, patch("pathlib.Path.is_dir", return_value=False), patch("pathlib.Path.exists", return_value=True), ): @@ -106,9 +107,14 @@ def fake_decode_action(action, embodiment_tag, state=None): p = Gr00tPolicy( embodiment_tag=EMBODIMENT, - model_path="/fake/path", + model_path=request.param, device="cpu", ) + if request.param.startswith("hf://"): + download.assert_called_once_with("owner/model") + else: + download.assert_not_called() + MockAutoModel.from_pretrained.assert_called_once_with(Path("/fake/path")) return p @@ -139,6 +145,12 @@ def test_policy_embodiment_tag(self, policy): class TestGr00tPolicyCheckObservation: + def test_extra_camera_cannot_be_silently_ignored(self, policy): + obs = _make_observation() + obs["video"]["second_external"] = obs["video"][VIDEO_KEYS[0]] + with pytest.raises(ValueError, match="Checkpoint cameras"): + policy.check_observation(obs) + def test_valid_observation_passes(self, policy): obs = _make_observation() policy.check_observation(obs) From b61c478840e5f79f80ba88ad08f58a09c9d66697 Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Thu, 10 Sep 2026 11:05:13 +0300 Subject: [PATCH 2/7] Use checkpoint image settings for fine-tuning --- docker/README.md | 40 ++++++++----------- gr00t/experiment/launch_finetune.py | 1 + .../experiment/test_finetune_checkpoint.py | 2 + tests/gr00t/policy/test_gr00t_policy.py | 17 ++++++-- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/docker/README.md b/docker/README.md index bc4797651..2499754e3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -15,7 +15,7 @@ Docker configuration for building and running a containerized GR00T environment From the repository root: ```bash -bash docker/build.sh +make -C docker build ``` This builds from `nvidia/cuda:12.8.0-devel-ubuntu24.04` and installs all dependencies into `/opt/gr00t-venv`. The image includes this fork at `/gr00t`, installed into `/opt/gr00t-venv`. Positronic launches that environment directly. @@ -36,45 +36,39 @@ The policy server accepts `--model-path hf://nvidia/GR00T-N1.7-DROID` and downlo ## Running the Container -**Recommended workflow: run the image, then clone or update the repo inside it.** - -Start an interactive shell: +Run the included fork from `/gr00t`: ```bash docker run -it --rm --gpus all \ --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ - gr00t + -v "$HOME/.cache/huggingface:/root/.cache/huggingface" \ + positro/gr00t-base:local ``` -Then, inside the container: +Inside the container: ```bash -git clone --recurse-submodules https://github.com/NVIDIA/Isaac-GR00T /workspace/Isaac-GR00T -cd /workspace/Isaac-GR00T -export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" -python -c "import gr00t; print('GR00T ready')" +cd /gr00t +uv run --no-sync python -c "import gr00t; print(gr00t.__file__)" ``` -The image venv is active by default (`/opt/gr00t-venv`; `/workspace/.venv` is a compatibility symlink), and uv is configured with `UV_PROJECT_ENVIRONMENT=/opt/gr00t-venv`. After setting `PYTHONPATH` to the checked-out repo, both `python ...` and `uv run ...` use the global image venv instead of creating a checkout-local `.venv`. If you are working on an existing checkout in the container, run `git pull --ff-only` from that checkout instead of cloning again. +The image includes the fork and its locked environment at `/opt/gr00t-venv`. +The Hugging Face account must have access to the gated `nvidia/Cosmos-Reason2-2B` backbone. +Provide its token through the mounted Hugging Face cache or `HF_TOKEN`. -The global venv records the `uv.lock` hash it was built from. If your checked-out repo uses a different lockfile, create a checkout-local venv before running commands. Reusing a uv cache keeps this path from starting cold: +For development, mount a compatible fork checkout over `/gr00t`: ```bash -export UV_CACHE_DIR="${UV_CACHE_DIR:-/workspace/uv-cache}" -export UV_LINK_MODE=copy -UV_PROJECT_ENVIRONMENT="$PWD/.venv" uv sync -source .venv/bin/activate +docker run -it --rm --gpus all --ipc=host \ + -v "$PWD:/gr00t" positro/gr00t-base:local ``` -Do not run a bare `uv sync` unless you intend to update the global image venv. Use `UV_PROJECT_ENVIRONMENT="$PWD/.venv" uv sync` when you want an isolated per-checkout environment. - -Avoid bind-mounting over `/workspace`, because that can hide the image's `/workspace/.venv` compatibility symlink. If you need to mount local source for live editing, mount it under a subdirectory: +If its lockfile differs from the image, create a separate environment: ```bash -docker run -it --rm --gpus all \ - --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ - -v "$(pwd):/workspace/Isaac-GR00T" \ - gr00t bash -c 'cd /workspace/Isaac-GR00T && export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" && bash' +export UV_PROJECT_ENVIRONMENT=/gr00t/.venv +uv sync --locked --extra dev +source /gr00t/.venv/bin/activate ``` ## Edge Device Containers diff --git a/gr00t/experiment/launch_finetune.py b/gr00t/experiment/launch_finetune.py index 3260c2618..5901b1d67 100644 --- a/gr00t/experiment/launch_finetune.py +++ b/gr00t/experiment/launch_finetune.py @@ -105,6 +105,7 @@ def build_config(ft_config: FinetuneConfig): if ft_config.shortest_image_edge is not None: config.model.shortest_image_edge = ft_config.shortest_image_edge config.model.crop_fraction = ft_config.crop_fraction + if config.model.shortest_image_edge is not None and config.model.crop_fraction is not None: config.model.image_crop_size = None config.model.image_target_size = None if ft_config.extra_augmentation_config: diff --git a/tests/gr00t/experiment/test_finetune_checkpoint.py b/tests/gr00t/experiment/test_finetune_checkpoint.py index 3ac2bc0a4..f71420750 100644 --- a/tests/gr00t/experiment/test_finetune_checkpoint.py +++ b/tests/gr00t/experiment/test_finetune_checkpoint.py @@ -8,6 +8,7 @@ from gr00t.configs.finetune_config import FinetuneConfig from gr00t.data.types import ActionFormat, ActionRepresentation, ActionType +from gr00t.experiment.experiment import warn_configs from gr00t.experiment.launch_finetune import build_config import pytest @@ -56,6 +57,7 @@ def test_finetuning_retains_droid_checkpoint_contract(cameras): assert not config.model.tune_llm assert not config.model.tune_visual assert config.training.resume_from_checkpoint + warn_configs(config) def test_camera_override_does_not_change_a_subsequent_run(): diff --git a/tests/gr00t/policy/test_gr00t_policy.py b/tests/gr00t/policy/test_gr00t_policy.py index 89a99000f..3fbee5a2b 100644 --- a/tests/gr00t/policy/test_gr00t_policy.py +++ b/tests/gr00t/policy/test_gr00t_policy.py @@ -49,8 +49,7 @@ def _build_modality_configs(): } -@pytest.fixture(params=["/fake/path", "hf://owner/model"]) -def policy(request): +def _make_policy(model_path): mock_model = MagicMock() mock_model.eval = MagicMock() mock_model.to = MagicMock(return_value=mock_model) @@ -107,10 +106,10 @@ def fake_decode_action(action, embodiment_tag, state=None): p = Gr00tPolicy( embodiment_tag=EMBODIMENT, - model_path=request.param, + model_path=model_path, device="cpu", ) - if request.param.startswith("hf://"): + if model_path.startswith("hf://"): download.assert_called_once_with("owner/model") else: download.assert_not_called() @@ -118,6 +117,16 @@ def fake_decode_action(action, embodiment_tag, state=None): return p +@pytest.fixture +def policy(): + return _make_policy("/fake/path") + + +@pytest.mark.parametrize("model_path", ["/fake/path", "hf://owner/model"]) +def test_policy_resolves_local_and_hub_checkpoints(model_path): + _make_policy(model_path) + + def _make_observation(batch_size=1): return { "video": { From 9d17a03fc4d46cc738fd65ca6705eacf3f3c3d6f Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Thu, 10 Sep 2026 16:17:47 +0300 Subject: [PATCH 3/7] Honor checkpoint overrides and inference input contracts --- docker/Makefile | 2 +- docker/README.md | 3 + gr00t/data/types.py | 3 + .../model/gr00t_n1d7/processing_gr00t_n1d7.py | 5 +- gr00t/policy/gr00t_policy.py | 90 ++++++++----------- tests/gr00t/model/test_gr00t_processor.py | 22 +++++ tests/gr00t/policy/test_gr00t_policy.py | 18 ++-- 7 files changed, 81 insertions(+), 62 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index c47f779c1..9f9ef66a6 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -5,7 +5,7 @@ GIT_SHA := $(shell git rev-parse --short HEAD) IMAGE_TAG ?= $(shell git branch --show-current | tr '/' '-') build: - docker build --platform linux/amd64 -f Dockerfile -t $(IMAGE_NAME):local .. + docker build -f Dockerfile -t $(IMAGE_NAME):local .. tag: build docker tag $(IMAGE_NAME):local $(IMAGE_NAME):$(IMAGE_TAG) diff --git a/docker/README.md b/docker/README.md index 2499754e3..0cf15b758 100644 --- a/docker/README.md +++ b/docker/README.md @@ -20,6 +20,9 @@ make -C docker build This builds from `nvidia/cuda:12.8.0-devel-ubuntu24.04` and installs all dependencies into `/opt/gr00t-venv`. The image includes this fork at `/gr00t`, installed into `/opt/gr00t-venv`. Positronic launches that environment directly. +Docker builds for the host architecture. To cross-build, use Docker's standard setting, +for example `DOCKER_DEFAULT_PLATFORM=linux/amd64 make -C docker build`. + ## Positronic base image ```bash diff --git a/gr00t/data/types.py b/gr00t/data/types.py index 7bd6d5fe5..28b18d781 100644 --- a/gr00t/data/types.py +++ b/gr00t/data/types.py @@ -22,6 +22,9 @@ from gr00t.data.embodiment_tags import EmbodimentTag +LANGUAGE = "language" + + class MessageType(Enum): START_OF_EPISODE = "start_of_episode" END_OF_EPISODE = "end_of_episode" diff --git a/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py b/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py index ec0d59f77..c56230015 100644 --- a/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py +++ b/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py @@ -36,6 +36,7 @@ from gr00t.data.embodiment_tags import EmbodimentTag from gr00t.data.interfaces import BaseProcessor from gr00t.data.state_action.state_action_processor import StateActionProcessor +from gr00t.data.types import LANGUAGE from gr00t.data.utils import parse_modality_configs, to_json_serializable from .image_augmentations import ( @@ -507,7 +508,7 @@ def process_observation(self, observation: dict[str, Any], embodiment_tag: Embod images_perm = images.permute(0, 1, 2, 5, 3, 4).reshape(B, T * V, img_C, img_H, img_W) transformed_images = self.eval_image_transform(images_perm).numpy() - language_key = modality_config["language"].modality_keys[0] + language_key = modality_config[LANGUAGE].modality_keys[0] language = [ re.sub(r"[^\w\s]", "", lang.lower()) if self.formalize_language else lang for lang in observation[language_key] @@ -872,6 +873,8 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | Path, **kwargs): "exclude_state", "state_dropout_prob", "use_mean_std", + "use_percentiles", + "extra_augmentation_config", "model_name", "model_type", "max_action_horizon", diff --git a/gr00t/policy/gr00t_policy.py b/gr00t/policy/gr00t_policy.py index 73289f6fc..523872679 100644 --- a/gr00t/policy/gr00t_policy.py +++ b/gr00t/policy/gr00t_policy.py @@ -30,7 +30,7 @@ from gr00t.data.embodiment_tags import FINETUNE_ONLY_TAGS, POSTTRAIN_TAGS, EmbodimentTag from gr00t.data.interfaces import BaseProcessor -from gr00t.data.types import MessageType, ModalityConfig, VLAStepData +from gr00t.data.types import LANGUAGE, MessageType, ModalityConfig, VLAStepData from .policy import BasePolicy, PolicyWrapper @@ -172,8 +172,8 @@ def __init__( # Extract and validate language configuration # Some embodiments (e.g. OXE_DROID) define multiple language keys for # training-time augmentation (paraphrases). At inference we only use the first key. - language_keys = self.modality_configs["language"].modality_keys - language_delta_indices = self.modality_configs["language"].delta_indices + language_keys = self.modality_configs[LANGUAGE].modality_keys + language_delta_indices = self.modality_configs[LANGUAGE].delta_indices assert len(language_keys) >= 1, "At least one language key is required" assert len(language_delta_indices) == 1, "Only one language delta index is supported" self.language_key = language_keys[0] @@ -196,7 +196,7 @@ def _unbatch_observation(self, value: dict[str, Any]) -> list[dict[str, Any]]: unbatched_value = { "video": {k: v[i] for k, v in value["video"].items()}, "state": {k: v[i] for k, v in value["state"].items()}, - "language": {k: v[i] for k, v in value["language"].items()}, + LANGUAGE: {k: v[i] for k, v in value[LANGUAGE].items()}, } unbatched_obs.append(unbatched_value) return unbatched_obs @@ -214,7 +214,7 @@ def _to_vla_step_data(self, observation: dict[str, Any]) -> VLAStepData: images=observation["video"], states=observation["state"], actions={}, # No ground truth actions during inference - text=observation["language"][self.language_key][0], + text=observation[LANGUAGE][self.language_key][0], embodiment=self.embodiment_tag, ) @@ -245,7 +245,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: AssertionError: If any validation check fails """ # Check that observation contains all required top-level modality keys - for modality in ["video", "state", "language"]: + for modality in ["video", "state", LANGUAGE]: assert modality in observation, f"Observation must contain a '{modality}' key" assert isinstance(observation[modality], dict), ( f"Observation '{modality}' must be a dictionary. Got {type(observation[modality])}: {observation[modality]}" @@ -341,51 +341,31 @@ def check_observation(self, observation: dict[str, Any]) -> None: f"State key '{state_key}'s horizon must be {len(self.modality_configs['state'].delta_indices)}. Got {batched_state.shape[1]}" ) - # ===== LANGUAGE VALIDATION ===== - # Validate each language stream defined in the modality config - for language_key in self.modality_configs["language"].modality_keys: - # Check that the expected language key exists in the observation - # (must happen before indexing — see video validation above) - assert language_key in observation["language"], ( - f"Language key '{language_key}' must be in observation" + language_key = self.language_key + assert language_key in observation[LANGUAGE], ( + f"Language key '{language_key}' must be in observation" + ) + batched_language = observation[LANGUAGE][language_key] + assert isinstance(batched_language, list), ( + f"Language key '{language_key}' must be a list. Got {type(batched_language)}" + ) + if bs != -1: + assert len(batched_language) == bs, ( + f"Language key '{language_key}' must have batch size {bs}. Got {len(batched_language)}" ) - - # Set or verify batch size consistency (language uses len instead of .shape) - if bs == -1: - bs = len(observation["language"][language_key]) - else: - assert len(observation["language"][language_key]) == bs, ( - f"Language key '{language_key}' must have batch size {bs}. Got {len(observation['language'][language_key])}" - ) - - batched_language: list[list[str]] = observation["language"][language_key] - - # Verify outer structure is a list (batch dimension) - assert isinstance(batched_language, list), ( - f"Language key '{language_key}' must be a list. Got {type(batched_language)}" + for batch_item in batched_language: + assert isinstance(batch_item, list), ( + f"Language batch item must be a list. Got {type(batch_item)}" + ) + assert len(batch_item) == len(self.modality_configs[LANGUAGE].delta_indices), ( + f"Language key '{language_key}'s horizon must be {len(self.modality_configs[LANGUAGE].delta_indices)}. Got {len(batch_item)}" + ) + assert len(batch_item) == 1, ( + f"Language batch item must have exactly one item. Got {len(batch_item)}" + ) + assert isinstance(batch_item[0], str), ( + f"Language batch item must be a string. Got {type(batch_item[0])}" ) - - # Validate each batch item - for batch_item in batched_language: - # Verify temporal dimension matches expected horizon - assert len(batch_item) == len(self.modality_configs["language"].delta_indices), ( - f"Language key '{language_key}'s horizon must be {len(self.modality_configs['language'].delta_indices)}. Got {len(batched_language)}" - ) - - # Verify inner structure is also a list (temporal dimension) - assert isinstance(batch_item, list), ( - f"Language batch item must be a list. Got {type(batch_item)}" - ) - - # Current implementation expects exactly one language instruction per timestep - assert len(batch_item) == 1, ( - f"Language batch item must have exactly one item. Got {len(batch_item)}" - ) - - # Verify the instruction itself is a string - assert isinstance(batch_item[0], str), ( - f"Language batch item must be a string. Got {type(batch_item[0])}" - ) def _get_action( self, observation: dict[str, Any], options: dict[str, Any] | None = None @@ -516,7 +496,7 @@ class Gr00tSimPolicyWrapper(PolicyWrapper): Key transformations performed by this wrapper: - Observation keys: 'video.cam' -> observation['video']['cam'] - Observation keys: 'state.joints' -> observation['state']['joints'] - - Language keys: 'task' or 'annotation.human.coarse_action' -> observation['language']['task'] + - Language keys: 'task' or 'annotation.human.coarse_action' -> observation[LANGUAGE]['task'] - Action keys: action['joints'] -> 'action.joints' """ @@ -529,7 +509,7 @@ def __init__(self, policy: Gr00tPolicy, *, strict: bool = True): """ super().__init__(policy, strict=strict) self.policy: Gr00tPolicy = policy - assert len(self.policy.modality_configs["language"].delta_indices) == 1, ( + assert len(self.policy.modality_configs[LANGUAGE].delta_indices) == 1, ( "Only one language delta index is supported" ) @@ -618,7 +598,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== LANGUAGE VALIDATION ===== # Check language modalities (special handling for DC environment compatibility) - for language_key in modality_configs["language"].modality_keys: + for language_key in modality_configs[LANGUAGE].modality_keys: # PATCH: Legacy compatibility for DC environments # DC envs use 'annotation.human.coarse_action' instead of 'task' if language_key == "task" and "annotation.human.coarse_action" in observation: @@ -670,10 +650,10 @@ def _get_action( """ # Transform flat observation format to nested format expected by Gr00tPolicy new_obs = {} - for modality in ["video", "state", "language"]: + for modality in ["video", "state", LANGUAGE]: new_obs[modality] = {} for key in self.policy.modality_configs[modality].modality_keys: - if modality == "language": + if modality == LANGUAGE: # PATCH: Legacy compatibility for DC environments if key == "task" and "annotation.human.coarse_action" in observation: parsed_key = "annotation.human.coarse_action" @@ -687,7 +667,7 @@ def _get_action( arr = observation[parsed_key] # Transform to nested format - if modality == "language": + if modality == LANGUAGE: arr = _sim_language_batch_to_sequence(arr) # Convert from tuple[str] or list[str] (B,) to list[list[str]] (B, 1) # Each element becomes a list with one string for temporal dimension diff --git a/tests/gr00t/model/test_gr00t_processor.py b/tests/gr00t/model/test_gr00t_processor.py index 590b0ea46..f7517775b 100644 --- a/tests/gr00t/model/test_gr00t_processor.py +++ b/tests/gr00t/model/test_gr00t_processor.py @@ -22,6 +22,7 @@ import json from pathlib import Path +import shutil import tempfile from unittest.mock import MagicMock, patch @@ -105,6 +106,27 @@ def fake_cached_file(path_or_repo_id, filename, **kwargs): } +@pytest.mark.parametrize("use_percentiles", [True, False]) +def test_checkpoint_overrides_reach_normalization_and_mask_transforms(tmp_path, use_percentiles): + from gr00t.model.gr00t_n1d7 import processing_gr00t_n1d7 as processor_module + from gr00t.model.gr00t_n1d7.image_augmentations import BackgroundNoiseTransform + + shutil.copytree(FIXTURE_DIR, tmp_path, dirs_exist_ok=True) + config_path = tmp_path / "processor_config.json" + config = json.loads(config_path.read_text()) + config["processor_kwargs"]["use_percentiles"] = not use_percentiles + config_path.write_text(json.dumps(config)) + augmentation = {"background_noise_transforms": [{"target_mask_values": [0], "p": 1.0}]} + with patch.object(processor_module, "build_processor", return_value=MagicMock()): + processor = processor_module.Gr00tN1d7Processor.from_pretrained( + tmp_path, use_percentiles=use_percentiles, extra_augmentation_config=augmentation + ) + assert processor.state_action_processor.use_percentiles is use_percentiles + assert processor.extra_augmentation_config == augmentation + assert len(processor.train_image_transform.mask_transforms) == 1 + assert isinstance(processor.train_image_transform.mask_transforms[0], BackgroundNoiseTransform) + + def _make_step_data(proc_config) -> VLAStepData: """Create synthetic VLAStepData matching the fixture config.""" import json as _json diff --git a/tests/gr00t/policy/test_gr00t_policy.py b/tests/gr00t/policy/test_gr00t_policy.py index 3fbee5a2b..dc3dd5c73 100644 --- a/tests/gr00t/policy/test_gr00t_policy.py +++ b/tests/gr00t/policy/test_gr00t_policy.py @@ -22,7 +22,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch -from gr00t.data.types import ModalityConfig +from gr00t.data.types import LANGUAGE, ModalityConfig import numpy as np import pytest import torch @@ -44,7 +44,7 @@ def _build_modality_configs(): "video": ModalityConfig(delta_indices=[0], modality_keys=VIDEO_KEYS), "state": ModalityConfig(delta_indices=[0], modality_keys=STATE_KEYS), "action": ModalityConfig(delta_indices=list(range(16)), modality_keys=ACTION_KEYS), - "language": ModalityConfig(delta_indices=[0], modality_keys=[LANGUAGE_KEY]), + LANGUAGE: ModalityConfig(delta_indices=[0], modality_keys=[LANGUAGE_KEY]), } } @@ -138,7 +138,7 @@ def _make_observation(batch_size=1): for k in STATE_KEYS[:-1] # all except gripper } | {"gripper": np.random.randn(batch_size, 1, 2).astype(np.float32)}, - "language": { + LANGUAGE: { LANGUAGE_KEY: [["pick up the apple"]] * batch_size, }, } @@ -154,6 +154,14 @@ def test_policy_embodiment_tag(self, policy): class TestGr00tPolicyCheckObservation: + def test_inference_requires_only_the_selected_language_key(self, policy): + policy.modality_configs[LANGUAGE].modality_keys.append("training_paraphrase") + observation = _make_observation() + policy.get_action(observation) + del observation[LANGUAGE][LANGUAGE_KEY] + with pytest.raises(AssertionError, match="Language key"): + policy.check_observation(observation) + def test_extra_camera_cannot_be_silently_ignored(self, policy): obs = _make_observation() obs["video"]["second_external"] = obs["video"][VIDEO_KEYS[0]] @@ -200,7 +208,7 @@ def __init__(self): modality_keys=["state"], ), "action": ModalityConfig(delta_indices=[0], modality_keys=["action"]), - "language": ModalityConfig( + LANGUAGE: ModalityConfig( delta_indices=[0], modality_keys=["annotation.human.action.task_description"], ), @@ -231,6 +239,6 @@ def test_sim_policy_wrapper_accepts_numpy_language_batches(): action, info = wrapper.get_action(observation) - assert policy.last_observation["language"][LANGUAGE_KEY] == [["follow the instruction"]] + assert policy.last_observation[LANGUAGE][LANGUAGE_KEY] == [["follow the instruction"]] assert "action.action" in action assert info == {} From bc91bc1dface2c42ebcedcb1f820ec64dc44f75e Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Thu, 10 Sep 2026 16:29:49 +0300 Subject: [PATCH 4/7] Document camera overrides through the fine-tuning wrapper --- docker/README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 0cf15b758..c4c7c359f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -34,7 +34,18 @@ The Makefile publishes `positro/gr00t-base` with branch and commit tags. It does `latest`. Positronic's `GROOT_BASE_IMAGE` selects this image for its adapter build. Fine-tuning defaults to the base checkpoint's saved model and modality configuration. -`--video-keys` selects the fine-tuning camera layout; omitted, it retains the checkpoint's views. +The Python launcher accepts `--video-keys` to select the fine-tuning camera layout; omitted, it retains the checkpoint's views. +When using `examples/finetune.sh`, put this option after the script's `--` passthrough delimiter: + +```bash +bash examples/finetune.sh \ + --base-model-path nvidia/GR00T-N1.7-DROID \ + --dataset-path /data/droid \ + --embodiment-tag oxe_droid_relative_eef_relative_joint \ + --output-dir /data/checkpoints \ + -- --video-keys exterior_image_1_left exterior_image_2_left wrist_image_left +``` + The policy server accepts `--model-path hf://nvidia/GR00T-N1.7-DROID` and downloads that snapshot. ## Running the Container From e63c6f70257574cd7cf41a5abd24d3f0cab1306c Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Thu, 10 Sep 2026 16:50:33 +0300 Subject: [PATCH 5/7] Use the selected inference instruction in the sim wrapper --- gr00t/data/types.py | 4 + gr00t/policy/gr00t_policy.py | 162 +++++++++++------------- tests/gr00t/policy/test_gr00t_policy.py | 22 +++- 3 files changed, 97 insertions(+), 91 deletions(-) diff --git a/gr00t/data/types.py b/gr00t/data/types.py index 28b18d781..859d23520 100644 --- a/gr00t/data/types.py +++ b/gr00t/data/types.py @@ -22,7 +22,11 @@ from gr00t.data.embodiment_tags import EmbodimentTag +VIDEO = "video" +STATE = "state" LANGUAGE = "language" +TASK_LANGUAGE_KEY = "task" +COARSE_ACTION_LANGUAGE_KEY = "annotation.human.coarse_action" class MessageType(Enum): diff --git a/gr00t/policy/gr00t_policy.py b/gr00t/policy/gr00t_policy.py index 523872679..7d4d4ccfc 100644 --- a/gr00t/policy/gr00t_policy.py +++ b/gr00t/policy/gr00t_policy.py @@ -30,7 +30,16 @@ from gr00t.data.embodiment_tags import FINETUNE_ONLY_TAGS, POSTTRAIN_TAGS, EmbodimentTag from gr00t.data.interfaces import BaseProcessor -from gr00t.data.types import LANGUAGE, MessageType, ModalityConfig, VLAStepData +from gr00t.data.types import ( + COARSE_ACTION_LANGUAGE_KEY, + LANGUAGE, + STATE, + TASK_LANGUAGE_KEY, + VIDEO, + MessageType, + ModalityConfig, + VLAStepData, +) from .policy import BasePolicy, PolicyWrapper @@ -189,13 +198,13 @@ def _unbatch_observation(self, value: dict[str, Any]) -> list[dict[str, Any]]: """ unbatched_obs = [] # Infer batch size from the first video key - batch_size = value["video"][list(value["video"].keys())[0]].shape[0] + batch_size = value[VIDEO][list(value[VIDEO].keys())[0]].shape[0] # Split each modality along the batch dimension for i in range(batch_size): unbatched_value = { - "video": {k: v[i] for k, v in value["video"].items()}, - "state": {k: v[i] for k, v in value["state"].items()}, + VIDEO: {k: v[i] for k, v in value[VIDEO].items()}, + STATE: {k: v[i] for k, v in value[STATE].items()}, LANGUAGE: {k: v[i] for k, v in value[LANGUAGE].items()}, } unbatched_obs.append(unbatched_value) @@ -211,8 +220,8 @@ def _to_vla_step_data(self, observation: dict[str, Any]) -> VLAStepData: VLAStepData object ready for processor input """ return VLAStepData( - images=observation["video"], - states=observation["state"], + images=observation[VIDEO], + states=observation[STATE], actions={}, # No ground truth actions during inference text=observation[LANGUAGE][self.language_key][0], embodiment=self.embodiment_tag, @@ -245,7 +254,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: AssertionError: If any validation check fails """ # Check that observation contains all required top-level modality keys - for modality in ["video", "state", LANGUAGE]: + for modality in [VIDEO, STATE, LANGUAGE]: assert modality in observation, f"Observation must contain a '{modality}' key" assert isinstance(observation[modality], dict), ( f"Observation '{modality}' must be a dictionary. Got {type(observation[modality])}: {observation[modality]}" @@ -255,27 +264,27 @@ def check_observation(self, observation: dict[str, Any]) -> None: bs = -1 # ===== VIDEO VALIDATION ===== - expected_cameras = set(self.modality_configs["video"].modality_keys) - if set(observation["video"]) - expected_cameras: + expected_cameras = set(self.modality_configs[VIDEO].modality_keys) + if set(observation[VIDEO]) - expected_cameras: raise ValueError( f"Checkpoint cameras {sorted(expected_cameras)} do not match " - f"observation cameras {sorted(observation['video'])}" + f"observation cameras {sorted(observation[VIDEO])}" ) # Validate each video stream defined in the modality config - for video_key in self.modality_configs["video"].modality_keys: - assert video_key in observation["video"], ( + for video_key in self.modality_configs[VIDEO].modality_keys: + assert video_key in observation[VIDEO], ( f"Video key '{video_key}' must be in observation" ) # Set or verify batch size consistency across all video keys if bs == -1: - bs = len(observation["video"][video_key]) + bs = len(observation[VIDEO][video_key]) else: - assert len(observation["video"][video_key]) == bs, ( - f"Video key '{video_key}' must have batch size {bs}. Got {len(observation['video'][video_key])}" + assert len(observation[VIDEO][video_key]) == bs, ( + f"Video key '{video_key}' must have batch size {bs}. Got {len(observation[VIDEO][video_key])}" ) - batched_video = observation["video"][video_key] + batched_video = observation[VIDEO][video_key] # Verify data type is numpy array assert isinstance(batched_video, np.ndarray), ( @@ -293,8 +302,8 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_video.shape[1] == len(self.modality_configs["video"].delta_indices), ( - f"Video key '{video_key}'s horizon must be {len(self.modality_configs['video'].delta_indices)}. Got {batched_video.shape[1]}" + assert batched_video.shape[1] == len(self.modality_configs[VIDEO].delta_indices), ( + f"Video key '{video_key}'s horizon must be {len(self.modality_configs[VIDEO].delta_indices)}. Got {batched_video.shape[1]}" ) # Verify channel dimension is 3 (RGB images) @@ -304,22 +313,22 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== STATE VALIDATION ===== # Validate each state stream defined in the modality config - for state_key in self.modality_configs["state"].modality_keys: + for state_key in self.modality_configs[STATE].modality_keys: # Check that the expected state key exists in the observation # (must happen before indexing — see video validation above) - assert state_key in observation["state"], ( + assert state_key in observation[STATE], ( f"State key '{state_key}' must be in observation" ) # Set or verify batch size consistency across all state keys if bs == -1: - bs = len(observation["state"][state_key]) + bs = len(observation[STATE][state_key]) else: - assert len(observation["state"][state_key]) == bs, ( - f"State key '{state_key}' must have batch size {bs}. Got {len(observation['state'][state_key])}" + assert len(observation[STATE][state_key]) == bs, ( + f"State key '{state_key}' must have batch size {bs}. Got {len(observation[STATE][state_key])}" ) - batched_state = observation["state"][state_key] + batched_state = observation[STATE][state_key] # Verify data type is numpy array assert isinstance(batched_state, np.ndarray), ( @@ -337,8 +346,8 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_state.shape[1] == len(self.modality_configs["state"].delta_indices), ( - f"State key '{state_key}'s horizon must be {len(self.modality_configs['state'].delta_indices)}. Got {batched_state.shape[1]}" + assert batched_state.shape[1] == len(self.modality_configs[STATE].delta_indices), ( + f"State key '{state_key}'s horizon must be {len(self.modality_configs[STATE].delta_indices)}. Got {batched_state.shape[1]}" ) language_key = self.language_key @@ -409,7 +418,7 @@ def _get_action( # Step 5: Decode actions from normalized space back to physical units batched_states = {} - for k in self.modality_configs["state"].modality_keys: + for k in self.modality_configs[STATE].modality_keys: batched_states[k] = np.stack([s[k] for s in states], axis=0) # (B, T, D) unnormalized_action = self.processor.decode_action( normalized_action.cpu().numpy(), self.embodiment_tag, batched_states @@ -494,8 +503,8 @@ class Gr00tSimPolicyWrapper(PolicyWrapper): This wrapper is only needed for compatibility with the existing Gr00t sim infrastructure. Key transformations performed by this wrapper: - - Observation keys: 'video.cam' -> observation['video']['cam'] - - Observation keys: 'state.joints' -> observation['state']['joints'] + - Observation keys: 'video.cam' -> observation[VIDEO]['cam'] + - Observation keys: 'state.joints' -> observation[STATE]['joints'] - Language keys: 'task' or 'annotation.human.coarse_action' -> observation[LANGUAGE]['task'] - Action keys: action['joints'] -> 'action.joints' """ @@ -513,6 +522,13 @@ def __init__(self, policy: Gr00tPolicy, *, strict: bool = True): "Only one language delta index is supported" ) + def _language_observation_key(self, observation: dict[str, Any]) -> str: + key = self.policy.language_key + # DC environments expose their instruction under this annotation key. + if key == TASK_LANGUAGE_KEY and COARSE_ACTION_LANGUAGE_KEY in observation: + return COARSE_ACTION_LANGUAGE_KEY + return key + def check_observation(self, observation: dict[str, Any]) -> None: """Validate observation from Gr00t sim environment format. @@ -535,7 +551,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== VIDEO VALIDATION ===== # Check video modalities with flat key format: 'video.camera_name' - for video_key in modality_configs["video"].modality_keys: + for video_key in modality_configs[VIDEO].modality_keys: # Construct flat key expected in Gr00t sim environment parsed_key = f"video.{video_key}" assert parsed_key in observation, f"Video key '{parsed_key}' must be in observation" @@ -558,8 +574,8 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_video.shape[1] == len(modality_configs["video"].delta_indices), ( - f"Video key '{video_key}'s horizon must be {len(modality_configs['video'].delta_indices)}. Got {batched_video.shape[1]}" + assert batched_video.shape[1] == len(modality_configs[VIDEO].delta_indices), ( + f"Video key '{video_key}'s horizon must be {len(modality_configs[VIDEO].delta_indices)}. Got {batched_video.shape[1]}" ) # Verify channel dimension is 3 (RGB images) @@ -569,7 +585,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== STATE VALIDATION ===== # Check state modalities with flat key format: 'state.state_name' - for state_key in modality_configs["state"].modality_keys: + for state_key in modality_configs[STATE].modality_keys: # Construct flat key expected in Gr00t sim environment parsed_key = f"state.{state_key}" assert parsed_key in observation, f"State key '{parsed_key}' must be in observation" @@ -592,38 +608,20 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_state.shape[1] == len(modality_configs["state"].delta_indices), ( - f"State key '{state_key}'s horizon must be {len(modality_configs['state'].delta_indices)}. Got {batched_state.shape[1]}" - ) - - # ===== LANGUAGE VALIDATION ===== - # Check language modalities (special handling for DC environment compatibility) - for language_key in modality_configs[LANGUAGE].modality_keys: - # PATCH: Legacy compatibility for DC environments - # DC envs use 'annotation.human.coarse_action' instead of 'task' - if language_key == "task" and "annotation.human.coarse_action" in observation: - language_key = "annotation.human.coarse_action" - # /PATCH - - # Check that the expected language key exists - assert language_key in observation, ( - f"Language key '{language_key}' must be in observation" + assert batched_state.shape[1] == len(modality_configs[STATE].delta_indices), ( + f"State key '{state_key}'s horizon must be {len(modality_configs[STATE].delta_indices)}. Got {batched_state.shape[1]}" ) - # In Gr00t sim format, language is a tuple of strings (B,) - batched_language = _sim_language_batch_to_sequence(observation[language_key]) - - # Verify outer structure is a tuple (batch dimension) - assert isinstance(batched_language, (tuple, list)), ( - f"Language key '{language_key}' must be a tuple, list, or numpy array. " - f"Got {type(observation[language_key])}" - ) - assert batched_language, f"Language key '{language_key}' must not be empty" - - # Verify each batch item is a string - assert isinstance(batched_language[0], str), ( - f"Language batch item must be a string. Got {type(batched_language[0])}" - ) + language_key = self._language_observation_key(observation) + assert language_key in observation, f"Language key '{language_key}' must be in observation" + batched_language = _sim_language_batch_to_sequence(observation[language_key]) + assert isinstance(batched_language, (tuple, list)), ( + f"Language key '{language_key}' must be a tuple, list, or numpy array. " + f"Got {type(observation[language_key])}" + ) + assert batched_language, f"Language key '{language_key}' must not be empty" + for item in batched_language: + assert isinstance(item, str), f"Language batch item must be a string. Got {type(item)}" def _get_action( self, observation: dict[str, Any], options: dict[str, Any] | None = None @@ -648,33 +646,17 @@ def _get_action( Returns: Tuple of (flat_actions_dict, info_dict) """ - # Transform flat observation format to nested format expected by Gr00tPolicy - new_obs = {} - for modality in ["video", "state", LANGUAGE]: - new_obs[modality] = {} - for key in self.policy.modality_configs[modality].modality_keys: - if modality == LANGUAGE: - # PATCH: Legacy compatibility for DC environments - if key == "task" and "annotation.human.coarse_action" in observation: - parsed_key = "annotation.human.coarse_action" - # /PATCH - else: - parsed_key = key - else: - # Construct flat key (e.g., 'video.camera' or 'state.joints') - parsed_key = f"{modality}.{key}" - - arr = observation[parsed_key] - - # Transform to nested format - if modality == LANGUAGE: - arr = _sim_language_batch_to_sequence(arr) - # Convert from tuple[str] or list[str] (B,) to list[list[str]] (B, 1) - # Each element becomes a list with one string for temporal dimension - new_obs[modality][key] = [[str(item)] for item in arr] - else: - # Video and state arrays are already in correct format (B, T, ...) - new_obs[modality][key] = arr + new_obs = { + modality: { + key: observation[f"{modality}.{key}"] + for key in self.policy.modality_configs[modality].modality_keys + } + for modality in (VIDEO, STATE) + } + language = _sim_language_batch_to_sequence( + observation[self._language_observation_key(observation)] + ) + new_obs[LANGUAGE] = {self.policy.language_key: [[str(item)] for item in language]} # Compute actions using the underlying Gr00tPolicy action, info = self.policy.get_action(new_obs, options) diff --git a/tests/gr00t/policy/test_gr00t_policy.py b/tests/gr00t/policy/test_gr00t_policy.py index dc3dd5c73..71d53eafa 100644 --- a/tests/gr00t/policy/test_gr00t_policy.py +++ b/tests/gr00t/policy/test_gr00t_policy.py @@ -213,6 +213,7 @@ def __init__(self): modality_keys=["annotation.human.action.task_description"], ), } + self.language_key = self.modality_configs[LANGUAGE].modality_keys[0] self.last_observation = None def get_modality_config(self): @@ -226,10 +227,12 @@ def reset(self, options=None): return {} -def test_sim_policy_wrapper_accepts_numpy_language_batches(): +@pytest.mark.parametrize("extra_language_keys", [[], ["training_paraphrase"]]) +def test_sim_policy_wrapper_accepts_numpy_language_batches(extra_language_keys): from gr00t.policy.gr00t_policy import Gr00tSimPolicyWrapper policy = _NumpyLanguageSimPolicy() + policy.modality_configs[LANGUAGE].modality_keys.extend(extra_language_keys) wrapper = Gr00tSimPolicyWrapper(policy) observation = { "video.camera": np.zeros((1, 1, 256, 256, 3), dtype=np.uint8), @@ -240,5 +243,22 @@ def test_sim_policy_wrapper_accepts_numpy_language_batches(): action, info = wrapper.get_action(observation) assert policy.last_observation[LANGUAGE][LANGUAGE_KEY] == [["follow the instruction"]] + assert list(policy.last_observation[LANGUAGE]) == [policy.language_key] assert "action.action" in action assert info == {} + + +def test_sim_policy_wrapper_requires_selected_language_key(): + from gr00t.policy.gr00t_policy import Gr00tSimPolicyWrapper + + policy = _NumpyLanguageSimPolicy() + policy.modality_configs[LANGUAGE].modality_keys.append("training_paraphrase") + wrapper = Gr00tSimPolicyWrapper(policy) + observation = { + "video.camera": np.zeros((1, 1, 256, 256, 3), dtype=np.uint8), + "state.state": np.zeros((1, 1, 3), dtype=np.float32), + "training_paraphrase": ["follow the instruction"], + } + with pytest.raises(AssertionError, match="Language key .* must be in observation"): + wrapper.get_action(observation) + assert policy.last_observation is None From ee9dd54d3f38150253c6c160fb21b18ae9673562 Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Fri, 11 Sep 2026 13:50:16 +0300 Subject: [PATCH 6/7] Keep the N1.7 upgrade focused on checkpoint compatibility --- .github/workflows/docker-build.yml | 31 ++- docker/Makefile | 81 +++++- docker/README.md | 48 ++-- gr00t/data/types.py | 3 - .../model/gr00t_n1d7/processing_gr00t_n1d7.py | 3 +- gr00t/policy/gr00t_policy.py | 236 ++++++++++-------- 6 files changed, 260 insertions(+), 142 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3af3c3dc4..079a30421 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,19 +1,36 @@ -name: Build and Push Positronic GR00T Base +name: Build and Push Gr00t Images on: push: - branches: [main-positronic] - workflow_dispatch: + branches: + - main-positronic jobs: build-and-push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: docker/login-action@v3 + - name: Checkout + uses: actions/checkout@v4 + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf "${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}/CodeQL" + sudo rm -rf "${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}/node" + sudo rm -rf "${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}/go" + sudo rm -rf /usr/lib/jvm + sudo docker system prune -af || true + df -h + + - name: Log in to Docker Hub + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push branch and commit tags + + - name: Build and Push working-directory: docker - run: make push + run: | + make push diff --git a/docker/Makefile b/docker/Makefile index 9f9ef66a6..22ae9a0db 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,16 +1,79 @@ -.PHONY: build tag push +.PHONY: all build tag push clean prune help -IMAGE_NAME := positro/gr00t-base -GIT_SHA := $(shell git rev-parse --short HEAD) -IMAGE_TAG ?= $(shell git branch --show-current | tr '/' '-') +# Image configuration +IMAGE_NAME_GROOT := positro/gr00t-base + +# Extract version from pyproject.toml (first literal version entry) +VERSION := $(shell sed -n 's/^version = "\([^"]*\)"/\1/p' ../pyproject.toml | head -n 1) + +GIT_SHA := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") + +REGISTRY_URL ?= docker.io + +TAG_GROOT_LATEST := $(IMAGE_NAME_GROOT):latest +TAG_GROOT_VERSION := $(IMAGE_NAME_GROOT):v$(VERSION) +TAG_GROOT_SHA := $(IMAGE_NAME_GROOT):$(GIT_SHA) +LOCAL_TAG_GROOT := $(IMAGE_NAME_GROOT):local + +help: + @echo "gr00t Groot Docker Build System" + @echo "" + @echo "Configuration:" + @echo " Groot Image: $(IMAGE_NAME_GROOT)" + @echo " Version: $(VERSION)" + @echo " Git SHA: $(GIT_SHA)" + @echo " Registry URL: $(REGISTRY_URL)" + @echo "" + @echo "Targets:" + @echo " make build Build the groot image" + @echo " make tag Tag the groot image" + @echo " make push Push groot tags to Docker Hub" + @echo " make clean Remove local groot images" + @echo " make prune Remove dangling/unused Docker images" + @echo " make help Show this help message" + @echo "" build: - docker build -f Dockerfile -t $(IMAGE_NAME):local .. + @echo "Building $(IMAGE_NAME_GROOT) image..." + @if [ -z "$(VERSION)" ]; then \ + echo "Error: Could not extract version from pyproject.toml"; \ + exit 1; \ + fi + docker build \ + -f Dockerfile \ + -t $(LOCAL_TAG_GROOT) .. tag: build - docker tag $(IMAGE_NAME):local $(IMAGE_NAME):$(IMAGE_TAG) - docker tag $(IMAGE_NAME):local $(IMAGE_NAME):$(GIT_SHA) + @echo "Tagging groot image with multiple tags..." + docker tag $(LOCAL_TAG_GROOT) $(TAG_GROOT_LATEST) + docker tag $(LOCAL_TAG_GROOT) $(TAG_GROOT_VERSION) + docker tag $(LOCAL_TAG_GROOT) $(TAG_GROOT_SHA) + @echo "Tagged with:" + @echo " - $(TAG_GROOT_LATEST)" + @echo " - $(TAG_GROOT_VERSION)" + @echo " - $(TAG_GROOT_SHA)" push: tag - docker push $(IMAGE_NAME):$(IMAGE_TAG) - docker push $(IMAGE_NAME):$(GIT_SHA) + @echo "Pushing groot images to Docker Hub..." + docker push $(TAG_GROOT_LATEST) + docker push $(TAG_GROOT_VERSION) + docker push $(TAG_GROOT_SHA) + @echo "" + @echo "Successfully pushed groot images to Docker Hub!" + @echo "To use on cloud instances, set:" + @echo " export IMAGE_REGISTRY=$(REGISTRY_URL)/" + +all: push + +clean: + @echo "Removing local groot images..." + -docker rmi $(LOCAL_TAG_GROOT) + -docker rmi $(TAG_GROOT_LATEST) + -docker rmi $(TAG_GROOT_VERSION) + -docker rmi $(TAG_GROOT_SHA) + @echo "Cleanup complete." + +prune: + @echo "Pruning dangling and unused Docker images..." + docker image prune -f + @echo "Prune complete." diff --git a/docker/README.md b/docker/README.md index c4c7c359f..a4e117105 100644 --- a/docker/README.md +++ b/docker/README.md @@ -15,23 +15,21 @@ Docker configuration for building and running a containerized GR00T environment From the repository root: ```bash -make -C docker build +bash docker/build.sh ``` -This builds from `nvidia/cuda:12.8.0-devel-ubuntu24.04` and installs all dependencies into `/opt/gr00t-venv`. The image includes this fork at `/gr00t`, installed into `/opt/gr00t-venv`. Positronic launches that environment directly. - -Docker builds for the host architecture. To cross-build, use Docker's standard setting, -for example `DOCKER_DEFAULT_PLATFORM=linux/amd64 make -C docker build`. +This builds from `nvidia/cuda:12.8.0-devel-ubuntu24.04` and installs all dependencies into `/opt/gr00t-venv`. The image includes this fork at `/gr00t`, installed into `/opt/gr00t-venv`. ## Positronic base image ```bash make -C docker build -make -C docker push IMAGE_TAG=my-branch +make -C docker push ``` -The Makefile publishes `positro/gr00t-base` with branch and commit tags. It does not replace -`latest`. Positronic's `GROOT_BASE_IMAGE` selects this image for its adapter build. +The Makefile publishes `positro/gr00t-base` with `latest`, version, and commit tags. +It builds for the host architecture; use `DOCKER_DEFAULT_PLATFORM=linux/amd64` to cross-build. +Positronic's `GROOT_BASE_IMAGE` selects an existing base image for its adapter build. Fine-tuning defaults to the base checkpoint's saved model and modality configuration. The Python launcher accepts `--video-keys` to select the fine-tuning camera layout; omitted, it retains the checkpoint's views. @@ -50,39 +48,43 @@ The policy server accepts `--model-path hf://nvidia/GR00T-N1.7-DROID` and downlo ## Running the Container -Run the included fork from `/gr00t`: +**Run the included fork from `/gr00t`.** + +Start an interactive shell: ```bash docker run -it --rm --gpus all \ --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ - -v "$HOME/.cache/huggingface:/root/.cache/huggingface" \ - positro/gr00t-base:local + gr00t ``` -Inside the container: +Then, inside the container: ```bash cd /gr00t -uv run --no-sync python -c "import gr00t; print(gr00t.__file__)" +python -c "import gr00t; print('GR00T ready')" ``` -The image includes the fork and its locked environment at `/opt/gr00t-venv`. -The Hugging Face account must have access to the gated `nvidia/Cosmos-Reason2-2B` backbone. -Provide its token through the mounted Hugging Face cache or `HF_TOKEN`. +The image venv is active by default (`/opt/gr00t-venv`; `/workspace/.venv` is a compatibility symlink), and uv is configured with `UV_PROJECT_ENVIRONMENT=/opt/gr00t-venv`. After setting `PYTHONPATH` to the checked-out repo, both `python ...` and `uv run ...` use the global image venv instead of creating a checkout-local `.venv`. If you are working on an existing checkout in the container, run `git pull --ff-only` from that checkout instead of cloning again. -For development, mount a compatible fork checkout over `/gr00t`: +The global venv records the `uv.lock` hash it was built from. If your checked-out repo uses a different lockfile, create a checkout-local venv before running commands. Reusing a uv cache keeps this path from starting cold: ```bash -docker run -it --rm --gpus all --ipc=host \ - -v "$PWD:/gr00t" positro/gr00t-base:local +export UV_CACHE_DIR="${UV_CACHE_DIR:-/workspace/uv-cache}" +export UV_LINK_MODE=copy +UV_PROJECT_ENVIRONMENT="$PWD/.venv" uv sync +source .venv/bin/activate ``` -If its lockfile differs from the image, create a separate environment: +Do not run a bare `uv sync` unless you intend to update the global image venv. Use `UV_PROJECT_ENVIRONMENT="$PWD/.venv" uv sync` when you want an isolated per-checkout environment. + +Avoid bind-mounting over `/workspace`, because that can hide the image's `/workspace/.venv` compatibility symlink. If you need to mount local source for live editing, mount it under a subdirectory: ```bash -export UV_PROJECT_ENVIRONMENT=/gr00t/.venv -uv sync --locked --extra dev -source /gr00t/.venv/bin/activate +docker run -it --rm --gpus all \ + --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ + -v "$(pwd):/workspace/Isaac-GR00T" \ + gr00t bash -c 'cd /workspace/Isaac-GR00T && export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" && bash' ``` ## Edge Device Containers diff --git a/gr00t/data/types.py b/gr00t/data/types.py index 859d23520..379241a3c 100644 --- a/gr00t/data/types.py +++ b/gr00t/data/types.py @@ -23,10 +23,7 @@ VIDEO = "video" -STATE = "state" LANGUAGE = "language" -TASK_LANGUAGE_KEY = "task" -COARSE_ACTION_LANGUAGE_KEY = "annotation.human.coarse_action" class MessageType(Enum): diff --git a/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py b/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py index c56230015..4be4715ec 100644 --- a/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py +++ b/gr00t/model/gr00t_n1d7/processing_gr00t_n1d7.py @@ -36,7 +36,6 @@ from gr00t.data.embodiment_tags import EmbodimentTag from gr00t.data.interfaces import BaseProcessor from gr00t.data.state_action.state_action_processor import StateActionProcessor -from gr00t.data.types import LANGUAGE from gr00t.data.utils import parse_modality_configs, to_json_serializable from .image_augmentations import ( @@ -508,7 +507,7 @@ def process_observation(self, observation: dict[str, Any], embodiment_tag: Embod images_perm = images.permute(0, 1, 2, 5, 3, 4).reshape(B, T * V, img_C, img_H, img_W) transformed_images = self.eval_image_transform(images_perm).numpy() - language_key = modality_config[LANGUAGE].modality_keys[0] + language_key = modality_config["language"].modality_keys[0] language = [ re.sub(r"[^\w\s]", "", lang.lower()) if self.formalize_language else lang for lang in observation[language_key] diff --git a/gr00t/policy/gr00t_policy.py b/gr00t/policy/gr00t_policy.py index 7d4d4ccfc..173d56e25 100644 --- a/gr00t/policy/gr00t_policy.py +++ b/gr00t/policy/gr00t_policy.py @@ -30,16 +30,7 @@ from gr00t.data.embodiment_tags import FINETUNE_ONLY_TAGS, POSTTRAIN_TAGS, EmbodimentTag from gr00t.data.interfaces import BaseProcessor -from gr00t.data.types import ( - COARSE_ACTION_LANGUAGE_KEY, - LANGUAGE, - STATE, - TASK_LANGUAGE_KEY, - VIDEO, - MessageType, - ModalityConfig, - VLAStepData, -) +from gr00t.data.types import LANGUAGE, VIDEO, MessageType, ModalityConfig, VLAStepData from .policy import BasePolicy, PolicyWrapper @@ -181,8 +172,8 @@ def __init__( # Extract and validate language configuration # Some embodiments (e.g. OXE_DROID) define multiple language keys for # training-time augmentation (paraphrases). At inference we only use the first key. - language_keys = self.modality_configs[LANGUAGE].modality_keys - language_delta_indices = self.modality_configs[LANGUAGE].delta_indices + language_keys = self.modality_configs["language"].modality_keys + language_delta_indices = self.modality_configs["language"].delta_indices assert len(language_keys) >= 1, "At least one language key is required" assert len(language_delta_indices) == 1, "Only one language delta index is supported" self.language_key = language_keys[0] @@ -198,14 +189,14 @@ def _unbatch_observation(self, value: dict[str, Any]) -> list[dict[str, Any]]: """ unbatched_obs = [] # Infer batch size from the first video key - batch_size = value[VIDEO][list(value[VIDEO].keys())[0]].shape[0] + batch_size = value["video"][list(value["video"].keys())[0]].shape[0] # Split each modality along the batch dimension for i in range(batch_size): unbatched_value = { - VIDEO: {k: v[i] for k, v in value[VIDEO].items()}, - STATE: {k: v[i] for k, v in value[STATE].items()}, - LANGUAGE: {k: v[i] for k, v in value[LANGUAGE].items()}, + "video": {k: v[i] for k, v in value["video"].items()}, + "state": {k: v[i] for k, v in value["state"].items()}, + "language": {k: v[i] for k, v in value["language"].items()}, } unbatched_obs.append(unbatched_value) return unbatched_obs @@ -220,10 +211,10 @@ def _to_vla_step_data(self, observation: dict[str, Any]) -> VLAStepData: VLAStepData object ready for processor input """ return VLAStepData( - images=observation[VIDEO], - states=observation[STATE], + images=observation["video"], + states=observation["state"], actions={}, # No ground truth actions during inference - text=observation[LANGUAGE][self.language_key][0], + text=observation["language"][self.language_key][0], embodiment=self.embodiment_tag, ) @@ -254,7 +245,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: AssertionError: If any validation check fails """ # Check that observation contains all required top-level modality keys - for modality in [VIDEO, STATE, LANGUAGE]: + for modality in ["video", "state", "language"]: assert modality in observation, f"Observation must contain a '{modality}' key" assert isinstance(observation[modality], dict), ( f"Observation '{modality}' must be a dictionary. Got {type(observation[modality])}: {observation[modality]}" @@ -271,20 +262,20 @@ def check_observation(self, observation: dict[str, Any]) -> None: f"observation cameras {sorted(observation[VIDEO])}" ) # Validate each video stream defined in the modality config - for video_key in self.modality_configs[VIDEO].modality_keys: - assert video_key in observation[VIDEO], ( + for video_key in self.modality_configs["video"].modality_keys: + assert video_key in observation["video"], ( f"Video key '{video_key}' must be in observation" ) # Set or verify batch size consistency across all video keys if bs == -1: - bs = len(observation[VIDEO][video_key]) + bs = len(observation["video"][video_key]) else: - assert len(observation[VIDEO][video_key]) == bs, ( - f"Video key '{video_key}' must have batch size {bs}. Got {len(observation[VIDEO][video_key])}" + assert len(observation["video"][video_key]) == bs, ( + f"Video key '{video_key}' must have batch size {bs}. Got {len(observation['video'][video_key])}" ) - batched_video = observation[VIDEO][video_key] + batched_video = observation["video"][video_key] # Verify data type is numpy array assert isinstance(batched_video, np.ndarray), ( @@ -302,8 +293,8 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_video.shape[1] == len(self.modality_configs[VIDEO].delta_indices), ( - f"Video key '{video_key}'s horizon must be {len(self.modality_configs[VIDEO].delta_indices)}. Got {batched_video.shape[1]}" + assert batched_video.shape[1] == len(self.modality_configs["video"].delta_indices), ( + f"Video key '{video_key}'s horizon must be {len(self.modality_configs['video'].delta_indices)}. Got {batched_video.shape[1]}" ) # Verify channel dimension is 3 (RGB images) @@ -313,22 +304,22 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== STATE VALIDATION ===== # Validate each state stream defined in the modality config - for state_key in self.modality_configs[STATE].modality_keys: + for state_key in self.modality_configs["state"].modality_keys: # Check that the expected state key exists in the observation # (must happen before indexing — see video validation above) - assert state_key in observation[STATE], ( + assert state_key in observation["state"], ( f"State key '{state_key}' must be in observation" ) # Set or verify batch size consistency across all state keys if bs == -1: - bs = len(observation[STATE][state_key]) + bs = len(observation["state"][state_key]) else: - assert len(observation[STATE][state_key]) == bs, ( - f"State key '{state_key}' must have batch size {bs}. Got {len(observation[STATE][state_key])}" + assert len(observation["state"][state_key]) == bs, ( + f"State key '{state_key}' must have batch size {bs}. Got {len(observation['state'][state_key])}" ) - batched_state = observation[STATE][state_key] + batched_state = observation["state"][state_key] # Verify data type is numpy array assert isinstance(batched_state, np.ndarray), ( @@ -346,36 +337,56 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_state.shape[1] == len(self.modality_configs[STATE].delta_indices), ( - f"State key '{state_key}'s horizon must be {len(self.modality_configs[STATE].delta_indices)}. Got {batched_state.shape[1]}" + assert batched_state.shape[1] == len(self.modality_configs["state"].delta_indices), ( + f"State key '{state_key}'s horizon must be {len(self.modality_configs['state'].delta_indices)}. Got {batched_state.shape[1]}" ) - language_key = self.language_key - assert language_key in observation[LANGUAGE], ( - f"Language key '{language_key}' must be in observation" - ) - batched_language = observation[LANGUAGE][language_key] - assert isinstance(batched_language, list), ( - f"Language key '{language_key}' must be a list. Got {type(batched_language)}" - ) - if bs != -1: - assert len(batched_language) == bs, ( - f"Language key '{language_key}' must have batch size {bs}. Got {len(batched_language)}" - ) - for batch_item in batched_language: - assert isinstance(batch_item, list), ( - f"Language batch item must be a list. Got {type(batch_item)}" - ) - assert len(batch_item) == len(self.modality_configs[LANGUAGE].delta_indices), ( - f"Language key '{language_key}'s horizon must be {len(self.modality_configs[LANGUAGE].delta_indices)}. Got {len(batch_item)}" - ) - assert len(batch_item) == 1, ( - f"Language batch item must have exactly one item. Got {len(batch_item)}" + # ===== LANGUAGE VALIDATION ===== + # Inference uses the first instruction key; additional keys are training paraphrases. + for language_key in self.modality_configs[LANGUAGE].modality_keys[:1]: + # Check that the expected language key exists in the observation + # (must happen before indexing — see video validation above) + assert language_key in observation["language"], ( + f"Language key '{language_key}' must be in observation" ) - assert isinstance(batch_item[0], str), ( - f"Language batch item must be a string. Got {type(batch_item[0])}" + + # Set or verify batch size consistency (language uses len instead of .shape) + if bs == -1: + bs = len(observation["language"][language_key]) + else: + assert len(observation["language"][language_key]) == bs, ( + f"Language key '{language_key}' must have batch size {bs}. Got {len(observation['language'][language_key])}" + ) + + batched_language: list[list[str]] = observation["language"][language_key] + + # Verify outer structure is a list (batch dimension) + assert isinstance(batched_language, list), ( + f"Language key '{language_key}' must be a list. Got {type(batched_language)}" ) + # Validate each batch item + for batch_item in batched_language: + # Verify temporal dimension matches expected horizon + assert len(batch_item) == len(self.modality_configs["language"].delta_indices), ( + f"Language key '{language_key}'s horizon must be {len(self.modality_configs['language'].delta_indices)}. Got {len(batched_language)}" + ) + + # Verify inner structure is also a list (temporal dimension) + assert isinstance(batch_item, list), ( + f"Language batch item must be a list. Got {type(batch_item)}" + ) + + # Current implementation expects exactly one language instruction per timestep + assert len(batch_item) == 1, ( + f"Language batch item must have exactly one item. Got {len(batch_item)}" + ) + + # Verify the instruction itself is a string + assert isinstance(batch_item[0], str), ( + f"Language batch item must be a string. Got {type(batch_item[0])}" + ) + def _get_action( self, observation: dict[str, Any], options: dict[str, Any] | None = None ) -> tuple[dict[str, Any], dict[str, Any]]: @@ -418,7 +429,7 @@ def _get_action( # Step 5: Decode actions from normalized space back to physical units batched_states = {} - for k in self.modality_configs[STATE].modality_keys: + for k in self.modality_configs["state"].modality_keys: batched_states[k] = np.stack([s[k] for s in states], axis=0) # (B, T, D) unnormalized_action = self.processor.decode_action( normalized_action.cpu().numpy(), self.embodiment_tag, batched_states @@ -503,9 +514,9 @@ class Gr00tSimPolicyWrapper(PolicyWrapper): This wrapper is only needed for compatibility with the existing Gr00t sim infrastructure. Key transformations performed by this wrapper: - - Observation keys: 'video.cam' -> observation[VIDEO]['cam'] - - Observation keys: 'state.joints' -> observation[STATE]['joints'] - - Language keys: 'task' or 'annotation.human.coarse_action' -> observation[LANGUAGE]['task'] + - Observation keys: 'video.cam' -> observation['video']['cam'] + - Observation keys: 'state.joints' -> observation['state']['joints'] + - Language keys: 'task' or 'annotation.human.coarse_action' -> observation['language']['task'] - Action keys: action['joints'] -> 'action.joints' """ @@ -518,17 +529,10 @@ def __init__(self, policy: Gr00tPolicy, *, strict: bool = True): """ super().__init__(policy, strict=strict) self.policy: Gr00tPolicy = policy - assert len(self.policy.modality_configs[LANGUAGE].delta_indices) == 1, ( + assert len(self.policy.modality_configs["language"].delta_indices) == 1, ( "Only one language delta index is supported" ) - def _language_observation_key(self, observation: dict[str, Any]) -> str: - key = self.policy.language_key - # DC environments expose their instruction under this annotation key. - if key == TASK_LANGUAGE_KEY and COARSE_ACTION_LANGUAGE_KEY in observation: - return COARSE_ACTION_LANGUAGE_KEY - return key - def check_observation(self, observation: dict[str, Any]) -> None: """Validate observation from Gr00t sim environment format. @@ -551,7 +555,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== VIDEO VALIDATION ===== # Check video modalities with flat key format: 'video.camera_name' - for video_key in modality_configs[VIDEO].modality_keys: + for video_key in modality_configs["video"].modality_keys: # Construct flat key expected in Gr00t sim environment parsed_key = f"video.{video_key}" assert parsed_key in observation, f"Video key '{parsed_key}' must be in observation" @@ -574,8 +578,8 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_video.shape[1] == len(modality_configs[VIDEO].delta_indices), ( - f"Video key '{video_key}'s horizon must be {len(modality_configs[VIDEO].delta_indices)}. Got {batched_video.shape[1]}" + assert batched_video.shape[1] == len(modality_configs["video"].delta_indices), ( + f"Video key '{video_key}'s horizon must be {len(modality_configs['video'].delta_indices)}. Got {batched_video.shape[1]}" ) # Verify channel dimension is 3 (RGB images) @@ -585,7 +589,7 @@ def check_observation(self, observation: dict[str, Any]) -> None: # ===== STATE VALIDATION ===== # Check state modalities with flat key format: 'state.state_name' - for state_key in modality_configs[STATE].modality_keys: + for state_key in modality_configs["state"].modality_keys: # Construct flat key expected in Gr00t sim environment parsed_key = f"state.{state_key}" assert parsed_key in observation, f"State key '{parsed_key}' must be in observation" @@ -608,20 +612,38 @@ def check_observation(self, observation: dict[str, Any]) -> None: ) # Verify temporal dimension matches the expected horizon from config - assert batched_state.shape[1] == len(modality_configs[STATE].delta_indices), ( - f"State key '{state_key}'s horizon must be {len(modality_configs[STATE].delta_indices)}. Got {batched_state.shape[1]}" + assert batched_state.shape[1] == len(modality_configs["state"].delta_indices), ( + f"State key '{state_key}'s horizon must be {len(modality_configs['state'].delta_indices)}. Got {batched_state.shape[1]}" ) - language_key = self._language_observation_key(observation) - assert language_key in observation, f"Language key '{language_key}' must be in observation" - batched_language = _sim_language_batch_to_sequence(observation[language_key]) - assert isinstance(batched_language, (tuple, list)), ( - f"Language key '{language_key}' must be a tuple, list, or numpy array. " - f"Got {type(observation[language_key])}" - ) - assert batched_language, f"Language key '{language_key}' must not be empty" - for item in batched_language: - assert isinstance(item, str), f"Language batch item must be a string. Got {type(item)}" + # ===== LANGUAGE VALIDATION ===== + # Validate the instruction selected for inference, including its DC environment alias. + for language_key in modality_configs[LANGUAGE].modality_keys[:1]: + # PATCH: Legacy compatibility for DC environments + # DC envs use 'annotation.human.coarse_action' instead of 'task' + if language_key == "task" and "annotation.human.coarse_action" in observation: + language_key = "annotation.human.coarse_action" + # /PATCH + + # Check that the expected language key exists + assert language_key in observation, ( + f"Language key '{language_key}' must be in observation" + ) + + # In Gr00t sim format, language is a tuple of strings (B,) + batched_language = _sim_language_batch_to_sequence(observation[language_key]) + + # Verify outer structure is a tuple (batch dimension) + assert isinstance(batched_language, (tuple, list)), ( + f"Language key '{language_key}' must be a tuple, list, or numpy array. " + f"Got {type(observation[language_key])}" + ) + assert batched_language, f"Language key '{language_key}' must not be empty" + + # Verify each batch item is a string + assert isinstance(batched_language[0], str), ( + f"Language batch item must be a string. Got {type(batched_language[0])}" + ) def _get_action( self, observation: dict[str, Any], options: dict[str, Any] | None = None @@ -646,17 +668,35 @@ def _get_action( Returns: Tuple of (flat_actions_dict, info_dict) """ - new_obs = { - modality: { - key: observation[f"{modality}.{key}"] - for key in self.policy.modality_configs[modality].modality_keys - } - for modality in (VIDEO, STATE) - } - language = _sim_language_batch_to_sequence( - observation[self._language_observation_key(observation)] - ) - new_obs[LANGUAGE] = {self.policy.language_key: [[str(item)] for item in language]} + # Transform flat observation format to nested format expected by Gr00tPolicy + new_obs = {} + for modality in ["video", "state", "language"]: + new_obs[modality] = {} + for key in self.policy.modality_configs[modality].modality_keys: + if modality == LANGUAGE and key != self.policy.language_key: + continue + if modality == "language": + # PATCH: Legacy compatibility for DC environments + if key == "task" and "annotation.human.coarse_action" in observation: + parsed_key = "annotation.human.coarse_action" + # /PATCH + else: + parsed_key = key + else: + # Construct flat key (e.g., 'video.camera' or 'state.joints') + parsed_key = f"{modality}.{key}" + + arr = observation[parsed_key] + + # Transform to nested format + if modality == "language": + arr = _sim_language_batch_to_sequence(arr) + # Convert from tuple[str] or list[str] (B,) to list[list[str]] (B, 1) + # Each element becomes a list with one string for temporal dimension + new_obs[modality][key] = [[str(item)] for item in arr] + else: + # Video and state arrays are already in correct format (B, T, ...) + new_obs[modality][key] = arr # Compute actions using the underlying Gr00tPolicy action, info = self.policy.get_action(new_obs, options) From be79d6244dda302ace1ff7a2cd55239aad2ad109 Mon Sep 17 00:00:00 2001 From: Sergey Arkhangelskiy Date: Fri, 11 Sep 2026 14:03:48 +0300 Subject: [PATCH 7/7] Preserve amd64 Docker builds from Mac --- docker/Makefile | 2 +- docker/README.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 22ae9a0db..a5035665e 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -39,7 +39,7 @@ build: echo "Error: Could not extract version from pyproject.toml"; \ exit 1; \ fi - docker build \ + docker build --platform linux/amd64 \ -f Dockerfile \ -t $(LOCAL_TAG_GROOT) .. diff --git a/docker/README.md b/docker/README.md index a4e117105..c8cf40d51 100644 --- a/docker/README.md +++ b/docker/README.md @@ -28,7 +28,8 @@ make -C docker push ``` The Makefile publishes `positro/gr00t-base` with `latest`, version, and commit tags. -It builds for the host architecture; use `DOCKER_DEFAULT_PLATFORM=linux/amd64` to cross-build. +It targets `linux/amd64`, including when built on an Apple Silicon Mac. +For a native ARM build, use `bash docker/build.sh` on the target host. Positronic's `GROOT_BASE_IMAGE` selects an existing base image for its adapter build. Fine-tuning defaults to the base checkpoint's saved model and modality configuration.