From 8154a87fbd9e12092b6af14b36d929978537ef20 Mon Sep 17 00:00:00 2001 From: mehhl Date: Sun, 22 Dec 2024 21:30:06 +0000 Subject: [PATCH 01/58] add lerobot dataset support --- .gitmodules | 6 + aloha_sim_insertion_scripted_image | 1 + lerobot | 1 + prismatic/vla/datasets/datasets.py | 132 ++++++++++- vla-scripts/finetune.py | 341 ++++++++++++++++------------- 5 files changed, 326 insertions(+), 155 deletions(-) create mode 100644 .gitmodules create mode 160000 aloha_sim_insertion_scripted_image create mode 160000 lerobot diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..2d9697f66 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "aloha_sim_insertion_scripted_image"] + path = aloha_sim_insertion_scripted_image + url = https://huggingface.co/datasets/lerobot/aloha_sim_insertion_scripted_image +[submodule "lerobot"] + path = lerobot + url = https://github.com/huggingface/lerobot diff --git a/aloha_sim_insertion_scripted_image b/aloha_sim_insertion_scripted_image new file mode 160000 index 000000000..f32ed18e5 --- /dev/null +++ b/aloha_sim_insertion_scripted_image @@ -0,0 +1 @@ +Subproject commit f32ed18e53003d5fb3d9727f35a1be5dcf27c431 diff --git a/lerobot b/lerobot new file mode 160000 index 000000000..44f9b21e7 --- /dev/null +++ b/lerobot @@ -0,0 +1 @@ +Subproject commit 44f9b21e74936c366b55609d1847b843bd04f3ab diff --git a/prismatic/vla/datasets/datasets.py b/prismatic/vla/datasets/datasets.py index 539b4144d..818ba6677 100644 --- a/prismatic/vla/datasets/datasets.py +++ b/prismatic/vla/datasets/datasets.py @@ -176,6 +176,136 @@ def __iter__(self) -> Dict[str, Any]: ] yield out +### +from typing import Callable + +from lerobot.common.datasets.lerobot_dataset import ( + LeRobotDataset, + LeRobotDatasetMetadata, +) + +# TODO: zrob dataset ktory by jednoczesnie byl LeRobotDatasetem i implementoeal to czego tam potrzebuje openvla -- te prompt buildery, itp. + +class OpenVLALeRobotDataset(LeRobotDataset): + + def __init__( + self, + repo_id: str, + action_tokenizer: ActionTokenizer, + base_tokenizer: PreTrainedTokenizerBase, + image_transform: ImageTransform, + prompt_builder_fn: Type[PromptBuilder], + *, + root: str | Path | None = None, + episodes: list[int] | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[list[float]] | None = None, + tolerance_s: float = 1e-4, + download_videos: bool = True, + local_files_only: bool = False, + video_backend: str | None = None, + ) -> None: + super().__init__( + repo_id, + root, + episodes, + image_transforms, + delta_timestamps, + tolerance_s, + download_videos, + local_files_only, + video_backend, + ) + assert isinstance(self.meta, LeRobotDatasetMetadata) + + self.action_tokenizer = action_tokenizer + self.base_tokenizer = base_tokenizer + self.image_transform = image_transform + self.prompt_builder_fn = prompt_builder_fn + + # Note =>> We expect the dataset to store statistics for action de-normalization. + self.dataset_statistics = { + "openvla_lerobot_dataset": { + "action": { + "q01": self.meta.stats["action"]["q01"].tolist(), + "q99": self.meta.stats["action"]["q99"].tolist(), + } + } + } + + # Retrieve the name of image observations within metadata. + metadata_feature_dict = self.meta.info['features'] + obs_image_keys = [ + k for k in metadata_feature_dict.keys() + if isinstance(metadata_feature_dict[k], dict) + and metadata_feature_dict[k].get("dtype") == "video" + ] + if len(obs_image_keys) == 0: + raise ValueError(f"Provided data contains no videos") + if len(obs_image_keys) > 1: + raise ValueError(f"Provided data contains >1 video per episode") + self.obs_image_key = obs_image_keys[0] + + + def __len__(self): + return self.meta.info['total_episodes'] + + # Retrieves a single (instruction, image, action) triple from the dataset. + def __getitem__(self, idx): + + hf_item = super().__getitem__(idx) + + # Retrieve image observation. + img_array: np.ndarray = ( + hf_item[self.obs_image_key] + .permute(1, 2, 0) + .numpy() * 255 + ).astype(np.uint8) + image = Image.fromarray(img_array) + + # Retrieve instruction. + task_idx: torch.Tensor = hf_item["task_index"] + task_idx: int = task_idx.item() + instruction = self.meta.tasks[task_idx] + + # Retrieve action. + action: torch.Tensor = hf_item["action"] + action: str = self.action_tokenizer(action) + + # Add instruction to VLA prompt. + prompt_builder = self.prompt_builder_fn("openvla") + conversation = [ + { + "from": "human", + "value": f"What action should the robot take to {instruction}?" + }, + { + "from": "gpt", + "value": f"{action}" + }, + ] + for turn in conversation: + prompt_builder.add_turn(turn["from"], turn["value"]) + prompt = prompt_builder.get_prompt() + + # Tokenize (w/ `base_tokenizer`) + input_ids = self.base_tokenizer( + prompt, + add_special_tokens=True + ).input_ids + labels = list(input_ids) + + # Tensorize =>> Run Image Transform to get `pixel_values` =>> Return + # =>> IMPORTANT :: IF WE'RE USING HF .forward(..., labels=labels), SHIFTING HAPPENS _INSIDE_ MODEL! + input_ids, labels = torch.tensor(input_ids), torch.tensor(labels) + pixel_values = self.image_transform(image) + + # [CRITICAL] We do not want to take the loss for anything but the predicted action tokens! + labels[: -(len(action) + 1)] = IGNORE_INDEX + + return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels) + + class DummyDataset(Dataset): def __init__( @@ -229,4 +359,4 @@ def __getitem__(self, idx): # [CRITICAL] We do not want to take the loss for anything but the predicted action tokens! labels[: -(len(action) + 1)] = IGNORE_INDEX - return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels) + return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels) \ No newline at end of file diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index ec51a6b3c..f8a4a80e1 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -33,7 +33,7 @@ from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import AdamW -from torch.utils.data import DataLoader +from torch.utils.data import DataLoader, RandomSampler from transformers import AutoModelForVision2Seq, AutoProcessor, BitsAndBytesConfig from transformers import AutoConfig, AutoImageProcessor from transformers.modeling_outputs import CausalLMOutputWithPast @@ -75,37 +75,41 @@ @dataclass class FinetuneConfig: # fmt: off - vla_path: str = "openvla/openvla-7b" # Path to OpenVLA model (on HuggingFace Hub) + vla_path: str = "openvla/openvla-7b" # Path to OpenVLA model (on HuggingFace Hub) # Directory Paths - data_root_dir: Path = Path("datasets/open-x-embodiment") # Path to Open-X dataset directory - dataset_name: str = "droid_wipe" # Name of fine-tuning dataset (e.g., `droid_wipe`) - run_root_dir: Path = Path("runs") # Path to directory to store logs & checkpoints - adapter_tmp_dir: Path = Path("adapter-tmp") # Temporary directory for LoRA weights before fusing + data_root_dir: Path = Path("datasets/open-x-embodiment") # Path to Open-X dataset directory + dataset_name: str = "droid_wipe" # Name of fine-tuning dataset (e.g., `droid_wipe`) + run_root_dir: Path = Path("runs") # Path to directory to store logs & checkpoints + adapter_tmp_dir: Path = Path("adapter-tmp") # Temporary directory for LoRA weights before fusing # Fine-tuning Parameters - batch_size: int = 16 # Fine-tuning batch size - max_steps: int = 200_000 # Max number of fine-tuning steps - save_steps: int = 5000 # Interval for checkpoint saving - learning_rate: float = 5e-4 # Fine-tuning learning rate - grad_accumulation_steps: int = 1 # Gradient accumulation steps - image_aug: bool = True # Whether to train with image augmentations - shuffle_buffer_size: int = 100_000 # Dataloader shuffle buffer size (can reduce if OOM) - save_latest_checkpoint_only: bool = True # Whether to save only one checkpoint per run and - # continually overwrite the latest checkpoint - # (If False, saves all checkpoints) + batch_size: int = 16 # Fine-tuning batch size + max_steps: int = 200_000 # Max number of fine-tuning steps + save_steps: int = 5000 # Interval for checkpoint saving + learning_rate: float = 5e-4 # Fine-tuning learning rate + grad_accumulation_steps: int = 1 # Number of batches to accumulate gradients over before performing + # an optimization step. Effectively multiplies the batch_size by this + # value while using less memory. Example: if batch_size=16 and + # grad_accumulation_steps=4, this simulates training with + # batch_size=64 but only requires memory for 16 samples at a time. + image_aug: bool = True # Whether to train with image augmentations + shuffle_buffer_size: int = 100_000 # Dataloader shuffle buffer size (can reduce if OOM) + save_latest_checkpoint_only: bool = True # Whether to save only one checkpoint per run and + # continually overwrite the latest checkpoint + # (If False, saves all checkpoints) # LoRA Arguments - use_lora: bool = True # Whether to use LoRA fine-tuning - lora_rank: int = 32 # Rank of LoRA weight matrix - lora_dropout: float = 0.0 # Dropout applied to LoRA weights - use_quantization: bool = False # Whether to 4-bit quantize VLA for LoRA fine-tuning - # => CAUTION: Reduces memory but hurts performance + use_lora: bool = True # Whether to use LoRA fine-tuning + lora_rank: int = 32 # Rank of LoRA weight matrix + lora_dropout: float = 0.0 # Dropout applied to LoRA weights + use_quantization: bool = False # Whether to 4-bit quantize VLA for LoRA fine-tuning + # => CAUTION: Reduces memory but hurts performance # Tracking Parameters - wandb_project: str = "openvla" # Name of W&B project to log to (use default!) - wandb_entity: str = "stanford-voltron" # Name of entity to log under - run_id_note: Optional[str] = None # Extra note for logging, Weights & Biases + wandb_project: str = "openvla" # Name of W&B project to log to (use default!) + wandb_entity: str = "stanford-voltron" # Name of entity to log under + run_id_note: Optional[str] = None # Extra note for logging, Weights & Biases # fmt: on @@ -190,36 +194,49 @@ def finetune(cfg: FinetuneConfig) -> None: # Create Action Tokenizer action_tokenizer = ActionTokenizer(processor.tokenizer) + # TODO: FIgure out what ActionTokenizer and processor.tokenizer do + # TODO: and how to duplicate this functionality in LeRobotDataset # Load Fine-tuning Dataset =>> note that we use an RLDS-formatted dataset following Open X-Embodiment by default. # =>> If you want to use a non-RLDS dataset (e.g., a standard PyTorch Dataset) see the following commented block. # =>> Note that our training code does not loop over epochs because the RLDS loader does this implicitly; if using # your own Dataset, make sure to add the appropriate logic to the training loop! - # + # # TODO: Figure this out # --- - # from prismatic.vla.datasets import DummyDataset - # - # vla_dataset = DummyDataset( + from prismatic.vla.datasets.datasets import OpenVLALeRobotDataset + + vla_dataset = OpenVLALeRobotDataset( + repo_id="NotRequired", + action_tokenizer=action_tokenizer, + base_tokenizer=processor.tokenizer, + image_transform=processor.image_processor.apply_transform, + prompt_builder_fn=( + PurePromptBuilder + if "v01" not in cfg.vla_path + else VicunaV15ChatPromptBuilder + ), + root=f"{cfg.data_root_dir}/{cfg.dataset_name}", + tolerance_s=3.0, + image_transforms=None, + download_videos=False, + local_files_only=True, + ) + + + # batch_transform = RLDSBatchTransform( # action_tokenizer, # processor.tokenizer, # image_transform=processor.image_processor.apply_transform, # prompt_builder_fn=PurePromptBuilder if "v01" not in cfg.vla_path else VicunaV15ChatPromptBuilder, # ) - # --- - batch_transform = RLDSBatchTransform( - action_tokenizer, - processor.tokenizer, - image_transform=processor.image_processor.apply_transform, - prompt_builder_fn=PurePromptBuilder if "v01" not in cfg.vla_path else VicunaV15ChatPromptBuilder, - ) - vla_dataset = RLDSDataset( - cfg.data_root_dir, - cfg.dataset_name, - batch_transform, - resize_resolution=tuple(vla.module.config.image_sizes), - shuffle_buffer_size=cfg.shuffle_buffer_size, - image_aug=cfg.image_aug, - ) + # vla_dataset = RLDSDataset( + # cfg.data_root_dir, + # cfg.dataset_name, + # batch_transform, + # resize_resolution=tuple(vla.module.config.image_sizes), + # shuffle_buffer_size=cfg.shuffle_buffer_size, + # image_aug=cfg.image_aug, + # ) # [Important] Save Dataset Statistics =>> used to de-normalize actions for inference! if distributed_state.is_main_process: @@ -232,9 +249,10 @@ def finetune(cfg: FinetuneConfig) -> None: dataloader = DataLoader( vla_dataset, batch_size=cfg.batch_size, - sampler=None, + sampler=RandomSampler(vla_dataset), collate_fn=collator, - num_workers=0, # Important =>> Set to 0 if using RLDS; TFDS rolls its own parallelism! + num_workers=0, # Set to 0 bc we don't use parallelism + # TODO: figure out if this is right? ) # Initialize Logging =>> W&B @@ -246,126 +264,141 @@ def finetune(cfg: FinetuneConfig) -> None: recent_action_accuracies = deque(maxlen=cfg.grad_accumulation_steps) recent_l1_losses = deque(maxlen=cfg.grad_accumulation_steps) + # Calculate number of epochs + steps_per_epoch = len(dataloader) # number of batches per epoch + min_epochs = (cfg.max_steps * cfg.grad_accumulation_steps) // steps_per_epoch + 1 + num_epochs = min_epochs + # Train! with tqdm.tqdm(total=cfg.max_steps, leave=False) as progress: vla.train() optimizer.zero_grad() - for batch_idx, batch in enumerate(dataloader): - with torch.autocast("cuda", dtype=torch.bfloat16): - output: CausalLMOutputWithPast = vla( - input_ids=batch["input_ids"].to(device_id), - attention_mask=batch["attention_mask"].to(device_id), - pixel_values=batch["pixel_values"].to(torch.bfloat16).to(device_id), - labels=batch["labels"], + # TODO: Debug + + for epoch in tqdm.tqdm(range(num_epochs), desc="Epoch"): + for batch_idx, batch in enumerate(dataloader): + with torch.autocast("cuda", dtype=torch.bfloat16): + output: CausalLMOutputWithPast = vla( + input_ids=batch["input_ids"].to(device_id), + attention_mask=batch["attention_mask"].to(device_id), + pixel_values=batch["pixel_values"].to(torch.bfloat16).to(device_id), + labels=batch["labels"], + ) + loss = output.loss + + # Normalize loss to account for gradient accumulation + normalized_loss = loss / cfg.grad_accumulation_steps + + # Backward pass + normalized_loss.backward() + + # Compute Accuracy and L1 Loss for Logging + # Will vla have a ".module"? + assert isinstance(vla, DDP) + # Is featurizer a vision transformer? + assert hasattr(vla.module.vision_backbone.featurizer, "patch_embed") + action_logits = output.logits[:, vla.module.vision_backbone.featurizer.patch_embed.num_patches: -1] + action_preds = action_logits.argmax(dim=2) + action_gt = batch["labels"][:, 1:].to(action_preds.device) + mask = action_gt > action_tokenizer.action_token_begin_idx + + # Compute Accuracy + correct_preds = (action_preds == action_gt) & mask + action_accuracy = correct_preds.sum().float() / mask.sum().float() + + # Compute L1 Loss on Predicted (Continuous) Actions + continuous_actions_pred = torch.tensor( + action_tokenizer.decode_token_ids_to_actions(action_preds[mask].cpu().numpy()) ) - loss = output.loss - - # Normalize loss to account for gradient accumulation - normalized_loss = loss / cfg.grad_accumulation_steps - - # Backward pass - normalized_loss.backward() - - # Compute Accuracy and L1 Loss for Logging - action_logits = output.logits[:, vla.module.vision_backbone.featurizer.patch_embed.num_patches : -1] - action_preds = action_logits.argmax(dim=2) - action_gt = batch["labels"][:, 1:].to(action_preds.device) - mask = action_gt > action_tokenizer.action_token_begin_idx - - # Compute Accuracy - correct_preds = (action_preds == action_gt) & mask - action_accuracy = correct_preds.sum().float() / mask.sum().float() - - # Compute L1 Loss on Predicted (Continuous) Actions - continuous_actions_pred = torch.tensor( - action_tokenizer.decode_token_ids_to_actions(action_preds[mask].cpu().numpy()) - ) - continuous_actions_gt = torch.tensor( - action_tokenizer.decode_token_ids_to_actions(action_gt[mask].cpu().numpy()) - ) - action_l1_loss = torch.nn.functional.l1_loss(continuous_actions_pred, continuous_actions_gt) - - # Store recent train metrics - recent_losses.append(loss.item()) - recent_action_accuracies.append(action_accuracy.item()) - recent_l1_losses.append(action_l1_loss.item()) - - # Compute gradient step index - gradient_step_idx = batch_idx // cfg.grad_accumulation_steps - - # Compute smoothened train metrics - # =>> Equal to current step metrics when not using gradient accumulation - # =>> Otherwise, equal to the average of metrics observed over micro-batches used for gradient accumulation - smoothened_loss = sum(recent_losses) / len(recent_losses) - smoothened_action_accuracy = sum(recent_action_accuracies) / len(recent_action_accuracies) - smoothened_l1_loss = sum(recent_l1_losses) / len(recent_l1_losses) - - # Push Metrics to W&B (every 10 gradient steps) - if distributed_state.is_main_process and gradient_step_idx % 10 == 0: - wandb.log( - { - "train_loss": smoothened_loss, - "action_accuracy": smoothened_action_accuracy, - "l1_loss": smoothened_l1_loss, - }, - step=gradient_step_idx, + continuous_actions_gt = torch.tensor( + action_tokenizer.decode_token_ids_to_actions(action_gt[mask].cpu().numpy()) ) - - # Optimizer Step - if (batch_idx + 1) % cfg.grad_accumulation_steps == 0: - optimizer.step() - optimizer.zero_grad() - progress.update() - - # Save Model Checkpoint =>> by default, only keeps the latest checkpoint, continually overwriting it! - if gradient_step_idx > 0 and gradient_step_idx % cfg.save_steps == 0: - if distributed_state.is_main_process: - print(f"Saving Model Checkpoint for Step {gradient_step_idx}") - - # If LoRA, we first save adapter weights, then merge into full model; otherwise, default save! - save_dir = adapter_dir if cfg.use_lora else run_dir - - # Save Processor & Weights - processor.save_pretrained(run_dir) - vla.module.save_pretrained(save_dir) - - # Wait for processor and adapter weights to be saved by main process - dist.barrier() - - # Merge LoRA weights into model backbone for faster inference - # =>> Note that merging is slow and can be done post-hoc to speed up training - if cfg.use_lora: - base_vla = AutoModelForVision2Seq.from_pretrained( - cfg.vla_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True + action_l1_loss = torch.nn.functional.l1_loss(continuous_actions_pred, continuous_actions_gt) + + # Store recent train metrics + recent_losses.append(loss.item()) + recent_action_accuracies.append(action_accuracy.item()) + recent_l1_losses.append(action_l1_loss.item()) + + # Compute gradient step index + gradient_step_idx = batch_idx // cfg.grad_accumulation_steps + + # Compute smoothened train metrics + # =>> Equal to current step metrics when not using gradient accumulation + # =>> Otherwise, equal to the average of metrics observed over micro-batches used for gradient accumulation + smoothened_loss = sum(recent_losses) / len(recent_losses) + smoothened_action_accuracy = sum(recent_action_accuracies) / len(recent_action_accuracies) + smoothened_l1_loss = sum(recent_l1_losses) / len(recent_l1_losses) + + # Push Metrics to W&B (every 10 gradient steps) + if distributed_state.is_main_process and gradient_step_idx % 10 == 0: + print( + { + "train_loss": smoothened_loss, + "action_accuracy": smoothened_action_accuracy, + "l1_loss": smoothened_l1_loss, + }, ) - merged_vla = PeftModel.from_pretrained(base_vla, adapter_dir) - merged_vla = merged_vla.merge_and_unload() - if distributed_state.is_main_process: - if cfg.save_latest_checkpoint_only: - # Overwrite latest checkpoint - merged_vla.save_pretrained(run_dir) - - print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {run_dir}") - else: - # Prepare to save checkpoint in new directory - checkpoint_dir = Path(str(run_dir) + f"--{gradient_step_idx}_chkpt") - os.makedirs(checkpoint_dir, exist_ok=True) - - # Save dataset statistics to new directory - save_dataset_statistics(vla_dataset.dataset_statistics, checkpoint_dir) - # Save processor and model weights to new directory - processor.save_pretrained(checkpoint_dir) - merged_vla.save_pretrained(checkpoint_dir) + # Optimizer Step + if (batch_idx + 1) % cfg.grad_accumulation_steps == 0: + optimizer.step() + optimizer.zero_grad() + progress.update() - print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {checkpoint_dir}") - - # Block on Main Process Checkpointing - dist.barrier() + # Save Model Checkpoint =>> by default, only keeps the latest checkpoint, continually overwriting it! + if gradient_step_idx > 0 and gradient_step_idx % cfg.save_steps == 0: + if distributed_state.is_main_process: + print(f"Saving Model Checkpoint for Step {gradient_step_idx}") + + # If LoRA, we first save adapter weights, then merge into full model; otherwise, default save! + save_dir = adapter_dir if cfg.use_lora else run_dir + + # Save Processor & Weights + processor.save_pretrained(run_dir) + vla.module.save_pretrained(save_dir) + + # Wait for processor and adapter weights to be saved by main process + dist.barrier() + + # Merge LoRA weights into model backbone for faster inference + # =>> Note that merging is slow and can be done post-hoc to speed up training + if cfg.use_lora: + base_vla = AutoModelForVision2Seq.from_pretrained( + cfg.vla_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True + ) + merged_vla = PeftModel.from_pretrained(base_vla, adapter_dir) + merged_vla = merged_vla.merge_and_unload() + if distributed_state.is_main_process: + if cfg.save_latest_checkpoint_only: + # Overwrite latest checkpoint + merged_vla.save_pretrained(run_dir) + + print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {run_dir}") + else: + # Prepare to save checkpoint in new directory + checkpoint_dir = Path(str(run_dir) + f"--{gradient_step_idx}_chkpt") + os.makedirs(checkpoint_dir, exist_ok=True) + + # Save dataset statistics to new directory + save_dataset_statistics(vla_dataset.dataset_statistics, checkpoint_dir) + + # Save processor and model weights to new directory + processor.save_pretrained(checkpoint_dir) + merged_vla.save_pretrained(checkpoint_dir) + + print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {checkpoint_dir}") + + # Block on Main Process Checkpointing + dist.barrier() + + # Stop training when max_steps is reached + if gradient_step_idx == cfg.max_steps: + print(f"Max step {cfg.max_steps} reached! Stopping training...") + break - # Stop training when max_steps is reached if gradient_step_idx == cfg.max_steps: - print(f"Max step {cfg.max_steps} reached! Stopping training...") + print("Yeah exiting") break From f11844a86420ebd98807cacfdb240fe3ebd288ce Mon Sep 17 00:00:00 2001 From: mehhl Date: Sun, 22 Dec 2024 22:40:35 +0000 Subject: [PATCH 02/58] fix indexing issue w/ gradient acc --- vla-scripts/finetune.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index f8a4a80e1..eff39a5e4 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -275,7 +275,13 @@ def finetune(cfg: FinetuneConfig) -> None: optimizer.zero_grad() # TODO: Debug - for epoch in tqdm.tqdm(range(num_epochs), desc="Epoch"): + # Compute the number of epochs needed to train for cfg.max_steps steps + # =>> This is used to set the number of epochs in the progress bar + steps_per_epoch = len(dataloader) # number of batches per epoch + min_epochs = (cfg.max_steps * cfg.grad_accumulation_steps) // steps_per_epoch + 1 + num_epochs = min_epochs + + for _ in range(num_epochs): for batch_idx, batch in enumerate(dataloader): with torch.autocast("cuda", dtype=torch.bfloat16): output: CausalLMOutputWithPast = vla( @@ -341,7 +347,10 @@ def finetune(cfg: FinetuneConfig) -> None: ) # Optimizer Step - if (batch_idx + 1) % cfg.grad_accumulation_steps == 0: + if ( + (batch_idx + 1) % cfg.grad_accumulation_steps == 0 + or batch_idx == len(dataloader) - 1 + ): optimizer.step() optimizer.zero_grad() progress.update() @@ -398,7 +407,6 @@ def finetune(cfg: FinetuneConfig) -> None: break if gradient_step_idx == cfg.max_steps: - print("Yeah exiting") break From 0a15af2388adb8520c8d0e4595c4c7bebea6e4ff Mon Sep 17 00:00:00 2001 From: mehhl Date: Sun, 22 Dec 2024 22:40:55 +0000 Subject: [PATCH 03/58] prettify OpenVLALeRobotDataset --- prismatic/vla/datasets/datasets.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/prismatic/vla/datasets/datasets.py b/prismatic/vla/datasets/datasets.py index 818ba6677..9ddd403da 100644 --- a/prismatic/vla/datasets/datasets.py +++ b/prismatic/vla/datasets/datasets.py @@ -333,17 +333,28 @@ def __len__(self): return 10000 def __getitem__(self, idx): - # TODO =>> Load image, action and instruction from disk -- we use dummy values - image = Image.fromarray(np.asarray(np.random.rand(224, 224, 3) * 255.0, dtype=np.uint8)) + """Get a single training example.""" + # Generate random image, action and instruction + image = Image.fromarray( + np.asarray(np.random.rand(224, 224, 3) * 255.0, dtype=np.uint8) + ) action = np.asarray(np.random.rand(7), dtype=np.float32) instruction = "do something spectacular" - # Add instruction to VLA prompt + # Build conversation prompt prompt_builder = self.prompt_builder_fn("openvla") conversation = [ - {"from": "human", "value": f"What action should the robot take to {instruction}?"}, - {"from": "gpt", "value": self.action_tokenizer(action)}, + { + "from": "human", + "value": f"What action should the robot take to {instruction}?" + }, + { + "from": "gpt", + "value": self.action_tokenizer(action) + } ] + + # Add conversation turns to prompt builder for turn in conversation: prompt_builder.add_turn(turn["from"], turn["value"]) From 234b4cfd268a9459f9b753914197ff50b5853fc6 Mon Sep 17 00:00:00 2001 From: Maciej Mehl <106269097+mehhl@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:26:43 +0100 Subject: [PATCH 04/58] Update README.md --- README.md | 640 +----------------------------------------------------- 1 file changed, 8 insertions(+), 632 deletions(-) diff --git a/README.md b/README.md index 2d6885e5e..e66825485 100644 --- a/README.md +++ b/README.md @@ -1,635 +1,11 @@ -# OpenVLA: An Open-Source Vision-Language-Action Model - -[![arXiv](https://img.shields.io/badge/arXiv-2406.09246-df2a2a.svg?style=for-the-badge)](https://arxiv.org/abs/2406.09246) -[![HF Models](https://img.shields.io/badge/%F0%9F%A4%97-Models-yellow?style=for-the-badge)](https://huggingface.co/openvla/openvla-7b) -[![PyTorch](https://img.shields.io/badge/PyTorch-2.2.0-EE4C2C.svg?style=for-the-badge&logo=pytorch)](https://pytorch.org/get-started/locally/) -[![Python](https://img.shields.io/badge/python-3.10-blue?style=for-the-badge)](https://www.python.org) -[![License](https://img.shields.io/github/license/TRI-ML/prismatic-vlms?style=for-the-badge)](LICENSE) - -[**Getting Started**](#getting-started) | [**Pretrained VLAs**](#pretrained-vlas) | [**Installation**](#installation) | [**Fine-Tuning OpenVLA via LoRA**](#fine-tuning-openvla-via-lora) | [**Fully Fine-Tuning OpenVLA**](#fully-fine-tuning-openvla) | -[**Training VLAs from Scratch**](#training-vlas-from-scratch) | [**Evaluating OpenVLA**](#evaluating-openvla) | [**Project Website**](https://openvla.github.io/) - - -
- -## Latest Updates -- [2024-10-15] Added a [VLA Performance Troubleshooting](#vla-performance-troubleshooting) section to the README with best practices for debugging poor VLA performance after fine-tuning. -- [2024-09-04] Added LIBERO simulation benchmark fine-tuning experiments to paper (see v2 on [arXiv](https://arxiv.org/abs/2406.09246)); - added instructions for reproducing OpenVLA results in [LIBERO Simulation Benchmark Evaluations](#libero-simulation-benchmark-evaluations) section -- [2024-08-14] Added new section, [Evaluating OpenVLA](#evaluating-openvla), with instructions for running BridgeData V2 WidowX robot evals -- [2024-07-08] Added new sections: [Fine-Tuning OpenVLA via LoRA](#fine-tuning-openvla-via-lora), [Fully Fine-Tuning OpenVLA](#fully-fine-tuning-openvla) -- [2024-06-13] Initial release - -
- -A simple and scalable codebase for training and fine-tuning vision-language-action models (VLAs) for generalist robotic -manipulation: - -- **Different Dataset Mixtures**: We natively support arbitrary datasets in RLDS format, including arbitrary mixtures of - data from the [Open X-Embodiment Dataset](https://robotics-transformer-x.github.io/). -- **Easy Scaling**: Powered by PyTorch FSDP and Flash-Attention, we can quickly and efficiently train models from 1B - - 34B parameters, with easily adaptable model architectures. -- **Native Fine-Tuning Support**: Built-in support (with examples) for various forms of fine-tuning (full, - partial, LoRA). - -Built on top of [Prismatic VLMs](https://github.com/TRI-ML/prismatic-vlms). - -## Getting Started - -To get started with loading and running OpenVLA models for inference, we provide a lightweight interface that leverages -HuggingFace `transformers` AutoClasses, with minimal dependencies. - -For example, to load `openvla-7b` for zero-shot instruction following in the -[BridgeData V2 environments](https://rail-berkeley.github.io/bridgedata/) with a WidowX robot: - -```python -# Install minimal dependencies (`torch`, `transformers`, `timm`, `tokenizers`, ...) -# > pip install -r https://raw.githubusercontent.com/openvla/openvla/main/requirements-min.txt -from transformers import AutoModelForVision2Seq, AutoProcessor -from PIL import Image - -import torch - -# Load Processor & VLA -processor = AutoProcessor.from_pretrained("openvla/openvla-7b", trust_remote_code=True) -vla = AutoModelForVision2Seq.from_pretrained( - "openvla/openvla-7b", - attn_implementation="flash_attention_2", # [Optional] Requires `flash_attn` - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - trust_remote_code=True -).to("cuda:0") - -# Grab image input & format prompt -image: Image.Image = get_from_camera(...) -prompt = "In: What action should the robot take to {}?\nOut:" - -# Predict Action (7-DoF; un-normalize for BridgeData V2) -inputs = processor(prompt, image).to("cuda:0", dtype=torch.bfloat16) -action = vla.predict_action(**inputs, unnorm_key="bridge_orig", do_sample=False) - -# Execute... -robot.act(action, ...) -``` - -We also provide an [example script for fine-tuning OpenVLA models for new tasks and -embodiments](./vla-scripts/finetune.py); this script supports different fine-tuning modes -- including (quantized) -low-rank adaptation (LoRA) supported by [HuggingFace's PEFT library](https://huggingface.co/docs/peft/en/index). - -For deployment, we provide a lightweight script for [serving OpenVLA models over a REST API](./vla-scripts/deploy.py), -providing an easy way to integrate OpenVLA models into existing robot control stacks, -removing any requirement for powerful on-device compute. - -## Pretrained VLAs - -We release two OpenVLA models trained as part of our work, with checkpoints, configs, and model cards available [on our -HuggingFace page](https://huggingface.co/openvla): -- [`openvla-7b`](https://huggingface.co/openvla/openvla-7b): The flagship model from our paper, trained from - the Prismatic `prism-dinosiglip-224px` VLM (based on a fused DINOv2 and SigLIP vision backbone, and Llama-2 LLM). - Trained on a large mixture of datasets from Open X-Embodiment spanning 970K trajectories - ([mixture details - see "Open-X Magic Soup++"](./prismatic/vla/datasets/rlds/oxe/mixtures.py)). -- [`openvla-v01-7b`](https://huggingface.co/openvla/openvla-7b-v01): An early model used during development, trained from - the Prismatic `siglip-224px` VLM (singular SigLIP vision backbone, and a Vicuña v1.5 LLM). Trained on the same mixture - of datasets as [Octo](https://github.com/octo-models/octo), but for significantly fewer GPU hours than our final model - ([mixture details - see "Open-X Magic Soup"](./prismatic/vla/datasets/rlds/oxe/mixtures.py)). - -**Explicit Notes on Model Licensing & Commercial Use**: While all code in this repository is released under an MIT -License, our pretrained models may inherit restrictions from the underlying base models we use. Specifically, both the -above models are derived from Llama-2, and as such are subject to the -[Llama Community License](https://ai.meta.com/llama/license/). - ---- - -## Installation - -> **Note**: These installation instructions are for full-scale pretraining (and distributed fine-tuning); if looking to - just run inference with OpenVLA models (or perform lightweight fine-tuning), see instructions above! - -This repository was built using Python 3.10, but should be backwards compatible with any Python >= 3.8. We require -PyTorch 2.2.* -- installation instructions [can be found here](https://pytorch.org/get-started/locally/). The latest -version of this repository was developed and thoroughly tested with: - - PyTorch 2.2.0, torchvision 0.17.0, transformers 4.40.1, tokenizers 0.19.1, timm 0.9.10, and flash-attn 2.5.5 - -**[5/21/24] Note**: Following reported regressions and breaking changes in later versions of `transformers`, `timm`, and -`tokenizers` we explicitly pin the above versions of the dependencies. We are working on implementing thorough tests, -and plan on relaxing these constraints as soon as we can. - -Use the setup commands below to get started: - -```bash -# Create and activate conda environment -conda create -n openvla python=3.10 -y -conda activate openvla - -# Install PyTorch. Below is a sample command to do this, but you should check the following link -# to find installation instructions that are specific to your compute platform: -# https://pytorch.org/get-started/locally/ -conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch -c nvidia -y # UPDATE ME! - -# Clone and install the openvla repo -git clone https://github.com/openvla/openvla.git -cd openvla -pip install -e . - -# Install Flash Attention 2 for training (https://github.com/Dao-AILab/flash-attention) -# =>> If you run into difficulty, try `pip cache remove flash_attn` first -pip install packaging ninja -ninja --version; echo $? # Verify Ninja --> should return exit code "0" -pip install "flash-attn==2.5.5" --no-build-isolation -``` - -If you run into any problems during the installation process, please file a GitHub Issue. - -**Note:** See `vla-scripts/` for full training and verification scripts for OpenVLA models. Note that `scripts/` is -mostly a holdover from the original (base) `prismatic-vlms` repository, with support for training and evaluating -visually-conditioned language models; while you can use this repo to train VLMs AND VLAs, note that trying to generate -language (via `scripts/generate.py`) with existing OpenVLA models will not work (as we only train current OpenVLA models -to generate actions, and actions alone). - -## Fine-Tuning OpenVLA via LoRA - -In this section, we discuss fine-tuning OpenVLA using Low-Rank Adaptation (LoRA) via the Hugging Face `transformers` library, -which is recommended if you do not have sufficient compute to fully fine-tune a 7B-parameter model. The main script for LoRA -fine-tuning is `vla-scripts/finetune.py`. (If you instead wish to do full fine-tuning, please see the -[Fully Fine-Tuning OpenVLA](#fully-fine-tuning-openvla) section.) - -Below we show an example of how you can fine-tune the main OpenVLA checkpoint ([`openvla-7b`](https://huggingface.co/openvla/openvla-7b)) -via LoRA. Here we fine-tune on [BridgeData V2](https://rail-berkeley.github.io/bridgedata/) using a single A100 -GPU with 80 GB VRAM. (You can also fine-tune with a smaller GPU, as long as it has at least ~27 GB of memory, -by modifying the batch size.) - -First, download the BridgeData V2 dataset: - -```bash -# Change directory to your base datasets folder -cd - -# Download the full dataset (124 GB) -wget -r -nH --cut-dirs=4 --reject="index.html*" https://rail.eecs.berkeley.edu/datasets/bridge_release/data/tfds/bridge_dataset/ - -# Rename the dataset to `bridge_orig` (NOTE: Omitting this step may lead to runtime errors later) -mv bridge_dataset bridge_orig -``` - -Now, launch the LoRA fine-tuning script, as shown below. Note that `--batch_size==16` with `--grad_accumulation_steps==1` -requires ~72 GB GPU memory. If you have a smaller GPU, you should reduce `--batch_size` and increase `--grad_accumulation_steps` -to maintain an effective batch size that is large enough for stable training. If you have multiple GPUs and wish to train via -PyTorch Distributed Data Parallel (DDP), simply set `--nproc-per-node` in the `torchrun` command below to the number of available GPUs. +OpenVLA but with `finetune.py` using a `LeRobotDataset` instead of a `RLDSDataset`. +To use: +1. Make a directory `data`: `cd openvla; mkdir data` +2. Put your dataset in LeRobot format inside that directory. +3. Run `finetune.py`, for example, like this. First line is to add the `openvla/lerobot` submodule to Python search path. This way of loading `lerobot` helped me avoid some dependency conflicts. ```bash -torchrun --standalone --nnodes 1 --nproc-per-node 1 vla-scripts/finetune.py \ - --vla_path "openvla/openvla-7b" \ - --data_root_dir \ - --dataset_name bridge_orig \ - --run_root_dir \ - --adapter_tmp_dir \ - --lora_rank 32 \ - --batch_size 16 \ - --grad_accumulation_steps 1 \ - --learning_rate 5e-4 \ - --image_aug \ - --wandb_project \ - --wandb_entity \ - --save_steps -``` - -Note: If you set `--image_aug==False` in the command above, you will observe nearly 100% `action_accuracy` in the training logs, -since the [`openvla-7b`](https://huggingface.co/openvla/openvla-7b) model is already pretrained (without augmentations) on a -superset of datasets that includes BridgeData V2. - -To LoRA fine-tune on a different dataset, you can download the dataset from the [Open X-Embodiment (OXE)](https://robotics-transformer-x.github.io/) -mixture (see [this custom script](https://github.com/moojink/rlds_dataset_mod/blob/main/prepare_open_x.sh) for an example of how to download datasets -from OXE). Alternatively, if you have a custom dataset that is not part of OXE, you can either (a) convert the dataset to the RLDS format which is -compatible with our fine-tuning script (see [this repo](https://github.com/kpertsch/rlds_dataset_builder) for instructions on this), or (b) use your own -custom PyTorch Dataset wrapper (see comments in `vla-scripts/finetune.py` for instructions). We recommend option (a) for most users; the RLDS dataset and -dataloader are tested more extensively since we used these for all of our pretraining and fine-tuning experiments. - -For option (a), after you converted your dataset to RLDS, you need to register it with our data loader, by registering a dataset -config [here](prismatic/vla/datasets/rlds/oxe/configs.py#L54) and a dataset transform function [here](prismatic/vla/datasets/rlds/oxe/transforms.py#L828). - -Once you have integrated your new dataset, you can launch LoRA fine-tuning with the same `vla-scripts/finetune.py` script above. If you run into any issues, -please visit the [VLA Troubleshooting](#vla-troubleshooting) section or search for a similar issue in the [OpenVLA GitHub Issues page](https://github.com/openvla/openvla/issues?q=) -(including "Closed" issues). If you cannot find a similar issue there, feel free to create a new issue. - -## Fully Fine-Tuning OpenVLA - -In this section, we discuss fully fine-tuning OpenVLA (all 7.5 billion parameters) via native PyTorch Fully Sharded Data Parallel (FSDP) -using the [Prismatic VLMs](https://github.com/TRI-ML/prismatic-vlms) training script. Full fine-tuning is more advanced/involved and is only recommended -if you have sufficient compute (e.g., a full node of 8 A100 GPUs) and if LoRA fine-tuning is insufficient for your use case (e.g., if the fine-tuning distribution -varies drastically from the pretraining distribution). Otherwise, we recommend that you try parameter-efficient fine-tuning via LoRA, which is described in the -[Fine-Tuning OpenVLA via LoRA](#fine-tuning-openvla-via-lora) section. - -For full fine-tuning, you will need to download [a different version of the OpenVLA model checkpoint](https://huggingface.co/openvla/openvla-7b-prismatic) that is compatible -with the Prismatic VLMs codebase, which we built on top of to develop the OpenVLA model. You can download this Prismatic-compatible OpenVLA checkpoint using the git commands below -(alternatively, you can download via the [Hugging Face CLI](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli)): - -```bash -# Change directory to your base model checkpoints folder -cd - -# Download checkpoint (30 GB) -- may take a few minutes -git clone git@hf.co:openvla/openvla-7b-prismatic - -# If the command above did not download the full checkpoint, -# manually fetch it via git Large File Storage (LFS) -# Note: You may have to configure an SSH key for this to work -cd openvla-7b-prismatic -git lfs fetch --all -``` - -We show how you can fully fine-tune OpenVLA on [BridgeData V2](https://rail-berkeley.github.io/bridgedata/) using a single node with 8 GPUs. If you wish to -use a different number of GPUs (or nodes), you can modify the VLA training configuration in [`prismatic/conf/vla.py`](prismatic/conf/vla.py). - -Download the BridgeData V2 dataset: - -```bash -# Change directory to your base datasets folder -cd - -# Download the full dataset (124 GB) -wget -r -nH --cut-dirs=4 --reject="index.html*" https://rail.eecs.berkeley.edu/datasets/bridge_release/data/tfds/bridge_dataset/ - -# Rename the dataset to `bridge_orig` (NOTE: Omitting this step may lead to runtime errors later) -mv bridge_dataset bridge_orig -``` - -Next, create a [Hugging Face user access token](https://huggingface.co/docs/hub/en/security-tokens) and copy the token value (a string that starts with -`hf_...`) into a file named `.hf_token` at the root directory of this repo (`openvla/.hf_token`). - -```bash -# Go to openvla root directory -cd openvla - -# Copy HF token value into token file. Replace "hf_..." with your own token value! -# See: https://huggingface.co/docs/hub/en/security-tokens -echo hf_... >>> .hf_token -``` - -Now, launch the training script. If you wish to use a different number of nodes or GPUs, modify the VLA training configuration in -[`prismatic/conf/vla.py`](prismatic/conf/vla.py) and then change the `--nnodes` and `--nproc-per-node` arguments below accordingly. - -```bash -torchrun --standalone --nnodes 1 --nproc-per-node 8 vla-scripts/train.py \ - --pretrained_checkpoint \ - --vla.type prism-dinosiglip-224px+mx-bridge \ - --data_root_dir \ - --run_root_dir \ - --run_id \ - --image_aug \ - --wandb_project \ - --wandb_entity \ - --save_interval \ - --is_resume False -``` - -Note that the `--is_resume` argument is set to `False` above since we are fine-tuning a pretrained checkpoint rather than resuming a paused training run. - -If your training run gets paused and you wish to resume from the latest checkpoint, change `--pretrained_checkpoint` to the latest checkpoint path, -and then set `--is_resume==True` and specify `--resume_step` and `--resume_epoch` as the step and epoch number, respectively. For example, if you wish to -resume training from a checkpoint named `step-010000-epoch-20-loss=0.0160.pt`, you would set `is_resume==True`, `resume_step==10000`, and `resume_epoch==20`. - -Note: If you run the BridgeData V2 fine-tuning command above, you should observe nearly 100% Action Token Accuracy in the training logs, since the -[`openvla-7b`](https://huggingface.co/openvla/openvla-7b) model is already pretrained on a superset of datasets that includes BridgeData V2. - -To fully fine-tune OpenVLA on a different dataset, you can download the dataset from the [Open X-Embodiment (OXE)](https://robotics-transformer-x.github.io/) -mixture (see [this custom script](https://github.com/moojink/rlds_dataset_mod/blob/main/prepare_open_x.sh) for an example of how to download datasets from OXE). -Alternatively, if you have a custom dataset that is not part of OXE, you can convert the dataset to the RLDS format, which is compatible with our fine-tuning script -(see [this repo](https://github.com/kpertsch/rlds_dataset_builder) for instructions on this). After downloading/converting the dataset, you will need to modify the following files: - -* [`prismatic/conf/vla.py`](prismatic/conf/vla.py): Add a new training configuration by creating an experiment class, and then register it in the `VLARegistry` at the bottom of the file. - * Make sure to create a new unique `vla_id` for your fine-tuning run, and adjust some configuration variables as needed – e.g., `expected_world_size` (number of GPUs), - `per_device_batch_size` (batch size per GPU), `global_batch_size` (total batch size), `shuffle_buffer_size` (number of samples in shuffle buffer per GPU), etc. See comments - under the `VLAConfig` class at the top of the file to understand the purpose of each variable. -* [`prismatic/vla/datasets/rlds/oxe/mixtures.py`](prismatic/vla/datasets/rlds/oxe/mixtures.py): Define a new mixture for your fine-tuning mixture in the `OXE_NAMED_MIXTURES` dictionary. -* [`prismatic/vla/datasets/rlds/oxe/transforms.py`](prismatic/vla/datasets/rlds/oxe/transforms.py): Define a new dataset transform function for your fine-tuning dataset, and add it to the -`OXE_STANDARDIZATION_TRANSFORMS` registry at the bottom of the file. -* [`prismatic/vla/datasets/rlds/oxe/configs.py`](prismatic/vla/datasets/rlds/oxe/configs.py): Add a new configuration specifying your fine-tuning dataset's observation and action spaces -to the `OXE_DATASET_CONFIGS` dictionary. - -After completing the steps above, you can start full fine-tuning using the `vla-scripts/train.py` script. Make sure to set the `--vla.type` argument to the new `vla_id` that you added in `prismatic/conf/vla.py`. - -When you are finished with fine-tuning, you will need to convert the final model checkpoint to a version that is -compatible with the Hugging Face `transformers` library. See the [Converting Prismatic Models to Hugging Face](#converting-prismatic-models-to-hugging-face) section for instructions. - -If you run into any issues, please visit the [VLA Troubleshooting](#vla-troubleshooting) section or search for a similar issue in the -[OpenVLA GitHub Issues page](https://github.com/openvla/openvla/issues?q=) (including "Closed" issues). If you cannot find a similar issue there, feel free to create a new issue. - -### Converting Prismatic Models to Hugging Face - -If you have used the Prismatic VLMs codebase to train your model (e.g., if you did full fine-tuning of OpenVLA on a -new dataset), you will need to convert the final checkpoint to a version that is compatible with Hugging Face -`transformers` AutoClasses. We discuss how to do so in this section. - -Let's say your training run directory is `PRISMATIC_RUN_DIR` (e.g., `prism-dinosiglip-224px+mx-oxe-magic-soup-plus+n8+b32+x7`). -Inside this directory, there should be a directory called `checkpoints` which contains saved model checkpoints (e.g., -`step-295000-epoch-40-loss=0.2200.pt`). The Prismatic-to-Hugging-Face conversion script -([convert_openvla_weights_to_hf.py](vla-scripts/extern/convert_openvla_weights_to_hf.py)) expects a checkpoint file -named `latest-checkpoint.pt`. Therefore, you should first create a symbolic link called `latest-checkpoint.pt` that -points to the checkpoint file that you wish to convert: - -```bash -# Go to your Prismatic training run's `checkpoints` directory -cd PRISMATIC_RUN_DIR/checkpoints - -# Create symbolic link pointing to your checkpoint file -ln -s latest-checkpoint.pt -``` - -Then, launch the conversion script to convert the checkpoint from the Prismatic VLMs format to the Hugging Face format: - -```bash -python vla-scripts/extern/convert_openvla_weights_to_hf.py \ - --openvla_model_path_or_id \ - --output_hf_model_local_path -``` - -The command above will save the HF-compatible checkpoint in `output_hf_model_local_path`. Now you can load the checkpoint -with HF AutoClasses as normal, as shown below. Note that there is an additional necessary step to register the OpenVLA model -to HF AutoClasses before loading it because you are loading a locally saved checkpoint rather than one that is pushed to the -HF Hub (see [here](https://huggingface.co/docs/transformers/en/custom_models#registering-a-model-with-custom-code-to-the-auto-classes) -for details). - -```python -import torch -from transformers import AutoConfig, AutoImageProcessor, AutoModelForVision2Seq, AutoProcessor - -from prismatic.extern.hf.configuration_prismatic import OpenVLAConfig -from prismatic.extern.hf.modeling_prismatic import OpenVLAForActionPrediction -from prismatic.extern.hf.processing_prismatic import PrismaticImageProcessor, PrismaticProcessor - -# Register OpenVLA model to HF AutoClasses (not needed if you pushed model to HF Hub) -AutoConfig.register("openvla", OpenVLAConfig) -AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) -AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) -AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) - -# Load Processor & VLA -processor = AutoProcessor.from_pretrained("", trust_remote_code=True) -vla = AutoModelForVision2Seq.from_pretrained( - "", - attn_implementation="flash_attention_2", # [Optional] Requires `flash_attn` - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - trust_remote_code=True, -).to("cuda:0") - -... -``` - -## Training VLAs from Scratch - -We provide full instructions and configurations for training VLA models on (arbitrary subsets of) the -[Open X-Embodiment (OXE) Dataset](https://robotics-transformer-x.github.io/). If you run in to any issues with -the following, see [VLA Troubleshooting](#vla-troubleshooting) below (or file a GitHub Issue). - -### VLA Pretraining Datasets - -We download and preprocess individual datasets from Open X-Embodiment in [RLDS format](https://github.com/google-research/rlds) following -[this custom script](https://github.com/moojink/rlds_dataset_mod/blob/main/prepare_open_x.sh). See -[mixtures.py](./prismatic/vla/datasets/rlds/oxe/mixtures.py) for the full list of component datasets (and mixture -weights) we use to train `openvla-7b`. -- **Important**: For the BridgeData V2 component, the version in OXE is out of date (as of 12/20/2023). Instead, - you should download the dataset from the [official website](https://rail.eecs.berkeley.edu/datasets/bridge_release/data/tfds/bridge_dataset/) and place it under the subdirectory `bridge_orig/`. - Replace any reference to `bridge` in the OXE code with `bridge_orig`. - -### VLA Configuration & Training Script - -The entry point for VLA training is [`vla-scripts/train.py`](vla-scripts/train.py). We use -[`draccus`](https://pypi.org/project/draccus) to provide a modular, dataclass-based interface for specifying VLA -training configurations; existing VLA configurations are in [`prismatic/conf/vla.py`](prismatic/conf/vla.py). You can -add your own training configuration and refer to it using the `--vla.type` command line argument. - -We use PyTorch Fully Sharded Data Parallel (FSDP) to distribute training across GPUs. Launch training via `torchrun`: - -```bash -# Train VLA on BridgeData V2 with the Prismatic DINO-SigLIP 224px Backbone on a Single Node (w/ 8 GPUs) -torchrun --standalone --nnodes 1 --nproc-per-node 8 vla-scripts/train.py \ - --vla.type "prism-dinosiglip-224px+mx-bridge" \ - --data_root_dir \ - --run_root_dir \ - --wandb_project "" \ - --wandb_entity "" -``` - -### VLA Troubleshooting - -The following are a list of known problems and corresponding fixes: - -```bash -FileNotFoundError: Failed to construct dataset "fractal20220817_data", builder_kwargs "{'data_dir': '/path/to/processed/datasets/'}": Could not load dataset info from fractal20220817_data/0.1.0/dataset_info.json -``` -- **Fix**: Downgrade `tensorflow-datasets` via `pip install tensorflow-datasets==4.9.3`. - - -```bash -AttributeError: 'DLataset' object has no attribute 'traj_map'. Did you mean: 'flat_map'? -``` -- **Fix**: Upgrade `dlimp` to the newest version. You may have to `--force-reinstall` like so: -`pip install --no-deps --force-reinstall git+https://github.com/moojink/dlimp_openvla` - ---- - -## Evaluating OpenVLA - -### BridgeData V2 WidowX Evaluations - -#### Setup - -Clone the [BridgeData V2 WidowX controller repo](https://github.com/rail-berkeley/bridge_data_robot) and install the `widowx_envs` package: - -```bash -git clone https://github.com/rail-berkeley/bridge_data_robot.git -cd bridge_data_robot -pip install -e widowx_envs -``` - -Additionally, install the [`edgeml`](https://github.com/youliangtan/edgeml) library: -```bash -git clone https://github.com/youliangtan/edgeml.git -cd edgeml -pip install -e . -``` - -Follow the instructions in the `bridge_data_robot` README to create the Bridge WidowX Docker container. - -#### Launching BridgeData V2 Evaluations - -There are multiple ways to run BridgeData V2 evaluations. We describe the server-client method below. - -In one Terminal window (e.g., in tmux), start the WidowX Docker container: - -```bash -cd bridge_data_robot -./generate_usb_config.sh -USB_CONNECTOR_CHART=$(pwd)/usb_connector_chart.yml docker compose up --build robonet -``` - -In a second Terminal window, run the WidowX robot server: - -```bash -cd bridge_data_robot -docker compose exec robonet bash -lic "widowx_env_service --server" -``` - -In a third Terminal window, run the OpenVLA policy evaluation script: - -```bash -cd openvla -python experiments/robot/bridge/run_bridgev2_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b -``` - -If you run into any problems with evaluations, please file a GitHub Issue. - - -### LIBERO Simulation Benchmark Evaluations - -In the [updated OpenVLA paper (v2)](https://arxiv.org/abs/2406.09246), we discuss fine-tuning OpenVLA -on a simulated benchmark, [LIBERO](https://libero-project.github.io/main.html), in Appendix E. -Please see the paper for details, such as how we modify the provided demonstration datasets to -improve the overall performance of all methods. - -We copy the results to the section below and then discuss how to reproduce the results for OpenVLA. - -#### OpenVLA Fine-Tuning Results - -| Method | LIBERO-Spatial | LIBERO-Object | LIBERO-Goal | LIBERO-Long | Average | -|--------|----------------|---------------|-------------|-------------|---------| -| Diffusion Policy from scratch | 78.3 ± 1.1% | **92.5 ± 0.7%** | 68.3 ± 1.2% | 50.5 ± 1.3% | 72.4 ± 0.7% | -| Octo fine-tuned | 78.9 ± 1.0% | 85.7 ± 0.9% | **84.6 ± 0.9%** | 51.1 ± 1.3% | 75.1 ± 0.6% | -| OpenVLA fine-tuned (ours) | **84.7 ± 0.9%** | 88.4 ± 0.8% | 79.2 ± 1.0% | **53.7 ± 1.3%** | **76.5 ± 0.6%** | - -Each success rate is the average over 3 random seeds x 500 rollouts each (10 tasks x 50 rollouts per task). - -#### LIBERO Setup - -Clone and install the [LIBERO repo](https://github.com/Lifelong-Robot-Learning/LIBERO): - -```bash -git clone https://github.com/Lifelong-Robot-Learning/LIBERO.git -cd LIBERO -pip install -e . -``` - -Additionally, install other required packages: -```bash -cd openvla -pip install -r experiments/robot/libero/libero_requirements.txt -``` - -(Optional) To download the modified versions of the LIBERO datasets that we used in our fine-tuning -experiments, run the command below. This will download the LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, -and LIBERO-10 datasets in RLDS data format (~10 GB total). You can use these to fine-tune OpenVLA or -train other methods. This step is optional since we provide pretrained OpenVLA checkpoints below. -(Also, you can find the script we used to generate the modified datasets in raw HDF5 format -[here](experiments/robot/libero/regenerate_libero_dataset.py) and the code we used to convert these -datasets to the RLDS format [here](https://github.com/moojink/rlds_dataset_builder).) -```bash -git clone git@hf.co:datasets/openvla/modified_libero_rlds -``` - -#### Launching LIBERO Evaluations - -We fine-tuned OpenVLA via LoRA (r=32) on four LIBERO task suites independently: LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, and LIBERO-10 (also called LIBERO-Long). -The four checkpoints are available on Hugging Face: -* [openvla/openvla-7b-finetuned-libero-spatial](https://huggingface.co/openvla/openvla-7b-finetuned-libero-spatial) -* [openvla/openvla-7b-finetuned-libero-object](https://huggingface.co/openvla/openvla-7b-finetuned-libero-object) -* [openvla/openvla-7b-finetuned-libero-goal](https://huggingface.co/openvla/openvla-7b-finetuned-libero-goal) -* [openvla/openvla-7b-finetuned-libero-10](https://huggingface.co/openvla/openvla-7b-finetuned-libero-10) - -To start evaluation with one of these checkpoints, run one of the commands below. Each will automatically download the appropriate checkpoint listed above. - -```bash -# Launch LIBERO-Spatial evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-spatial \ - --task_suite_name libero_spatial \ - --center_crop True - -# Launch LIBERO-Object evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-object \ - --task_suite_name libero_object \ - --center_crop True - -# Launch LIBERO-Goal evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-goal \ - --task_suite_name libero_goal \ - --center_crop True - -# Launch LIBERO-10 (LIBERO-Long) evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-10 \ - --task_suite_name libero_10 \ - --center_crop True -``` - -Notes: -* The evaluation script will run 500 trials by default (10 tasks x 500 episodes each). You can modify the number of - trials per task by setting `--num_trials_per_task`. You can also change the random seed via `--seed`. -* **NOTE: Setting `--center_crop True` is important** because we fine-tuned OpenVLA with random crop augmentations - (we took a random crop with 90% area in every training sample, so at test time we simply take the center 90% crop). -* The evaluation script logs results locally. You can also log results in Weights & Biases - by setting `--use_wandb True` and specifying `--wandb_project ` and `--wandb_entity `. -* The results reported in our paper were obtained using **Python 3.10.13, PyTorch 2.2.0, transformers 4.40.1, and - flash-attn 2.5.5** on an **NVIDIA A100 GPU**, averaged over three random seeds. Please stick to these package versions. - Note that results may vary slightly if you use a different GPU for evaluation due to GPU nondeterminism in large models - (though we have tested that results were consistent across different machines with A100 GPUs). - -Please file a GitHub Issue if you run into any problems. - ---- - -## Repository Structure - -High-level overview of repository/project file-tree: - -+ `prismatic` - Package source; provides core utilities for model loading, training, data preprocessing, etc. -+ `vla-scripts/` - Core scripts for training, fine-tuning, and deploying VLAs. -+ `experiments/` - Code for evaluating OpenVLA policies in robot environments. -+ `LICENSE` - All code is made available under the MIT License; happy hacking! -+ `Makefile` - Top-level Makefile (by default, supports linting - checking & auto-fix); extend as needed. -+ `pyproject.toml` - Full project configuration details (including dependencies), as well as tool configurations. -+ `README.md` - You are here! - ---- - - -# VLA Performance Troubleshooting - -In this section we cover best practices for debugging poor VLA performance after fine-tuning on your target domain robot dataset. - -**Note**: OpenVLA typically requires fine-tuning on a small demonstration dataset (~100 demos) from your target domain robot. Out-of-the-box, it only works well on domains from the training dataset. - -**Sanity checks**: -- replay the actions from a demonstration from your fine-tuning dataset and make sure that the robot can execute the task successfully (this ensures that your data collection pipeline is correct) -- once you fine-tuned a model, load the model in your inference pipeline (as if you would run it to control the robot), but feed images from the fine-tuning dataset into the model (pretending they come from the robot) and verify that you can reproduce the token accuracies / L1 errors from training (this ensures that your inference pipeline is correct) - -**Best practices for fine-tuning data collection**: -If your setup passed the above two sanity checks, the issue may not be in model training, but in the data you fine-tuned the model with. Some best practices for data collection: -- *Collect at a control frequency around 5-10Hz.* OpenVLA is not trained with action chunking, empirically the model struggles with high-frequency data. If your robot setup uses a high-frequency controller (eg 50 Hz), consider downsampling your actions to 5Hz. Verify first that your robot can still solve the task when using 5Hz actions (ie repeat sanity check (1) above with 5Hz actions) -- *Avoid pauses / small actions during data collection.* Because OpenVLA is trained without action chunking, the model can be sensitive to idle actions in the fine-tuning data. If your data contains steps in which the robot barely moves, the model may "get stuck" in these steps at inference time. Try to collect fine-tuning demonstrations with continuous, slow movement. -- *Ensure sufficient data coverage.* If you plan to test the model with some variation, e.g. different initial positions of objects, make sure that your fine-tuning data contains sufficient diversity of such conditions as well, e.g. shows demonstrations with diverse initial conditions. -- *Use consistent task strategies during data collection.* This is not a hard constraint, but may make your life easier. Try to demonstrate tasks in consistent ways, e.g. approach objects from the same side, perform sub-steps in the same order even if they could be performed in arbitrary sequences. Being consistent gives you a less multi-modal fine-tuning dataset, which makes the modeling problem easier. - - ---- - -#### Citation - -If you find our code or models useful in your work, please cite [our paper](https://arxiv.org/abs/2406.09246): - -```bibtex -@article{kim24openvla, - title={OpenVLA: An Open-Source Vision-Language-Action Model}, - author={{Moo Jin} Kim and Karl Pertsch and Siddharth Karamcheti and Ted Xiao and Ashwin Balakrishna and Suraj Nair and Rafael Rafailov and Ethan Foster and Grace Lam and Pannag Sanketi and Quan Vuong and Thomas Kollar and Benjamin Burchfiel and Russ Tedrake and Dorsa Sadigh and Sergey Levine and Percy Liang and Chelsea Finn}, - journal = {arXiv preprint arXiv:2406.09246}, - year={2024} -} +PYTHONPATH="$PYTHONPATH:$(pwd)/lerobot" \ +torchrun --standalone --nnodes 1 --nproc-per-node 1 vla-scripts/finetune.py --vla_path "openvla/openvla-7b" --data_root_dir "data" --dataset_name converted_libero_v6 --run_root_dir .runs --adapter_tmp_dir .adapter --lora_rank 32 --batch_size 2 --grad_accumulation_steps 8 --learning_rate 5e-4 --image_aug True --save_steps 10 ``` +Here we assume you have a dataset called `converted_libero_v6` inside `data`. From 3262db0a94f20a1be3a6c957224c0d2b4a7440a9 Mon Sep 17 00:00:00 2001 From: mehhl Date: Fri, 27 Dec 2024 19:26:16 +0000 Subject: [PATCH 05/58] added script for converting raw nomagic-ur5e to lerobot --- .../additional-datasets/nomagic_ur5e_raw.py | 340 ++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 scripts/additional-datasets/nomagic_ur5e_raw.py diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py new file mode 100644 index 000000000..53dae5e31 --- /dev/null +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -0,0 +1,340 @@ +""" +convert_nomagic_ur5e_raw.py + +Convert raw CSV/MP4 data from Nomagic's UR5e robot arm into a LeRobotDataset. + +The data should be in a format similar to the following: + +raw/trajectories/ +urXPose_20241219_133722.csv +urXPose_20241219_133814.csv + ... + +raw/videos/ + 2024-12-19-12:37:26:897865_d5fb919d-2b3a-4d4b-b885-1f890a255b66.mp4 + 2024-12-19-12:38:19:058352_99a53149-eb87-4a0d-979f-01b1283c5804.mp4 + ... + +The final directory will look like: + +data/my_lerobot_dataset/ + data/ + chunk-000/ + episode_000000.parquet + episode_000001.parquet + ... + meta/ + info.json + stats.json + episodes.jsonl + tasks.jsonl + videos/ + chunk-000/ + observation.images.side/ + episode_000000.mp4 + episode_000001.mp4 + ... + +Notes: + - This script assumes that the CSV files have a matching MP4 file + by a shared timestamp in the filename, e.g."urXPose_20241219_133722.csv" + ↔ "2024-12-19-12:37:26:897865_d5fb919d-2b3a-4d4b-b885-1f890a255b66.mp4" +""" + +import os +import re +import json +import shutil +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import cv2 + +from pathlib import Path +from typing import List, Dict +from dataclasses import dataclass +from scipy.spatial.transform import Rotation as R + +import logging + +# Set up logging +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler() # Add a stream handler to show messages in console + ] +) + +@dataclass +class LeRobotFrame: + """ + Represents a single entry (frame) in the final LeRobot parquet data. + For example: + timestamp (sec), + episode_index, + next.done (bool), + task_index, + index, # global index in the episode + frame_index, # might be same as index, or you can offset if needed + action # a list of [dx, dy, dz, dox, doy, doz, grip] + """ + timestamp: float + episode_index: int + next_done: bool + task_index: int + index: int + frame_index: int + action: List[float] + +def find_pairs(raw_traj_dir: Path, raw_video_dir: Path): + csv_files = sorted(raw_traj_dir.glob("*.csv")) + mp4_files = sorted(raw_video_dir.glob("*.mp4")) + + logging.debug(f"Found CSV files: {[f.name for f in csv_files]}") + logging.debug(f"Found MP4 files: {[f.name for f in mp4_files]}") + + # Just pair them up in order since they're already sorted chronologically + pairs = list(zip(csv_files, mp4_files)) + + for csv_f, mp4_f in pairs: + logging.debug(f"Paired {csv_f.name} with {mp4_f.name}") + + return pairs + +def compute_actions_from_rows(rowA: pd.Series, rowB: pd.Series): + """ + Convert two consecutive CSV rows into a delta action (dx, dy, dz, dox, doy, doz, grip). + Calculate Euler angle difference (dox, doy, doz) from quaternion pair using scipy.Rotation. + """ + # Position deltas + dx = float(rowB["PositionX"] - rowA["PositionX"]) + dy = float(rowB["PositionY"] - rowA["PositionY"]) + dz = float(rowB["PositionZ"] - rowA["PositionZ"]) + + # Get quaternions in [x,y,z,w] format for scipy.Rotation + q1 = [rowA["OrientationX"], rowA["OrientationY"], + rowA["OrientationZ"], rowA["OrientationW"]] + q2 = [rowB["OrientationX"], rowB["OrientationY"], + rowB["OrientationZ"], rowB["OrientationW"]] + + # Calculate orientation difference using quaternion_difference logic + r1 = R.from_quat(q1) + r2 = R.from_quat(q2) + r_diff = r2 * r1.inv() + euler_diff = r_diff.as_euler("xyz") + dox, doy, doz = euler_diff + + # Gripper - map Gripper::Action values to float + grip_action_map = { + "Gripper::Action::NONE": 0.0, + "Gripper::Action::RELEASE": -1.0, + "Gripper::Action::GRAB": 1.0 + } + if rowA["GripperAction"] not in grip_action_map: + raise ValueError(f"Unknown gripper action: {rowA['GripperAction']}") + grip_val = grip_action_map[rowA["GripperAction"]] + + return [dx, dy, dz, dox, doy, doz, grip_val] + +def convert_single_episode( + csv_file: Path, + mp4_file: Path, + episode_index: int, + out_dir: Path +) -> None: + """ + Convert one CSV + MP4 into a single "episode_{:06d}.parquet" and + copy the MP4 to "episode_{:06d}.mp4" in observation.images.side subdir. + """ + # Load CSV data + df = pd.read_csv(csv_file) + # Convert timestamps to relative seconds from start of episode + df["Timestamp"] = pd.to_datetime(df["Timestamp"]) + first_time = df["Timestamp"].iloc[0] + df["timestamp"] = (df["Timestamp"] - first_time).dt.total_seconds() + + # For convenience, define a small list of frames. We'll fill them up. + final_frames: List[LeRobotFrame] = [] + + for i in range(len(df) - 1): + rowA = df.iloc[i] + rowB = df.iloc[i + 1] + # Build an action + action_vals = compute_actions_from_rows(rowA, rowB) + # Use the rowA's timestamp (relative seconds from start of episode) + ts_val = float(rowA["timestamp"]) + # next.done is usually False unless e.g. i == len(df) - 2 + next_done = i == (len(df) - 2) + # Build the frame record + final_frames.append( + LeRobotFrame( + timestamp=ts_val, + episode_index=episode_index, + next_done=next_done, + task_index=0, + index=i, + frame_index=i, + action=action_vals + ) + ) + + # Write out as a parquet file + out_parquet = out_dir / f"episode_{episode_index:06d}.parquet" + pa_frames = pa.Table.from_pydict({ + "timestamp": [f.timestamp for f in final_frames], + "episode_index": [f.episode_index for f in final_frames], + "next.done": [f.next_done for f in final_frames], + "task_index": [f.task_index for f in final_frames], + "index": [f.index for f in final_frames], + "frame_index": [f.frame_index for f in final_frames], + "action": pa.array([f.action for f in final_frames], type=pa.list_(pa.float32())) + }) + pq.write_table(pa_frames, out_parquet) + print(f"[Episode {episode_index}] Saved parquet => {out_parquet}") + + # Copy MP4 to the expected output location + video_outdir = out_dir.parent.parent \ + / "videos" \ + / "chunk-000" \ + / "observation.images.side" + video_outdir.mkdir(parents=True, exist_ok=True) + episode_mp4 = video_outdir / f"episode_{episode_index:06d}.mp4" + shutil.copy(mp4_file, episode_mp4) + print(f"[Episode {episode_index}] Saved MP4 => {episode_mp4}") + +def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[int]): + meta_dir = out_root / "meta" + meta_dir.mkdir(exist_ok=True) + + # Read video metadata from first video file + first_video = next((out_root / "videos" / "chunk-000" / "observation.images.side").glob("*.mp4")) + cap = cv2.VideoCapture(str(first_video)) + + # Get basic video properties + fps = cap.get(cv2.CAP_PROP_FPS) + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + # Read the first frame to get number of channels + ret, frame = cap.read() + channels = frame.shape[2] if ret else 3 + + # Get codec information + fourcc = int(cap.get(cv2.CAP_PROP_FOURCC)) + codec = "".join([chr((fourcc >> 8 * i) & 0xFF) for i in range(4)]) + + cap.release() + + # info.json + info_data = { + "codebase_version": "v2.0", + "robot_type": "UR5e", + "total_episodes": total_episodes, + "total_frames": sum(episode_lengths), # Total frames across all episodes + "total_tasks": 1, + "total_videos": total_episodes, + "total_chunks": 1, + "chunks_size": total_episodes, + "fps": fps, + "splits": {"train": f"0:{total_episodes}"}, + "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet", + "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4", + "features": { + "observation.images.side": { + "dtype": "video", + "shape": [height, width, channels], + "names": ["height", "width", "channel"], + "video_info": { + "video.fps": fps, + "video.codec": codec, + "pix_fmt": "yuv420p", # This is still hardcoded as it's not easily accessible via OpenCV + "has_audio": False # OpenCV doesn't expose audio info, but these are known to be video-only + } + }, + "action": { + "dtype": "float32", + "shape": [7], + "names": [ + "PositionX", "PositionY", "PositionZ", + "OrientationX", "OrientationY", "OrientationZ", + "GripperAction" + ] + } + } + } + with open(meta_dir / "info.json", "w") as f: + json.dump(info_data, f, indent=2) + + # stats.json + stats_data = { + "action": { + "q01": [-0.03, -0.03, -0.03, -0.03, -0.03, -0.03, -1], + "q99": [0.03, 0.03, 0.03, 0.03, 0.03, 0.03, 1] + } + } + with open(meta_dir / "stats.json", "w") as f: + json.dump(stats_data, f, indent=2) + + # episodes.jsonl + with open(meta_dir / "episodes.jsonl", "w") as f: + for eidx, length in enumerate(episode_lengths): + row = { + "episode_index": eidx, + "tasks": ["Pick up the object"], + "length": length # Store actual # frames + } + f.write(json.dumps(row) + "\n") + + # tasks.jsonl + with open(meta_dir / "tasks.jsonl", "w") as f: + row = { + "task_index": 0, + "task": "Demonstration from raw robot data" + } + f.write(json.dumps(row) + "\n") + +def main(raw_data_prefix: Path = None): + print(f"\nStarting conversion with raw_data_prefix: {raw_data_prefix}") + logging.debug("Starting the conversion process.") # Logging line + # Where is your raw data? + if raw_data_prefix is None: + raw_data_prefix = Path(".") + raw_traj_dir = raw_data_prefix / "raw/trajectories" + raw_video_dir = raw_data_prefix / "raw/videos" + print(f"Looking for data in:\n {raw_traj_dir}\n {raw_video_dir}") + # Where do you want the new dataset to live? + out_root = raw_data_prefix / "data/my_lerobot_dataset" + data_out_dir = out_root / "data" / "chunk-000" + data_out_dir.mkdir(parents=True, exist_ok=True) + + # Pair up CSV + MP4 + pairs = find_pairs(raw_traj_dir, raw_video_dir) + + # Convert each episode + episode_lengths = [] + for episode_index, (csv_f, mp4_f) in enumerate(pairs): + logging.debug(f"Processing episode {episode_index} with CSV: {csv_f.name} and MP4: {mp4_f.name}") + convert_single_episode( + csv_file=csv_f, + mp4_file=mp4_f, + episode_index=episode_index, + out_dir=data_out_dir + ) + # Append the length of each episode + episode_length = len(pd.read_csv(csv_f)) - 1 # Decrement by 1 + episode_lengths.append(episode_length) + logging.debug(f"Episode {episode_index} length: {episode_length}") + + # Build meta files + build_meta_files(out_root=out_root, total_episodes=len(pairs), episode_lengths=episode_lengths) + print("\nDone creating LeRobot-style dataset at:", out_root) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--raw_data_prefix", type=Path, default=None, + help="Path prefix to raw data directory") + args = parser.parse_args() + print("Running with args:", args) + main(raw_data_prefix=args.raw_data_prefix) \ No newline at end of file From 3fd90e57a5fb4cad767e905c809e0cc26a999a52 Mon Sep 17 00:00:00 2001 From: mehhl Date: Fri, 27 Dec 2024 19:36:01 +0000 Subject: [PATCH 06/58] fixes and parametrizations to nomagic-ur5e converter --- .../additional-datasets/nomagic_ur5e_raw.py | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index 53dae5e31..f02550473 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -207,12 +207,36 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ meta_dir = out_root / "meta" meta_dir.mkdir(exist_ok=True) + # Read trajectory fps, average over all episodes + out_parquet_dir = out_root / "data" / "chunk-000" + parquet_files = os.listdir(out_parquet_dir) + parquet_files = [f for f in parquet_files if f.endswith(".parquet")] + parquet_file_fps = [] + for parquetf in parquet_files: + df = pd.read_parquet(out_parquet_dir / parquetf) + inv_fps = df['timestamp'].diff().mean() + fps = 1.0 / inv_fps + parquet_file_fps.append(fps) + print(f"{fps=}") + mean_trajectory_fps = int(sum(parquet_file_fps) / len(parquet_file_fps)) + + # Get total number of video frames + video_dir = out_root / "videos" / "chunk-000" / "observation.images.side" + video_frame_counts = [] + for episode_idx in range(total_episodes): + video_path = video_dir / f"episode_{episode_idx:06d}.mp4" + cap = cv2.VideoCapture(str(video_path)) + frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + video_frame_counts.append(frame_count) + cap.release() + total_video_frames = sum(video_frame_counts) + # Read video metadata from first video file first_video = next((out_root / "videos" / "chunk-000" / "observation.images.side").glob("*.mp4")) cap = cv2.VideoCapture(str(first_video)) # Get basic video properties - fps = cap.get(cv2.CAP_PROP_FPS) + video_fps = cap.get(cv2.CAP_PROP_FPS) width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) @@ -231,12 +255,12 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ "codebase_version": "v2.0", "robot_type": "UR5e", "total_episodes": total_episodes, - "total_frames": sum(episode_lengths), # Total frames across all episodes + "total_frames": total_video_frames, "total_tasks": 1, "total_videos": total_episodes, "total_chunks": 1, "chunks_size": total_episodes, - "fps": fps, + "fps": mean_trajectory_fps, "splits": {"train": f"0:{total_episodes}"}, "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet", "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4", @@ -246,7 +270,7 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ "shape": [height, width, channels], "names": ["height", "width", "channel"], "video_info": { - "video.fps": fps, + "video.fps": video_fps, "video.codec": codec, "pix_fmt": "yuv420p", # This is still hardcoded as it's not easily accessible via OpenCV "has_audio": False # OpenCV doesn't expose audio info, but these are known to be video-only @@ -278,11 +302,11 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ # episodes.jsonl with open(meta_dir / "episodes.jsonl", "w") as f: - for eidx, length in enumerate(episode_lengths): + for eidx, frame_count in enumerate(video_frame_counts): row = { "episode_index": eidx, "tasks": ["Pick up the object"], - "length": length # Store actual # frames + "length": frame_count } f.write(json.dumps(row) + "\n") @@ -294,7 +318,7 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ } f.write(json.dumps(row) + "\n") -def main(raw_data_prefix: Path = None): +def main(raw_data_prefix: Path = None, out_root: Path = None): print(f"\nStarting conversion with raw_data_prefix: {raw_data_prefix}") logging.debug("Starting the conversion process.") # Logging line # Where is your raw data? @@ -303,8 +327,10 @@ def main(raw_data_prefix: Path = None): raw_traj_dir = raw_data_prefix / "raw/trajectories" raw_video_dir = raw_data_prefix / "raw/videos" print(f"Looking for data in:\n {raw_traj_dir}\n {raw_video_dir}") + # Where do you want the new dataset to live? - out_root = raw_data_prefix / "data/my_lerobot_dataset" + if out_root is None: + out_root = raw_data_prefix / "data/my_lerobot_dataset" data_out_dir = out_root / "data" / "chunk-000" data_out_dir.mkdir(parents=True, exist_ok=True) @@ -335,6 +361,8 @@ def main(raw_data_prefix: Path = None): parser = argparse.ArgumentParser() parser.add_argument("--raw_data_prefix", type=Path, default=None, help="Path prefix to raw data directory") + parser.add_argument("--out_root", type=Path, default=None, + help="Output directory for the dataset") args = parser.parse_args() print("Running with args:", args) - main(raw_data_prefix=args.raw_data_prefix) \ No newline at end of file + main(raw_data_prefix=args.raw_data_prefix, out_root=args.out_root) \ No newline at end of file From 91a81ae2135122fb9770f2d241438d4183993a60 Mon Sep 17 00:00:00 2001 From: mehhl Date: Sat, 28 Dec 2024 22:05:51 +0000 Subject: [PATCH 07/58] revamp of nomagic ur5e -> lerobot converter --- .../additional-datasets/nomagic_ur5e_raw.py | 355 ++++++++++++++---- 1 file changed, 272 insertions(+), 83 deletions(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index f02550473..0da346c4d 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -41,6 +41,7 @@ ↔ "2024-12-19-12:37:26:897865_d5fb919d-2b3a-4d4b-b885-1f890a255b66.mp4" """ +import exiftool import os import re import json @@ -52,9 +53,17 @@ from pathlib import Path from typing import List, Dict +import torch +import numpy as np from dataclasses import dataclass from scipy.spatial.transform import Rotation as R +from datasets import Dataset +from lerobot.common.datasets.utils import ( + check_timestamps_sync, + calculate_episode_data_index +) + import logging # Set up logging @@ -67,7 +76,7 @@ ) @dataclass -class LeRobotFrame: +class LeRobotTrajectoryStep: """ Represents a single entry (frame) in the final LeRobot parquet data. For example: @@ -87,20 +96,33 @@ class LeRobotFrame: frame_index: int action: List[float] -def find_pairs(raw_traj_dir: Path, raw_video_dir: Path): - csv_files = sorted(raw_traj_dir.glob("*.csv")) - mp4_files = sorted(raw_video_dir.glob("*.mp4")) - logging.debug(f"Found CSV files: {[f.name for f in csv_files]}") - logging.debug(f"Found MP4 files: {[f.name for f in mp4_files]}") +def pair_trajectories_and_videos( + csv_trajectories_dir: Path, + mp4_videos_dir: Path +) -> list[tuple[Path, Path]]: + """ + Pair up CSV trajectories and MP4 videos. + Uses alphanumeric order on filenames. + """ + + csv_trajectories = sorted(csv_trajectories_dir.glob("*.csv")) + mp4_videos = sorted(mp4_videos_dir.glob("*.mp4")) + + logging.debug(f"Found CSV files: {[f.name for f in csv_trajectories]}") + logging.debug(f"Found MP4 files: {[f.name for f in mp4_videos]}") - # Just pair them up in order since they're already sorted chronologically - pairs = list(zip(csv_files, mp4_files)) + # We pair them up in order since they're already sorted chronologically + # TODO: compare actual timestamps in filename, not alphanumeric order + + # We pair them up alphanumerically + trajectory_video_pairs = list(zip(csv_trajectories, mp4_videos)) - for csv_f, mp4_f in pairs: - logging.debug(f"Paired {csv_f.name} with {mp4_f.name}") + for csv_trajectory, mp4_video in trajectory_video_pairs: + logging.debug(f"Paired {csv_trajectory.name} with {mp4_video.name}") + - return pairs + return trajectory_video_pairs def compute_actions_from_rows(rowA: pd.Series, rowB: pd.Series): """ @@ -137,89 +159,174 @@ def compute_actions_from_rows(rowA: pd.Series, rowB: pd.Series): return [dx, dy, dz, dox, doy, doz, grip_val] -def convert_single_episode( - csv_file: Path, - mp4_file: Path, +def get_frames(video_path: str) -> list: + vidcap = cv2.VideoCapture(video_path) + success, image = vidcap.read() + frames = [] + while success: + frames.append(image) + success, image = vidcap.read() + return frames + +def make_video_timestamps(mp4_filepath: str, frames) -> list[int]: + with exiftool.ExifToolHelper() as et: + metadata = et.get_metadata(mp4_filepath) + video_timestamps = [int(i) for i in metadata[0]["XMP:Timestamps"]] + # ExifTool doesn't always return all frames, so we need to estimate + # TODO: can we just use cv2 to get fps and divide length by that? + avg_frame = \ + int((video_timestamps[-1] - video_timestamps[0]) \ + / (len(video_timestamps) - 1)) + while len(video_timestamps) < len(frames): + video_timestamps.append(video_timestamps[-1] + avg_frame) + # Convert to Unix seconds + nanoseconds_to_miliseconds = 1_000_000 + video_timestamps = [ + int(i / nanoseconds_to_miliseconds) + for i in video_timestamps + ] + return video_timestamps + +def select_ts_closest_to_reference( + csv_trajectory_timestamps: pd.Series, + video_timestamps: list[int] +) -> list[int]: + trajectory_timestamps = [] + for video_ts in video_timestamps: + ts_not_in_csv_trajectory_range = ( + video_ts < csv_trajectory_timestamps.min() or + video_ts > csv_trajectory_timestamps.max() + ) + if ts_not_in_csv_trajectory_range: + continue + try: + closest_csv_trajectory_ts = csv_trajectory_timestamps.iloc[ + csv_trajectory_timestamps.searchsorted(video_ts) + ] + trajectory_timestamps.append(closest_csv_trajectory_ts) + except IndexError: + continue + return trajectory_timestamps + +def csv_to_lerobot_trajectory( + csv_trajectory_filepath: Path, + mp4_filepath: Path, episode_index: int, - out_dir: Path -) -> None: + time_delta: int = 50, +) -> pa.Table: """ Convert one CSV + MP4 into a single "episode_{:06d}.parquet" and copy the MP4 to "episode_{:06d}.mp4" in observation.images.side subdir. """ + # Load CSV data - df = pd.read_csv(csv_file) - # Convert timestamps to relative seconds from start of episode - df["Timestamp"] = pd.to_datetime(df["Timestamp"]) - first_time = df["Timestamp"].iloc[0] - df["timestamp"] = (df["Timestamp"] - first_time).dt.total_seconds() - - # For convenience, define a small list of frames. We'll fill them up. - final_frames: List[LeRobotFrame] = [] - - for i in range(len(df) - 1): - rowA = df.iloc[i] - rowB = df.iloc[i + 1] + csv_trajectory_df = pd.read_csv(csv_trajectory_filepath) + + # Get CSV trajectory timesteps in in Unix seconds + csv_trajectory_df["seconds"] = ( + pd + .to_datetime(csv_trajectory_df["Timestamp"]) + .add(pd.Timedelta(hours=-1)) + .apply(lambda x: x.timestamp() * 1000) + .astype(int) + ) + # Get video timestamps in Unix miliseconds + frames = get_frames(mp4_filepath) + video_timestamps = make_video_timestamps(mp4_filepath, frames) + + + lerobot_trajectory: List[LeRobotTrajectoryStep] = [] + for frame_idx, video_ts in enumerate(video_timestamps): + video_ts_not_in_csv_trajectory_range = ( + video_ts < csv_trajectory_df["seconds"].min() or + video_ts > csv_trajectory_df["seconds"].max() + ) + if video_ts_not_in_csv_trajectory_range: + continue + try: + rowA = csv_trajectory_df.iloc[ + csv_trajectory_df["seconds"].searchsorted(video_ts) + ] + rowB = csv_trajectory_df.iloc[ + csv_trajectory_df["seconds"].searchsorted(video_ts + time_delta) + ] + except IndexError: + continue + # Build an action action_vals = compute_actions_from_rows(rowA, rowB) # Use the rowA's timestamp (relative seconds from start of episode) - ts_val = float(rowA["timestamp"]) + # divide by 1000 to convert from miliseconds to seconds + ts_val = float(rowA["seconds"] - csv_trajectory_df["seconds"].min()) / 1000 # next.done is usually False unless e.g. i == len(df) - 2 - next_done = i == (len(df) - 2) + next_done = frame_idx == (len(video_timestamps) - 2) + # Build the frame record - final_frames.append( - LeRobotFrame( + lerobot_trajectory.append( + LeRobotTrajectoryStep( timestamp=ts_val, episode_index=episode_index, next_done=next_done, task_index=0, - index=i, - frame_index=i, + index=frame_idx, + frame_index=frame_idx, action=action_vals ) ) # Write out as a parquet file - out_parquet = out_dir / f"episode_{episode_index:06d}.parquet" - pa_frames = pa.Table.from_pydict({ - "timestamp": [f.timestamp for f in final_frames], - "episode_index": [f.episode_index for f in final_frames], - "next.done": [f.next_done for f in final_frames], - "task_index": [f.task_index for f in final_frames], - "index": [f.index for f in final_frames], - "frame_index": [f.frame_index for f in final_frames], - "action": pa.array([f.action for f in final_frames], type=pa.list_(pa.float32())) + pa_trajectory = pa.Table.from_pydict({ + "timestamp": [f.timestamp for f in lerobot_trajectory], + "episode_index": [f.episode_index for f in lerobot_trajectory], + "next.done": [f.next_done for f in lerobot_trajectory], + "task_index": [f.task_index for f in lerobot_trajectory], + "index": [f.index for f in lerobot_trajectory], + "frame_index": [f.frame_index for f in lerobot_trajectory], + "action": pa.array([f.action for f in lerobot_trajectory], type=pa.list_(pa.float32())) }) - pq.write_table(pa_frames, out_parquet) - print(f"[Episode {episode_index}] Saved parquet => {out_parquet}") + return pa_trajectory - # Copy MP4 to the expected output location - video_outdir = out_dir.parent.parent \ +def save_single_trajectory( + trajectory: pa.Table, + out_dir: Path, + episode_index: int, +) -> None: + """ + Save a single trajectory to a parquet file at + out_dir / data / chunk-000 / f"episode_{episode_index:06d}.parquet" + """ + trajectory_outdir = out_dir \ + / "data" \ + / "chunk-000" \ + / f"episode_{episode_index:06d}.parquet" + pq.write_table(trajectory, trajectory_outdir) + logging.debug( + f"[Episode {episode_index}] Saved parquet => {trajectory_outdir}" + ) + +def save_single_video( + video_path: Path, + out_dir: Path, + episode_index: int, +) -> None: + """ + Copy a single video to the expected output location at + out_dir / "videos" / "chunk-000" / "observation.images.side" + / f"episode_{episode_index:06d}.mp4" + """ + video_outdir = out_dir \ / "videos" \ / "chunk-000" \ / "observation.images.side" video_outdir.mkdir(parents=True, exist_ok=True) episode_mp4 = video_outdir / f"episode_{episode_index:06d}.mp4" - shutil.copy(mp4_file, episode_mp4) - print(f"[Episode {episode_index}] Saved MP4 => {episode_mp4}") + shutil.copy(video_path, episode_mp4) + logging.debug(f"[Episode {episode_index}] Saved MP4 => {episode_mp4}") def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[int]): meta_dir = out_root / "meta" meta_dir.mkdir(exist_ok=True) - # Read trajectory fps, average over all episodes - out_parquet_dir = out_root / "data" / "chunk-000" - parquet_files = os.listdir(out_parquet_dir) - parquet_files = [f for f in parquet_files if f.endswith(".parquet")] - parquet_file_fps = [] - for parquetf in parquet_files: - df = pd.read_parquet(out_parquet_dir / parquetf) - inv_fps = df['timestamp'].diff().mean() - fps = 1.0 / inv_fps - parquet_file_fps.append(fps) - print(f"{fps=}") - mean_trajectory_fps = int(sum(parquet_file_fps) / len(parquet_file_fps)) - # Get total number of video frames video_dir = out_root / "videos" / "chunk-000" / "observation.images.side" video_frame_counts = [] @@ -248,6 +355,7 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ fourcc = int(cap.get(cv2.CAP_PROP_FOURCC)) codec = "".join([chr((fourcc >> 8 * i) & 0xFF) for i in range(4)]) + cap.release() # info.json @@ -260,7 +368,7 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ "total_videos": total_episodes, "total_chunks": 1, "chunks_size": total_episodes, - "fps": mean_trajectory_fps, + "fps": video_fps, "splits": {"train": f"0:{total_episodes}"}, "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet", "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4", @@ -291,10 +399,26 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ json.dump(info_data, f, indent=2) # stats.json + # Collect all actions across episodes into a list + all_actions = [] + for episode_idx in range(total_episodes): + episode_path = out_root \ + / "data" \ + / "chunk-000" \ + / f"episode_{episode_idx:06d}.parquet" + table = pq.read_table(episode_path) + actions = table["action"].to_numpy() + all_actions.extend(actions) + + # Convert to numpy array and compute quantiles along first axis + all_actions = np.array(all_actions) + q01 = np.quantile(all_actions, 0.01, axis=0).tolist() + q99 = np.quantile(all_actions, 0.99, axis=0).tolist() + stats_data = { "action": { - "q01": [-0.03, -0.03, -0.03, -0.03, -0.03, -0.03, -1], - "q99": [0.03, 0.03, 0.03, 0.03, 0.03, 0.03, 1] + "q01": q01, + "q99": q99 } } with open(meta_dir / "stats.json", "w") as f: @@ -302,11 +426,11 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ # episodes.jsonl with open(meta_dir / "episodes.jsonl", "w") as f: - for eidx, frame_count in enumerate(video_frame_counts): + for eidx, length in enumerate(episode_lengths): row = { "episode_index": eidx, "tasks": ["Pick up the object"], - "length": frame_count + "length": length } f.write(json.dumps(row) + "\n") @@ -318,9 +442,12 @@ def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[ } f.write(json.dumps(row) + "\n") -def main(raw_data_prefix: Path = None, out_root: Path = None): - print(f"\nStarting conversion with raw_data_prefix: {raw_data_prefix}") - logging.debug("Starting the conversion process.") # Logging line +def main( + raw_data_prefix: Path = None, + out_root: Path = None, + tolerance_s: float = 1e-5 +): + # Where is your raw data? if raw_data_prefix is None: raw_data_prefix = Path(".") @@ -335,34 +462,96 @@ def main(raw_data_prefix: Path = None, out_root: Path = None): data_out_dir.mkdir(parents=True, exist_ok=True) # Pair up CSV + MP4 - pairs = find_pairs(raw_traj_dir, raw_video_dir) + pairs = pair_trajectories_and_videos(raw_traj_dir, raw_video_dir) # Convert each episode - episode_lengths = [] + lerobot_episode_lengths = [] + lerobot_episode_index = 0 for episode_index, (csv_f, mp4_f) in enumerate(pairs): - logging.debug(f"Processing episode {episode_index} with CSV: {csv_f.name} and MP4: {mp4_f.name}") - convert_single_episode( - csv_file=csv_f, - mp4_file=mp4_f, + logging.debug(( + f"Processing episode {episode_index} " + f"with CSV: {csv_f.name} and MP4: {mp4_f.name}" + )) + lerobot_trajectory: pa.Table = csv_to_lerobot_trajectory( + csv_trajectory_filepath=csv_f, + mp4_filepath=mp4_f, episode_index=episode_index, - out_dir=data_out_dir ) - # Append the length of each episode - episode_length = len(pd.read_csv(csv_f)) - 1 # Decrement by 1 - episode_lengths.append(episode_length) - logging.debug(f"Episode {episode_index} length: {episode_length}") + + # Check that the trajectory satisfies the tolerance. + timestamps = lerobot_trajectory["timestamp"].to_numpy() + diffs = np.diff(timestamps) + fps = cv2.VideoCapture(mp4_f).get(cv2.CAP_PROP_FPS) + within_tolerance = torch.tensor( + np.abs(diffs - 1/fps) <= tolerance_s + ) + if not torch.all(within_tolerance): + # Find indices where tolerance check failed + failed_indices = torch.where(~within_tolerance)[0] + failed_diffs = diffs[failed_indices] + expected_interval = 1/fps + + logging.debug( + f"Episode {episode_index} failed tolerance check and will not be included in the LeRobot dataset.\n" + f"Found {len(failed_indices)} timestamp intervals outside tolerance of {tolerance_s}s:\n" + f"- Expected interval between frames: {expected_interval:.6f}s\n" + f"- Maximum deviation from expected: {np.max(np.abs(failed_diffs - expected_interval)):.6f}s\n" + f"- Maximum allowed deviation: ±{tolerance_s:.6f}s" + ) + continue + + # Save the trajectory and video + save_single_trajectory( + lerobot_trajectory, + out_root, + lerobot_episode_index + ) + save_single_video( + mp4_f, + out_root, + lerobot_episode_index + ) + + # Save the length of the trajectory. + lerobot_episode_length = len(lerobot_trajectory) - 2 + lerobot_episode_lengths.append(lerobot_episode_length) + + logging.debug(( + f"Episode {episode_index} " + f"will be saved in LeRobot dataset as episode {lerobot_episode_index} " + f"trajectory length: {lerobot_episode_length} " + f"number of frames: " + f"{cv2.VideoCapture(mp4_f).get(cv2.CAP_PROP_FRAME_COUNT)}" + )) + + lerobot_episode_index += 1 + + lerobot_episodes_total = len(lerobot_episode_lengths) # Build meta files - build_meta_files(out_root=out_root, total_episodes=len(pairs), episode_lengths=episode_lengths) + build_meta_files( + out_root=out_root, + total_episodes=lerobot_episodes_total, + episode_lengths=lerobot_episode_lengths + ) print("\nDone creating LeRobot-style dataset at:", out_root) if __name__ == "__main__": + import argparse parser = argparse.ArgumentParser() parser.add_argument("--raw_data_prefix", type=Path, default=None, help="Path prefix to raw data directory") parser.add_argument("--out_root", type=Path, default=None, help="Output directory for the dataset") + parser.add_argument("--tolerance_s", type=float, default=1e-5, + help=("Maximum allowed deviation from expected " + "frame interval (in seconds)")) args = parser.parse_args() print("Running with args:", args) - main(raw_data_prefix=args.raw_data_prefix, out_root=args.out_root) \ No newline at end of file + + main( + raw_data_prefix=args.raw_data_prefix, + out_root=args.out_root, + tolerance_s=args.tolerance_s + ) \ No newline at end of file From 93ab2aac880cee0ef2c6930e156cb1b379a54e35 Mon Sep 17 00:00:00 2001 From: mehhl Date: Sat, 28 Dec 2024 22:13:45 +0000 Subject: [PATCH 08/58] remove infinite tolerance from finetune.py --- vla-scripts/finetune.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index eff39a5e4..d60b8b255 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -216,7 +216,6 @@ def finetune(cfg: FinetuneConfig) -> None: else VicunaV15ChatPromptBuilder ), root=f"{cfg.data_root_dir}/{cfg.dataset_name}", - tolerance_s=3.0, image_transforms=None, download_videos=False, local_files_only=True, From f12d3276f37f8ae053f41bc9f296ba39b8452eb9 Mon Sep 17 00:00:00 2001 From: mehhl Date: Sat, 28 Dec 2024 22:25:24 +0000 Subject: [PATCH 09/58] fixed ep length calculation --- scripts/additional-datasets/nomagic_ur5e_raw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index 0da346c4d..87dad0234 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -513,7 +513,7 @@ def main( ) # Save the length of the trajectory. - lerobot_episode_length = len(lerobot_trajectory) - 2 + lerobot_episode_length = len(lerobot_trajectory) lerobot_episode_lengths.append(lerobot_episode_length) logging.debug(( From 180b9e403eb98e7c3a367257256497d01c2f12e0 Mon Sep 17 00:00:00 2001 From: mehhl Date: Sat, 28 Dec 2024 22:56:44 +0000 Subject: [PATCH 10/58] added video clipping to match trajs --- .../additional-datasets/nomagic_ur5e_raw.py | 77 +++++++++++-------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index 87dad0234..8736d013d 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -187,36 +187,17 @@ def make_video_timestamps(mp4_filepath: str, frames) -> list[int]: ] return video_timestamps -def select_ts_closest_to_reference( - csv_trajectory_timestamps: pd.Series, - video_timestamps: list[int] -) -> list[int]: - trajectory_timestamps = [] - for video_ts in video_timestamps: - ts_not_in_csv_trajectory_range = ( - video_ts < csv_trajectory_timestamps.min() or - video_ts > csv_trajectory_timestamps.max() - ) - if ts_not_in_csv_trajectory_range: - continue - try: - closest_csv_trajectory_ts = csv_trajectory_timestamps.iloc[ - csv_trajectory_timestamps.searchsorted(video_ts) - ] - trajectory_timestamps.append(closest_csv_trajectory_ts) - except IndexError: - continue - return trajectory_timestamps - def csv_to_lerobot_trajectory( csv_trajectory_filepath: Path, mp4_filepath: Path, episode_index: int, time_delta: int = 50, -) -> pa.Table: +) -> tuple[pa.Table, np.ndarray]: """ Convert one CSV + MP4 into a single "episode_{:06d}.parquet" and copy the MP4 to "episode_{:06d}.mp4" in observation.images.side subdir. + + Also save a video containing only those frames that were matched. """ # Load CSV data @@ -234,14 +215,15 @@ def csv_to_lerobot_trajectory( frames = get_frames(mp4_filepath) video_timestamps = make_video_timestamps(mp4_filepath, frames) - lerobot_trajectory: List[LeRobotTrajectoryStep] = [] + video_timestamp_is_matched = [None] * len(video_timestamps) for frame_idx, video_ts in enumerate(video_timestamps): video_ts_not_in_csv_trajectory_range = ( video_ts < csv_trajectory_df["seconds"].min() or video_ts > csv_trajectory_df["seconds"].max() ) if video_ts_not_in_csv_trajectory_range: + video_timestamp_is_matched[frame_idx] = False continue try: rowA = csv_trajectory_df.iloc[ @@ -251,7 +233,9 @@ def csv_to_lerobot_trajectory( csv_trajectory_df["seconds"].searchsorted(video_ts + time_delta) ] except IndexError: + video_timestamp_is_matched[frame_idx] = False continue + video_timestamp_is_matched[frame_idx] = True # Build an action action_vals = compute_actions_from_rows(rowA, rowB) @@ -273,6 +257,7 @@ def csv_to_lerobot_trajectory( action=action_vals ) ) + video_timestamp_is_matched[frame_idx] = True # Write out as a parquet file pa_trajectory = pa.Table.from_pydict({ @@ -284,7 +269,15 @@ def csv_to_lerobot_trajectory( "frame_index": [f.frame_index for f in lerobot_trajectory], "action": pa.array([f.action for f in lerobot_trajectory], type=pa.list_(pa.float32())) }) - return pa_trajectory + # Build a video containing only the matched frames + matched_frames = np.array([ + frame + for frame, is_matched in zip(frames, video_timestamp_is_matched) + if is_matched + ]) + matched_video = matched_frames.astype(np.uint8) + + return pa_trajectory, matched_video def save_single_trajectory( trajectory: pa.Table, @@ -305,12 +298,13 @@ def save_single_trajectory( ) def save_single_video( - video_path: Path, + video: np.ndarray, out_dir: Path, episode_index: int, ) -> None: """ - Copy a single video to the expected output location at + Save a single video, given a numpy array of frames (dtype=uint8), + to the expected output location at out_dir / "videos" / "chunk-000" / "observation.images.side" / f"episode_{episode_index:06d}.mp4" """ @@ -320,7 +314,21 @@ def save_single_video( / "observation.images.side" video_outdir.mkdir(parents=True, exist_ok=True) episode_mp4 = video_outdir / f"episode_{episode_index:06d}.mp4" - shutil.copy(video_path, episode_mp4) + + if len(video) == 0: + logging.debug(f"[Episode {episode_index}] No frames to save for MP4 => {episode_mp4}") + return + + # Assume video shape is (num_frames, height, width, channels) + height, width, channels = video[0].shape + fps = 30.0 # or retrieve from context if available + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + writer = cv2.VideoWriter(str(episode_mp4), fourcc, fps, (width, height)) + + for frame in video: + writer.write(frame) + + writer.release() logging.debug(f"[Episode {episode_index}] Saved MP4 => {episode_mp4}") def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[int]): @@ -472,7 +480,7 @@ def main( f"Processing episode {episode_index} " f"with CSV: {csv_f.name} and MP4: {mp4_f.name}" )) - lerobot_trajectory: pa.Table = csv_to_lerobot_trajectory( + lerobot_trajectory, matched_video = csv_to_lerobot_trajectory( csv_trajectory_filepath=csv_f, mp4_filepath=mp4_f, episode_index=episode_index, @@ -507,7 +515,7 @@ def main( lerobot_episode_index ) save_single_video( - mp4_f, + matched_video, out_root, lerobot_episode_index ) @@ -515,13 +523,18 @@ def main( # Save the length of the trajectory. lerobot_episode_length = len(lerobot_trajectory) lerobot_episode_lengths.append(lerobot_episode_length) - + lerobot_frame_count = cv2.VideoCapture( # Read it back from the saved video + out_root \ + / "videos" \ + / "chunk-000" \ + / "observation.images.side" \ + / f"episode_{lerobot_episode_index:06d}.mp4" + ).get(cv2.CAP_PROP_FRAME_COUNT) logging.debug(( f"Episode {episode_index} " f"will be saved in LeRobot dataset as episode {lerobot_episode_index} " f"trajectory length: {lerobot_episode_length} " - f"number of frames: " - f"{cv2.VideoCapture(mp4_f).get(cv2.CAP_PROP_FRAME_COUNT)}" + f"number of frames: {int(lerobot_frame_count)}" )) lerobot_episode_index += 1 From 8cb586d926001efe4ac305d93ef03f7b5579b8c1 Mon Sep 17 00:00:00 2001 From: mehhl Date: Sat, 28 Dec 2024 23:18:42 +0000 Subject: [PATCH 11/58] fix no frame match for last timestep issue --- scripts/additional-datasets/nomagic_ur5e_raw.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index 8736d013d..ad0938158 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -233,7 +233,10 @@ def csv_to_lerobot_trajectory( csv_trajectory_df["seconds"].searchsorted(video_ts + time_delta) ] except IndexError: - video_timestamp_is_matched[frame_idx] = False + # TODO: this is false but we need the last timestep + # TODO: to match to some video closer than tolerance + # TODO: maybe we should mark (frame_idx + 1) as True instead? + video_timestamp_is_matched[frame_idx] = True continue video_timestamp_is_matched[frame_idx] = True @@ -269,6 +272,7 @@ def csv_to_lerobot_trajectory( "frame_index": [f.frame_index for f in lerobot_trajectory], "action": pa.array([f.action for f in lerobot_trajectory], type=pa.list_(pa.float32())) }) + # Build a video containing only the matched frames matched_frames = np.array([ frame @@ -301,6 +305,7 @@ def save_single_video( video: np.ndarray, out_dir: Path, episode_index: int, + fps: float, ) -> None: """ Save a single video, given a numpy array of frames (dtype=uint8), @@ -321,7 +326,6 @@ def save_single_video( # Assume video shape is (num_frames, height, width, channels) height, width, channels = video[0].shape - fps = 30.0 # or retrieve from context if available fourcc = cv2.VideoWriter_fourcc(*'mp4v') writer = cv2.VideoWriter(str(episode_mp4), fourcc, fps, (width, height)) @@ -517,7 +521,8 @@ def main( save_single_video( matched_video, out_root, - lerobot_episode_index + lerobot_episode_index, + fps=cv2.VideoCapture(mp4_f).get(cv2.CAP_PROP_FPS) ) # Save the length of the trajectory. From c718bc75e2905c49a08c7cca264ac388a12fb624 Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 30 Dec 2024 00:32:33 +0000 Subject: [PATCH 12/58] added action normalization and fixed __len__ in OpenVLALeRobotDataset --- prismatic/vla/datasets/datasets.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/prismatic/vla/datasets/datasets.py b/prismatic/vla/datasets/datasets.py index 9ddd403da..f1dc66e36 100644 --- a/prismatic/vla/datasets/datasets.py +++ b/prismatic/vla/datasets/datasets.py @@ -227,8 +227,8 @@ def __init__( self.dataset_statistics = { "openvla_lerobot_dataset": { "action": { - "q01": self.meta.stats["action"]["q01"].tolist(), - "q99": self.meta.stats["action"]["q99"].tolist(), + "q01": np.array(self.meta.stats["action"]["q01"]), + "q99": np.array(self.meta.stats["action"]["q99"]), } } } @@ -248,7 +248,7 @@ def __init__( def __len__(self): - return self.meta.info['total_episodes'] + return self.num_frames # Retrieves a single (instruction, image, action) triple from the dataset. def __getitem__(self, idx): @@ -270,6 +270,9 @@ def __getitem__(self, idx): # Retrieve action. action: torch.Tensor = hf_item["action"] + q01 = np.array(self.dataset_statistics["openvla_lerobot_dataset"]["action"]["q01"]) + q99 = np.array(self.dataset_statistics["openvla_lerobot_dataset"]["action"]["q99"]) + action = (2*action - q01 - q99) / (q99 - q01) # normalize to [-1, 1] action: str = self.action_tokenizer(action) # Add instruction to VLA prompt. From 12eae3b39086b49428ec10d02b119dfb9dee1564 Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 30 Dec 2024 00:33:23 +0000 Subject: [PATCH 13/58] correct episode index writing into trajectory file --- scripts/additional-datasets/nomagic_ur5e_raw.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index ad0938158..20ab51d8c 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -191,6 +191,7 @@ def csv_to_lerobot_trajectory( csv_trajectory_filepath: Path, mp4_filepath: Path, episode_index: int, + maybe_lerobot_episode_index: int, time_delta: int = 50, ) -> tuple[pa.Table, np.ndarray]: """ @@ -252,7 +253,7 @@ def csv_to_lerobot_trajectory( lerobot_trajectory.append( LeRobotTrajectoryStep( timestamp=ts_val, - episode_index=episode_index, + episode_index=maybe_lerobot_episode_index, next_done=next_done, task_index=0, index=frame_idx, @@ -488,6 +489,7 @@ def main( csv_trajectory_filepath=csv_f, mp4_filepath=mp4_f, episode_index=episode_index, + maybe_lerobot_episode_index=lerobot_episode_index, ) # Check that the trajectory satisfies the tolerance. From e896cb2e5a3263065a628fc567815a23257aa650 Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 30 Dec 2024 00:33:48 +0000 Subject: [PATCH 14/58] Numerous fixes --- vla-scripts/finetune.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index d60b8b255..0779992a4 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -111,9 +111,13 @@ class FinetuneConfig: wandb_entity: str = "stanford-voltron" # Name of entity to log under run_id_note: Optional[str] = None # Extra note for logging, Weights & Biases + # Parameter for LeRobotDataset + tolerance_s: float = 0.15 + # fmt: on + @draccus.wrap() def finetune(cfg: FinetuneConfig) -> None: print(f"Fine-tuning OpenVLA Model `{cfg.vla_path}` on `{cfg.dataset_name}`") @@ -216,6 +220,7 @@ def finetune(cfg: FinetuneConfig) -> None: else VicunaV15ChatPromptBuilder ), root=f"{cfg.data_root_dir}/{cfg.dataset_name}", + tolerance_s=cfg.tolerance_s, image_transforms=None, download_videos=False, local_files_only=True, @@ -245,10 +250,13 @@ def finetune(cfg: FinetuneConfig) -> None: collator = PaddedCollatorForActionPrediction( processor.tokenizer.model_max_length, processor.tokenizer.pad_token_id, padding_side="right" ) + sampler = RandomSampler( + vla_dataset, num_samples=vla_dataset.num_frames + ) dataloader = DataLoader( vla_dataset, batch_size=cfg.batch_size, - sampler=RandomSampler(vla_dataset), + sampler=sampler, collate_fn=collator, num_workers=0, # Set to 0 bc we don't use parallelism # TODO: figure out if this is right? @@ -327,6 +335,9 @@ def finetune(cfg: FinetuneConfig) -> None: # Compute gradient step index gradient_step_idx = batch_idx // cfg.grad_accumulation_steps + print(f"{batch_idx=}") + print(f"{cfg.grad_accumulation_steps=}") + print(f"{gradient_step_idx=}") # Compute smoothened train metrics # =>> Equal to current step metrics when not using gradient accumulation @@ -336,7 +347,7 @@ def finetune(cfg: FinetuneConfig) -> None: smoothened_l1_loss = sum(recent_l1_losses) / len(recent_l1_losses) # Push Metrics to W&B (every 10 gradient steps) - if distributed_state.is_main_process and gradient_step_idx % 10 == 0: + if distributed_state.is_main_process and gradient_step_idx % 1 == 0: print( { "train_loss": smoothened_loss, From 3579dd3c97e8e12df09ac82c1ff9a89a10e8dbd9 Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 30 Dec 2024 00:36:37 +0000 Subject: [PATCH 15/58] added fientune.py outdirs to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6160ebcb1..8b76ce8a5 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,5 @@ data/ # Rollout videos and wandb logs rollouts/ wandb/ +.tmp +.runs From 2966fdbdaad42692d081eb6a1e4e02cc8930ca71 Mon Sep 17 00:00:00 2001 From: mehhl Date: Tue, 31 Dec 2024 04:37:59 +0000 Subject: [PATCH 16/58] fixes to loop design in finetune.py --- vla-scripts/finetune.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 0779992a4..c20db0693 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -20,6 +20,7 @@ """ import os +import json from collections import deque from dataclasses import dataclass from pathlib import Path @@ -288,6 +289,7 @@ def finetune(cfg: FinetuneConfig) -> None: min_epochs = (cfg.max_steps * cfg.grad_accumulation_steps) // steps_per_epoch + 1 num_epochs = min_epochs + total_optimizer_steps = 0 for _ in range(num_epochs): for batch_idx, batch in enumerate(dataloader): with torch.autocast("cuda", dtype=torch.bfloat16): @@ -346,16 +348,6 @@ def finetune(cfg: FinetuneConfig) -> None: smoothened_action_accuracy = sum(recent_action_accuracies) / len(recent_action_accuracies) smoothened_l1_loss = sum(recent_l1_losses) / len(recent_l1_losses) - # Push Metrics to W&B (every 10 gradient steps) - if distributed_state.is_main_process and gradient_step_idx % 1 == 0: - print( - { - "train_loss": smoothened_loss, - "action_accuracy": smoothened_action_accuracy, - "l1_loss": smoothened_l1_loss, - }, - ) - # Optimizer Step if ( (batch_idx + 1) % cfg.grad_accumulation_steps == 0 @@ -364,9 +356,18 @@ def finetune(cfg: FinetuneConfig) -> None: optimizer.step() optimizer.zero_grad() progress.update() + total_optimizer_steps += 1 + print( + { + "total_optimizer_steps": total_optimizer_steps, + "train_loss": smoothened_loss, + "action_accuracy": smoothened_action_accuracy, + "l1_loss": smoothened_l1_loss, + }, + ) # Save Model Checkpoint =>> by default, only keeps the latest checkpoint, continually overwriting it! - if gradient_step_idx > 0 and gradient_step_idx % cfg.save_steps == 0: + if total_optimizer_steps > 0 and total_optimizer_steps % cfg.save_steps == 0: if distributed_state.is_main_process: print(f"Saving Model Checkpoint for Step {gradient_step_idx}") @@ -396,12 +397,21 @@ def finetune(cfg: FinetuneConfig) -> None: print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {run_dir}") else: # Prepare to save checkpoint in new directory - checkpoint_dir = Path(str(run_dir) + f"--{gradient_step_idx}_chkpt") + checkpoint_dir = Path(str(run_dir) + f"--{total_optimizer_steps}_chkpt") os.makedirs(checkpoint_dir, exist_ok=True) # Save dataset statistics to new directory save_dataset_statistics(vla_dataset.dataset_statistics, checkpoint_dir) + # Save training statistics to new directory + with open(checkpoint_dir / "training_stats.json", "w") as f: + json.dump({ + "total_optimizer_steps": total_optimizer_steps, + "train_loss": smoothened_loss, + "action_accuracy": smoothened_action_accuracy, + "l1_loss": smoothened_l1_loss, + }, f, indent=4) + # Save processor and model weights to new directory processor.save_pretrained(checkpoint_dir) merged_vla.save_pretrained(checkpoint_dir) @@ -412,11 +422,11 @@ def finetune(cfg: FinetuneConfig) -> None: dist.barrier() # Stop training when max_steps is reached - if gradient_step_idx == cfg.max_steps: + if total_optimizer_steps == cfg.max_steps: print(f"Max step {cfg.max_steps} reached! Stopping training...") break - if gradient_step_idx == cfg.max_steps: + if total_optimizer_steps == cfg.max_steps: break From 50bfede09440f19d5ea677f4b42406255717fad1 Mon Sep 17 00:00:00 2001 From: mehhl Date: Fri, 17 Jan 2025 21:39:49 +0000 Subject: [PATCH 17/58] numerous fixes in conversion to lerobot --- .../additional-datasets/nomagic_ur5e_raw.py | 322 +++++++++++------- 1 file changed, 207 insertions(+), 115 deletions(-) diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py index 20ab51d8c..22626f4f2 100644 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ b/scripts/additional-datasets/nomagic_ur5e_raw.py @@ -50,6 +50,7 @@ import pyarrow as pa import pyarrow.parquet as pq import cv2 +from typing import Optional from pathlib import Path from typing import List, Dict @@ -121,7 +122,6 @@ def pair_trajectories_and_videos( for csv_trajectory, mp4_video in trajectory_video_pairs: logging.debug(f"Paired {csv_trajectory.name} with {mp4_video.name}") - return trajectory_video_pairs def compute_actions_from_rows(rowA: pd.Series, rowB: pd.Series): @@ -140,59 +140,94 @@ def compute_actions_from_rows(rowA: pd.Series, rowB: pd.Series): q2 = [rowB["OrientationX"], rowB["OrientationY"], rowB["OrientationZ"], rowB["OrientationW"]] - # Calculate orientation difference using quaternion_difference logic - r1 = R.from_quat(q1) - r2 = R.from_quat(q2) + # Calculate orientation difference + r1 = R.from_rotvec( + [ + rowA["OrientationX"], + rowA["OrientationY"], + rowA["OrientationZ"] + ] + ) + r2 = R.from_rotvec( + [ + rowB["OrientationX"], + rowB["OrientationY"], + rowB["OrientationZ"] + ] + ) r_diff = r2 * r1.inv() euler_diff = r_diff.as_euler("xyz") dox, doy, doz = euler_diff # Gripper - map Gripper::Action values to float - grip_action_map = { - "Gripper::Action::NONE": 0.0, - "Gripper::Action::RELEASE": -1.0, - "Gripper::Action::GRAB": 1.0 - } - if rowA["GripperAction"] not in grip_action_map: - raise ValueError(f"Unknown gripper action: {rowA['GripperAction']}") - grip_val = grip_action_map[rowA["GripperAction"]] + grip_val = rowA["GripperAction"] return [dx, dy, dz, dox, doy, doz, grip_val] -def get_frames(video_path: str) -> list: - vidcap = cv2.VideoCapture(video_path) - success, image = vidcap.read() - frames = [] - while success: - frames.append(image) - success, image = vidcap.read() - return frames - -def make_video_timestamps(mp4_filepath: str, frames) -> list[int]: +def read_frame_list_from_path( + video_path: Path | str +) -> list[cv2.typing.MatLike]: + """ + Given a path to an MP4 file, read all frames from the file into a list. + """ + cap = cv2.VideoCapture(str(video_path)) + frame_list = list() + while (ret := cap.read())[0]: # ret[0] is success flag, ret[1] is the frame + frame_list.append(ret[1]) + return frame_list + +def make_video_timestamps( + mp4_filepath: Path | str, + frame_list: list[cv2.typing.MatLike], +) -> list[int]: + """ + Given a path to an MP4 file and a list of frames from that file, + use ExifTool to read a timestamp of as many frames as possible from the + video metadata, then interpolate evenly to remaining frames. + """ + # Read timestamps from metadata. Result is given in Unix nanoseconds. with exiftool.ExifToolHelper() as et: - metadata = et.get_metadata(mp4_filepath) - video_timestamps = [int(i) for i in metadata[0]["XMP:Timestamps"]] - # ExifTool doesn't always return all frames, so we need to estimate - # TODO: can we just use cv2 to get fps and divide length by that? - avg_frame = \ - int((video_timestamps[-1] - video_timestamps[0]) \ - / (len(video_timestamps) - 1)) - while len(video_timestamps) < len(frames): - video_timestamps.append(video_timestamps[-1] + avg_frame) - # Convert to Unix seconds - nanoseconds_to_miliseconds = 1_000_000 - video_timestamps = [ - int(i / nanoseconds_to_miliseconds) - for i in video_timestamps + mp4_metadata = et.get_metadata(str(mp4_filepath)) + timestamp_list: list[int] = [ + int(t) # From str + for t in mp4_metadata[0]["XMP:Timestamps"] + ] + + # Emit a warning if not all frames are retrieved from the metadata. + if len(timestamp_list) < len(frame_list): + logging.warning( + f"ExifTool found {len(timestamp_list)} timestamps " + f"for {len(frame_list)} frames of video at {mp4_filepath}. " + "Will append interpolated timestamps to match frame count." + ) + + # Interpolate timestamps. + fps = mp4_metadata[0]["QuickTime:VideoFrameRate"] + delta_t = int((1 / fps) * 1e9) # In nanoseconds + while len(timestamp_list) < len(frame_list): + timestamp_list.append(timestamp_list[-1] + delta_t) + + # Convert timestamps to Unix miliseconds. + timestamp_list = [ + int(t / 1e6) # In miliseconds + for t in timestamp_list ] - return video_timestamps + + # Check max deviation of frame timestamp deltas from period. + timestamp_array = np.array(timestamp_list, dtype=np.float64) / 1e3 + diffs = np.diff(timestamp_array) + logging.debug( + "Max deviation of frame timestamp deltas from period " + f"set by {fps=} is {np.max(np.abs(diffs - (1/fps))):.6f}s" + ) + + return timestamp_list def csv_to_lerobot_trajectory( csv_trajectory_filepath: Path, mp4_filepath: Path, - episode_index: int, maybe_lerobot_episode_index: int, - time_delta: int = 50, + tolerance_s: float = 1e-5, ) -> tuple[pa.Table, np.ndarray]: """ Convert one CSV + MP4 into a single "episode_{:06d}.parquet" and @@ -201,69 +236,125 @@ def csv_to_lerobot_trajectory( Also save a video containing only those frames that were matched. """ - # Load CSV data + # Load CSV data. csv_trajectory_df = pd.read_csv(csv_trajectory_filepath) - # Get CSV trajectory timesteps in in Unix seconds - csv_trajectory_df["seconds"] = ( + # --- Preprocess the trajectory data. --- + # Drop non-synchronized rows entirely. "Synchronized" is a boolean column + # intended to show if ViperLink was connected to UR5e at timestamp. + sync_mask = csv_trajectory_df["IsSynchronized"] == 1 + csv_trajectory_df = csv_trajectory_df[sync_mask] + + # Append binary grip action to each row. + # In ViperLink, `Gripper::Action::NONE` has the effect of taking previous + # action (or `RELEASE` at trajectory start). We preserve this behavior. + current_grip_action = -1.0 # Initial action is -1.0 (RELEASE) + for idx, row in csv_trajectory_df.iterrows(): + if row["GripperAction"] == "Gripper::Action::RELEASE": + current_grip_action = -1.0 + elif row["GripperAction"] == "Gripper::Action::GRAB": + current_grip_action = 1.0 + csv_trajectory_df.at[idx, "GripperAction"] = current_grip_action + + # Append timestamp in Unix miliseconds to each CSV row. + csv_trajectory_df["miliseconds"] = ( pd .to_datetime(csv_trajectory_df["Timestamp"]) - .add(pd.Timedelta(hours=-1)) - .apply(lambda x: x.timestamp() * 1000) + .add(pd.Timedelta(hours=-1)) # UTC-1 + .apply(lambda x: x.timestamp() * 1e3) # Miliseconds .astype(int) ) - # Get video timestamps in Unix miliseconds - frames = get_frames(mp4_filepath) - video_timestamps = make_video_timestamps(mp4_filepath, frames) - - lerobot_trajectory: List[LeRobotTrajectoryStep] = [] - video_timestamp_is_matched = [None] * len(video_timestamps) - for frame_idx, video_ts in enumerate(video_timestamps): - video_ts_not_in_csv_trajectory_range = ( - video_ts < csv_trajectory_df["seconds"].min() or - video_ts > csv_trajectory_df["seconds"].max() - ) - if video_ts_not_in_csv_trajectory_range: - video_timestamp_is_matched[frame_idx] = False - continue + + # --- Preprocess the video. --- + # Read video timestamp. + mp4_frame_list = read_frame_list_from_path(mp4_filepath) + mp4_timestamp_list: list[int] = make_video_timestamps( + mp4_filepath=mp4_filepath, # Should be seconds + frame_list=mp4_frame_list + ) + + # --- Match trajectory steps to video frames. --- + row_ts_begin = csv_trajectory_df["miliseconds"].min() + row_ts_end = csv_trajectory_df["miliseconds"].max() + matched_rows_by_frame: list[Optional[pd.Series]] \ + = [None] * len(mp4_timestamp_list) + tolerance_unix_ms = tolerance_s * 1e3 + for frame_idx, frame_ts in enumerate(mp4_timestamp_list): + if ( + frame_ts < row_ts_begin - tolerance_unix_ms or + frame_ts > row_ts_end + tolerance_unix_ms + ): + continue # Unmatchable within tolerance + + # Find a match by binary search on the trajectory timestamps. try: - rowA = csv_trajectory_df.iloc[ - csv_trajectory_df["seconds"].searchsorted(video_ts) - ] - rowB = csv_trajectory_df.iloc[ - csv_trajectory_df["seconds"].searchsorted(video_ts + time_delta) + matching_row = csv_trajectory_df.iloc[ + csv_trajectory_df["miliseconds"].searchsorted(frame_ts) ] - except IndexError: - # TODO: this is false but we need the last timestep - # TODO: to match to some video closer than tolerance - # TODO: maybe we should mark (frame_idx + 1) as True instead? - video_timestamp_is_matched[frame_idx] = True - continue - video_timestamp_is_matched[frame_idx] = True - - # Build an action - action_vals = compute_actions_from_rows(rowA, rowB) - # Use the rowA's timestamp (relative seconds from start of episode) - # divide by 1000 to convert from miliseconds to seconds - ts_val = float(rowA["seconds"] - csv_trajectory_df["seconds"].min()) / 1000 - # next.done is usually False unless e.g. i == len(df) - 2 - next_done = frame_idx == (len(video_timestamps) - 2) - - # Build the frame record - lerobot_trajectory.append( - LeRobotTrajectoryStep( - timestamp=ts_val, - episode_index=maybe_lerobot_episode_index, - next_done=next_done, - task_index=0, - index=frame_idx, - frame_index=frame_idx, - action=action_vals + except IndexError: # Likely means frame is outside trajectory, + continue # but within tolerance. Ignore it for now. + + # If the match is further than the tolerance, don't include it. + if ( + frame_ts < matching_row["miliseconds"] - tolerance_unix_ms or + frame_ts > matching_row["miliseconds"] + tolerance_unix_ms + ): + logging.warning( + f"Matching candidate timestep further " + f"({np.abs(frame_ts - matching_row['miliseconds']):.8f}ms) from frame than " + f"allowed by tolerance of {tolerance_unix_ms}ms. Won't match this frame." ) + continue # Unmatchable within tolerance + + # Save the match. + matched_rows_by_frame[frame_idx] = matching_row + + # Verify that matched rows form a single, continuous trajectory. + switch_count = int(matched_rows_by_frame[0] != None) + for row_idx, row in enumerate(matched_rows_by_frame[:-1]): + next_row = matched_rows_by_frame[row_idx + 1] + if type(row) != type(next_row): + switch_count += 1 + if switch_count > 2: + raise ValueError( + "Trajectory data forms multiple trajectories when " + "matched against video." + ) + + # --- Build the trajectory. --- + trajectory = [ + (row_frame_pair_idx, row) + for row_frame_pair_idx, row + in enumerate(matched_rows_by_frame) + if row is not None # If frame[frame_idx] matches some row + ] + # This is the timestamp of the initial frame in the trajectory. + # It is (brittly) guaranteed to be >=0, since we match steps to + # frames by binsearch on step timestamps, and skip index errors. + # Since the timestep of the first frame is zeroed out by LeRobotDataset + # initializer, we take it as the zero-point relative to step timestamps, + # i.e. saved timestamp is `timestamp_isn_unix_ms - trajectory_ts_begin`. + trajectory_init_frame_idx = trajectory[0][0] + trajectory_ts_begin = mp4_timestamp_list[trajectory_init_frame_idx] + lerobot_trajectory: List[LeRobotTrajectoryStep] = [ + LeRobotTrajectoryStep( + timestamp=float(row["miliseconds"] - trajectory_ts_begin) / 1e3, + episode_index=maybe_lerobot_episode_index, # ^ seconds + next_done=next_row is None or next_row_idx == len(trajectory) - 1, # TODO: Not ideal + task_index=0, # TODO: Assumes (incorrectly) only single task in data + index=step_idx, + frame_index=step_idx, + action=compute_actions_from_rows(row, next_row) ) - video_timestamp_is_matched[frame_idx] = True + for step_idx, ((_, row), (next_row_idx, next_row)) + in enumerate(zip(trajectory[:-1], trajectory[1:])) + ] + matched_video = np.array([ + mp4_frame_list[frame_idx] + for frame_idx, _ in trajectory[:-1] + ], dtype=np.uint8) - # Write out as a parquet file + # Write trajectory out as a parquet. pa_trajectory = pa.Table.from_pydict({ "timestamp": [f.timestamp for f in lerobot_trajectory], "episode_index": [f.episode_index for f in lerobot_trajectory], @@ -274,14 +365,6 @@ def csv_to_lerobot_trajectory( "action": pa.array([f.action for f in lerobot_trajectory], type=pa.list_(pa.float32())) }) - # Build a video containing only the matched frames - matched_frames = np.array([ - frame - for frame, is_matched in zip(frames, video_timestamp_is_matched) - if is_matched - ]) - matched_video = matched_frames.astype(np.uint8) - return pa_trajectory, matched_video def save_single_trajectory( @@ -307,7 +390,7 @@ def save_single_video( out_dir: Path, episode_index: int, fps: float, -) -> None: +) -> Path: """ Save a single video, given a numpy array of frames (dtype=uint8), to the expected output location at @@ -335,6 +418,7 @@ def save_single_video( writer.release() logging.debug(f"[Episode {episode_index}] Saved MP4 => {episode_mp4}") + return episode_mp4 def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[int]): meta_dir = out_root / "meta" @@ -464,23 +548,36 @@ def main( # Where is your raw data? if raw_data_prefix is None: raw_data_prefix = Path(".") - raw_traj_dir = raw_data_prefix / "raw/trajectories" - raw_video_dir = raw_data_prefix / "raw/videos" - print(f"Looking for data in:\n {raw_traj_dir}\n {raw_video_dir}") + raw_traj_dir = raw_data_prefix / "trajectories" + print(f"Looking for data in:\n {raw_traj_dir}") # Where do you want the new dataset to live? if out_root is None: - out_root = raw_data_prefix / "data/my_lerobot_dataset" + raise ValueError("give a outfile location") data_out_dir = out_root / "data" / "chunk-000" data_out_dir.mkdir(parents=True, exist_ok=True) # Pair up CSV + MP4 - pairs = pair_trajectories_and_videos(raw_traj_dir, raw_video_dir) + pairs: list[tuple[Path, Path]] = [] + for ep_idx in os.listdir(raw_traj_dir): + ep_files = os.listdir(raw_traj_dir / ep_idx) + csv_file = [ + ep_file + for ep_file in ep_files + if ep_file.endswith(".csv") + ][0] + mp4_file = [ + ep_file + for ep_file in ep_files + if ("side_view" in ep_file) and ep_file.endswith(".mp4") + ][0] + pairs.append((raw_traj_dir / ep_idx / csv_file, raw_traj_dir / ep_idx / mp4_file)) # Convert each episode lerobot_episode_lengths = [] lerobot_episode_index = 0 for episode_index, (csv_f, mp4_f) in enumerate(pairs): + print("\n") logging.debug(( f"Processing episode {episode_index} " f"with CSV: {csv_f.name} and MP4: {mp4_f.name}" @@ -488,8 +585,8 @@ def main( lerobot_trajectory, matched_video = csv_to_lerobot_trajectory( csv_trajectory_filepath=csv_f, mp4_filepath=mp4_f, - episode_index=episode_index, maybe_lerobot_episode_index=lerobot_episode_index, + tolerance_s=tolerance_s, ) # Check that the trajectory satisfies the tolerance. @@ -509,6 +606,7 @@ def main( f"Episode {episode_index} failed tolerance check and will not be included in the LeRobot dataset.\n" f"Found {len(failed_indices)} timestamp intervals outside tolerance of {tolerance_s}s:\n" f"- Expected interval between frames: {expected_interval:.6f}s\n" + f"- Number of failed indices: {failed_indices.shape[0]}\n" f"- Maximum deviation from expected: {np.max(np.abs(failed_diffs - expected_interval)):.6f}s\n" f"- Maximum allowed deviation: ±{tolerance_s:.6f}s" ) @@ -520,7 +618,7 @@ def main( out_root, lerobot_episode_index ) - save_single_video( + video_path = save_single_video( matched_video, out_root, lerobot_episode_index, @@ -530,16 +628,10 @@ def main( # Save the length of the trajectory. lerobot_episode_length = len(lerobot_trajectory) lerobot_episode_lengths.append(lerobot_episode_length) - lerobot_frame_count = cv2.VideoCapture( # Read it back from the saved video - out_root \ - / "videos" \ - / "chunk-000" \ - / "observation.images.side" \ - / f"episode_{lerobot_episode_index:06d}.mp4" - ).get(cv2.CAP_PROP_FRAME_COUNT) + lerobot_frame_count = cv2.VideoCapture(video_path).get(cv2.CAP_PROP_FRAME_COUNT) logging.debug(( f"Episode {episode_index} " - f"will be saved in LeRobot dataset as episode {lerobot_episode_index} " + f"will be saved in LeRobot dataset as episode {lerobot_episode_index}.\n " f"trajectory length: {lerobot_episode_length} " f"number of frames: {int(lerobot_frame_count)}" )) From 0ca0e835fc9326411dac99615318eb5660f56ff3 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Tue, 11 Feb 2025 14:29:26 +0000 Subject: [PATCH 18/58] updated finetune.py --- vla-scripts/finetune.py | 94 +++++++++++++++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 13 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index c20db0693..7e16110e6 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -270,7 +270,15 @@ def finetune(cfg: FinetuneConfig) -> None: # Deque to store recent train metrics (used for computing smoothened metrics for gradient accumulation) recent_losses = deque(maxlen=cfg.grad_accumulation_steps) recent_action_accuracies = deque(maxlen=cfg.grad_accumulation_steps) + recent_action_accuracies_components = { + action_dim_name: deque(maxlen=cfg.grad_accumulation_steps) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + } recent_l1_losses = deque(maxlen=cfg.grad_accumulation_steps) + recent_l1_loss_components = { + action_dim_name: deque(maxlen=cfg.grad_accumulation_steps) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + } # Calculate number of epochs steps_per_epoch = len(dataloader) # number of batches per epoch @@ -320,8 +328,18 @@ def finetune(cfg: FinetuneConfig) -> None: # Compute Accuracy correct_preds = (action_preds == action_gt) & mask action_accuracy = correct_preds.sum().float() / mask.sum().float() + action_accuracy_components = { + action_dim_name: ( + ( + (action_preds[:, i::7] == action_gt[:, i::7]) & mask[:, i::7] + ).sum().float() + / mask[:, i::7].sum().float() + ) + for i, action_dim_name in enumerate(['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip']) + } # Compute L1 Loss on Predicted (Continuous) Actions + continuous_actions_pred = torch.tensor( action_tokenizer.decode_token_ids_to_actions(action_preds[mask].cpu().numpy()) ) @@ -329,24 +347,45 @@ def finetune(cfg: FinetuneConfig) -> None: action_tokenizer.decode_token_ids_to_actions(action_gt[mask].cpu().numpy()) ) action_l1_loss = torch.nn.functional.l1_loss(continuous_actions_pred, continuous_actions_gt) + action_l1_loss_components = { + action_dim_name: torch.nn.functional.l1_loss( + continuous_actions_pred[i::7], + continuous_actions_gt[i::7] + ) + for i, action_dim_name in enumerate(['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip']) + } # Store recent train metrics recent_losses.append(loss.item()) recent_action_accuracies.append(action_accuracy.item()) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip']: + recent_action_accuracies_components[action_dim_name].append(action_accuracy_components[action_dim_name].item()) recent_l1_losses.append(action_l1_loss.item()) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip']: + recent_l1_loss_components[action_dim_name].append(action_l1_loss_components[action_dim_name].item()) # Compute gradient step index gradient_step_idx = batch_idx // cfg.grad_accumulation_steps - print(f"{batch_idx=}") - print(f"{cfg.grad_accumulation_steps=}") - print(f"{gradient_step_idx=}") + wandb.log({ + "batch_idx": batch_idx, + "grad_accumulation_steps": cfg.grad_accumulation_steps, + "gradient_step_idx": gradient_step_idx + }) # Compute smoothened train metrics # =>> Equal to current step metrics when not using gradient accumulation # =>> Otherwise, equal to the average of metrics observed over micro-batches used for gradient accumulation smoothened_loss = sum(recent_losses) / len(recent_losses) smoothened_action_accuracy = sum(recent_action_accuracies) / len(recent_action_accuracies) + smoothened_action_accuracy_components = { + action_dim_name: sum(recent_action_accuracies_components[action_dim_name]) / len(recent_action_accuracies_components[action_dim_name]) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + } smoothened_l1_loss = sum(recent_l1_losses) / len(recent_l1_losses) + smoothened_l1_loss_components = { + action_dim_name: sum(recent_l1_loss_components[action_dim_name]) / len(recent_l1_loss_components[action_dim_name]) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + } # Optimizer Step if ( @@ -357,14 +396,38 @@ def finetune(cfg: FinetuneConfig) -> None: optimizer.zero_grad() progress.update() total_optimizer_steps += 1 - print( - { - "total_optimizer_steps": total_optimizer_steps, - "train_loss": smoothened_loss, - "action_accuracy": smoothened_action_accuracy, - "l1_loss": smoothened_l1_loss, + # Convert recent actions to tensors for logging, split by action dimension + action_pred_by_dim = { + action_dim_name: continuous_actions_pred[i::7].detach().cpu() + for i, action_dim_name in enumerate(['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip']) + } + action_gt_by_dim = { + action_dim_name: continuous_actions_gt[i::7].detach().cpu() + for i, action_dim_name in enumerate(['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip']) + } + + wandb.log({ + "total_optimizer_steps": total_optimizer_steps, + "train_loss": smoothened_loss, + "action_accuracy": smoothened_action_accuracy, + "l1_loss": smoothened_l1_loss, + **{ + f"l1_loss_{action_dim_name}": smoothened_l1_loss_components[action_dim_name] + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] }, - ) + **{ + f"action_accuracy_{action_dim_name}": smoothened_action_accuracy_components[action_dim_name] + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + }, + **{ + f"recent_actions_pred_{action_dim_name}": wandb.Histogram(action_pred_by_dim[action_dim_name].numpy()) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + }, + **{ + f"recent_actions_gt_{action_dim_name}": wandb.Histogram(action_gt_by_dim[action_dim_name].numpy()) + for action_dim_name in ['dx', 'dy', 'dz', 'dox', 'doy', 'doz', 'grip'] + } + }) # Save Model Checkpoint =>> by default, only keeps the latest checkpoint, continually overwriting it! if total_optimizer_steps > 0 and total_optimizer_steps % cfg.save_steps == 0: @@ -372,10 +435,10 @@ def finetune(cfg: FinetuneConfig) -> None: print(f"Saving Model Checkpoint for Step {gradient_step_idx}") # If LoRA, we first save adapter weights, then merge into full model; otherwise, default save! - save_dir = adapter_dir if cfg.use_lora else run_dir + save_dir = f"{adapter_dir}-{total_optimizer_steps}_chkpt" if cfg.use_lora else run_dir # Save Processor & Weights - processor.save_pretrained(run_dir) + processor.save_pretrained(f"run_dir--{total_optimizer_steps}_chkpt") vla.module.save_pretrained(save_dir) # Wait for processor and adapter weights to be saved by main process @@ -384,6 +447,7 @@ def finetune(cfg: FinetuneConfig) -> None: # Merge LoRA weights into model backbone for faster inference # =>> Note that merging is slow and can be done post-hoc to speed up training if cfg.use_lora: + continue base_vla = AutoModelForVision2Seq.from_pretrained( cfg.vla_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True ) @@ -392,7 +456,11 @@ def finetune(cfg: FinetuneConfig) -> None: if distributed_state.is_main_process: if cfg.save_latest_checkpoint_only: # Overwrite latest checkpoint - merged_vla.save_pretrained(run_dir) + # Save locally + # merged_vla.save_pretrained(run_dir) + + # NOTE: don't save, we'll merge later + continue print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {run_dir}") else: From 202edc81455f0e422c8416c49f58fc9d4a62fce7 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 20 Feb 2025 14:17:28 +0000 Subject: [PATCH 19/58] containerization --- .dockerignore | 13 +++++++++++++ .gitignore | 1 + Dockerfile | 22 ++++++++++++++++++++++ docker-compose.yml | 17 +++++++++++++++++ finetune_lerobot.sh | 19 +++++++++++++++++++ 5 files changed, 72 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 finetune_lerobot.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..fdf1b4af0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Prevent these from being sent to Docker daemon +.git/ +.venv/ +__pycache__/ +*.py[cod] +.DS_Store +.env +data/ +.runs/ +.adapter/ +*.swp +*.swo +*.log \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8b76ce8a5..4dd11c4ae 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,4 @@ rollouts/ wandb/ .tmp .runs +.preprocessors diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..93f1b63ba --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-devel + +WORKDIR /home/marvin/alan/openvla_finetuner + +# Install system dependencies for flash-attn +RUN apt-get update && apt-get install -y \ + git \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +# Install openvla, lerobot, and flash-attn; download openvla-7b +COPY . /workspaces/openvla_finetuner +WORKDIR /workspaces/openvla_finetuner +RUN pip install -e . && \ + cd lerobot && \ + pip install -e . && \ + cd .. && \ + pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' | xargs pip install && \ + pip install packaging ninja && \ + pip install "flash-attn==2.5.5" --no-build-isolation && \ + pip install huggingface-hub && \ + huggingface-cli download openvla/openvla-7b diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..55fefc208 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ + +services: + openvla-finetuner: + image: openvla-finetuner:latest + build: . + runtime: nvidia + environment: + - WANDB_API_KEY=${WANDB_API_KEY} + - WANDB_MODE=online + - HF_HOME=/root/.cache/huggingface + volumes: + - ./:/workspaces/openvla_finetuner + - finetuner-cache:/root/.cache + command: sleep infinity + +volumes: + finetuner-cache: \ No newline at end of file diff --git a/finetune_lerobot.sh b/finetune_lerobot.sh new file mode 100755 index 000000000..91aafa1bf --- /dev/null +++ b/finetune_lerobot.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +PYTHONPATH="$PYTHONPATH:$(pwd)/lerobot" && \ +torchrun --standalone --nnodes 1 --nproc-per-node 1 vla-scripts/finetune.py \ + --vla_path "openvla/openvla-7b" \ + --data_root_dir "data/simple_task_lerobot" \ + --dataset_name "main" \ + --run_root_dir ".runs/" \ + --adapter_tmp_dir ".adapter/" \ + --lora_rank 32 \ + --batch_size 2 \ + --grad_accumulation_steps 8 \ + --learning_rate 5e-4 \ + --image_aug True \ + --max_steps 10000 \ + --save_steps 1000 \ + --save_latest_checkpoint_only False \ + --tolerance_s 0.01 \ + --wandb_entity robotgeneralist \ + --wandb_project ur5e From 375824570606463e15c08ca9701e53f78ca83a11 Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Thu, 27 Feb 2025 17:01:15 -0800 Subject: [PATCH 20/58] Public release: OpenVLA-OFT First commit for paper: "Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success" - Moo Jin Kim, Chelsea Finn, Percy Liang Website: https://openvla-oft.github.io/ --- ALOHA.md | 30 + LIBERO.md | 123 ++ LICENSE | 2 +- README.md | 679 +-------- SETUP.md | 24 + experiments/robot/aloha/aloha_utils.py | 85 ++ experiments/robot/aloha/constants.py | 52 + .../aloha/preprocess_split_aloha_data.py | 260 ++++ experiments/robot/aloha/real_env.py | 213 +++ experiments/robot/aloha/robot_utils.py | 187 +++ experiments/robot/aloha/run_aloha_eval.py | 384 ++++++ experiments/robot/libero/libero_utils.py | 28 +- experiments/robot/libero/run_libero_eval.py | 588 +++++--- .../sample_libero_spatial_observation.pkl | Bin 0 -> 301501 bytes experiments/robot/openvla_utils.py | 794 +++++++++-- experiments/robot/robot_utils.py | 179 ++- prismatic/extern/hf/modeling_prismatic.py | 729 ++++++++-- prismatic/models/action_heads.py | 211 +++ prismatic/models/film_vit_wrapper.py | 276 ++++ prismatic/models/projectors.py | 49 + .../training/strategies/base_strategy.py | 86 +- prismatic/training/train_utils.py | 56 + prismatic/util/data_utils.py | 24 +- prismatic/vla/constants.py | 86 ++ prismatic/vla/datasets/datasets.py | 55 +- prismatic/vla/datasets/rlds/dataset.py | 9 +- prismatic/vla/datasets/rlds/oxe/configs.py | 77 +- .../vla/datasets/rlds/oxe/materialize.py | 13 +- prismatic/vla/datasets/rlds/oxe/mixtures.py | 16 +- prismatic/vla/datasets/rlds/oxe/transforms.py | 10 + .../vla/datasets/rlds/traj_transforms.py | 36 +- .../vla/datasets/rlds/utils/data_utils.py | 11 +- pyproject.toml | 27 +- vla-scripts/deploy.py | 151 +- vla-scripts/finetune.py | 1213 ++++++++++++++--- vla-scripts/merge_lora_weights_and_save.py | 73 + 36 files changed, 5386 insertions(+), 1450 deletions(-) create mode 100644 ALOHA.md create mode 100644 LIBERO.md create mode 100644 SETUP.md create mode 100644 experiments/robot/aloha/aloha_utils.py create mode 100644 experiments/robot/aloha/constants.py create mode 100644 experiments/robot/aloha/preprocess_split_aloha_data.py create mode 100644 experiments/robot/aloha/real_env.py create mode 100644 experiments/robot/aloha/robot_utils.py create mode 100644 experiments/robot/aloha/run_aloha_eval.py create mode 100644 experiments/robot/libero/sample_libero_spatial_observation.pkl create mode 100644 prismatic/models/action_heads.py create mode 100644 prismatic/models/film_vit_wrapper.py create mode 100644 prismatic/models/projectors.py create mode 100644 prismatic/training/train_utils.py create mode 100644 prismatic/vla/constants.py create mode 100644 vla-scripts/merge_lora_weights_and_save.py diff --git a/ALOHA.md b/ALOHA.md new file mode 100644 index 000000000..1072c26ac --- /dev/null +++ b/ALOHA.md @@ -0,0 +1,30 @@ +# OpenVLA-OFT+ in Real-World ALOHA Robot Tasks + +## Relevant Files + +Evaluation +* `experiments/robot/aloha/`: ALOHA eval files + * `run_aloha_eval.py`: ALOHA eval script + * `aloha_utils.py`: ALOHA eval utils + * Other ALOHA robot environment files copied from the original [ALOHA GitHub repo](https://github.com/tonyzhaozh/aloha): + * `constants.py` + * `real_env.py` + * `robot_utils.py` +* `experiments/robot/`: General eval utils files + * `openvla_utils.py`: OpenVLA-specific eval utils + * `robot_utils.py`: Other eval utils + +Training +* `vla-scripts/finetune.py`: VLA fine-tuning script + +## Setup + +(Coming soon!) + +## Launching LIBERO Evaluations + +(Coming soon!) + +## Fine-Tuning on LIBERO Datasets + +(Coming soon!) diff --git a/LIBERO.md b/LIBERO.md new file mode 100644 index 000000000..fcadfc6fd --- /dev/null +++ b/LIBERO.md @@ -0,0 +1,123 @@ +# OpenVLA-OFT in the LIBERO Simulation Benchmark + +## Relevant Files + +Evaluation +* `experiments/robot/libero/`: LIBERO eval files + * `run_libero_eval.py`: LIBERO eval script + * `libero_utils.py`: LIBERO eval utils +* `experiments/robot/`: General eval utils files + * `openvla_utils.py`: OpenVLA-specific eval utils + * `robot_utils.py`: Other eval utils + +Training +* `vla-scripts/finetune.py`: VLA fine-tuning script + + +## Setup + +Set up a conda environment (see instructions in [SETUP.md](SETUP.md)). + +Clone and install the [LIBERO repo](https://github.com/Lifelong-Robot-Learning/LIBERO) and required packages: + +```bash +git clone https://github.com/Lifelong-Robot-Learning/LIBERO.git +pip install -e LIBERO +pip install -r experiments/robot/libero/libero_requirements.txt # From openvla-oft base dir +``` + +(Optional, if you plan to launch training) To download the [LIBERO datasets](https://huggingface.co/datasets/openvla/modified_libero_rlds) that we used in our fine-tuning +experiments, run the command below. This will download the LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, +and LIBERO-10 datasets in RLDS data format (~10 GB total). You can use these to fine-tune OpenVLA or +train other methods. This step is optional since we provide pretrained OpenVLA-OFT checkpoints below. +Note that these are the same datasets used in the original OpenVLA project. If needed, see details on how to download the original non-RLDS datasets [here](https://github.com/openvla/openvla?tab=readme-ov-file#libero-setup). +```bash +git clone git@hf.co:datasets/openvla/modified_libero_rlds +``` + +## Launching LIBERO Evaluations + +We fine-tuned OpenVLA via LoRA (r=32) with our OFT recipe on four LIBERO task suites independently: LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, and LIBERO-10 (also called LIBERO-Long). +The four OpenVLA-OFT checkpoints for LIBERO are available on Hugging Face: +* [moojink/openvla-7b-oft-finetuned-libero-spatial](https://huggingface.co/moojink/openvla-7b-oft-finetuned-libero-spatial) +* [moojink/openvla-7b-oft-finetuned-libero-object](https://huggingface.co/moojink/openvla-7b-oft-finetuned-libero-object) +* [moojink/openvla-7b-oft-finetuned-libero-goal](https://huggingface.co/moojink/openvla-7b-oft-finetuned-libero-goal) +* [moojink/openvla-7b-oft-finetuned-libero-10](https://huggingface.co/moojink/openvla-7b-oft-finetuned-libero-10) + +To start evaluations with one of these checkpoints, run one of the commands below. Each will automatically download the appropriate checkpoint listed above. You can set the `TRANSFORMERS_CACHE` and `HF_HOME` environment variable to change where the checkpoint files get cached. + +```bash +# Launch LIBERO-Spatial evals +python experiments/robot/libero/run_libero_eval.py \ + --pretrained_checkpoint moojink/openvla-7b-oft-finetuned-libero-spatial \ + --task_suite_name libero_spatial + +# Launch LIBERO-Object evals +python experiments/robot/libero/run_libero_eval.py \ + --pretrained_checkpoint moojink/openvla-7b-oft-finetuned-libero-object \ + --task_suite_name libero_object + +# Launch LIBERO-Goal evals +python experiments/robot/libero/run_libero_eval.py \ + --pretrained_checkpoint moojink/openvla-7b-oft-finetuned-libero-goal \ + --task_suite_name libero_goal + +# Launch LIBERO-10 (LIBERO-Long) evals +python experiments/robot/libero/run_libero_eval.py \ + --pretrained_checkpoint moojink/openvla-7b-oft-finetuned-libero-10 \ + --task_suite_name libero_10 +``` + +Notes: +* The evaluation script will run 500 trials by default (10 tasks x 50 episodes each). You can modify the number of + trials per task by setting `--num_trials_per_task`. You can also change the random seed via `--seed`. There are + other arguments in the script; we set them to the default values that work with the OpenVLA-OFT checkpoints above. +* **NOTE: Setting `--center_crop True` is important** because we fine-tuned OpenVLA with random crop augmentations + (we took a random crop with 90% area in every training sample, so at test time we simply take the center 90% crop). +* The evaluation script logs results locally. You can also log results in Weights & Biases + by setting `--use_wandb True` and specifying `--wandb_project ` and `--wandb_entity `. +* The results reported in our paper were obtained using **Python 3.10.14, PyTorch 2.2.0, and our + [custom transformers v4.40.1 fork](https://github.com/moojink/transformers-openvla-oft.git)** + on an **NVIDIA A100 GPU**, averaged over three random seeds. Please stick to these package versions if possible. + Note that results may vary slightly if you use a different GPU than the A100. If the discrepancy is large, + please post a GitHub issue, and we will look into it. + +## Fine-Tuning on LIBERO Datasets + +First, download the LIBERO datasets as mentioned above in the Setup section above: `libero_spatial_no_noops`, `libero_object_no_noops`, `libero_goal_no_noops`, `libero_10_no_noops`. (`"_no_noops"` stands for no no-op actions, i.e., training samples with near-zero actions are filtered out). + +Then, launch the fine-tuning script with the OFT configuration below, replacing `X` in the first line with the number of GPUs. The command below launches fine-tuning on LIBERO-Spatial with the hyperparameters that we used in our paper. Here, batch size 8 per GPU will require ~62 GB VRAM, and batch size 1 per GPU will require ~25 GB VRAM. + +```bash +torchrun --standalone --nnodes 1 --nproc-per-node X vla-scripts/finetune.py \ + --vla_path openvla/openvla-7b \ + --data_root_dir /PATH/TO/RLDS/DATASETS/DIR/ \ + --dataset_name libero_spatial_no_noops \ + --run_root_dir /YOUR/CHECKPOINTS/AND/LOG/DIR/ \ + --use_l1_regression True \ + --use_diffusion False \ + --use_film False \ + --num_images_in_input 2 \ + --use_proprio True \ + --batch_size 8 \ + --learning_rate 5e-4 \ + --num_steps_before_decay 100000 \ + --max_steps 150005 \ + --save_freq 10000 \ + --save_latest_checkpoint_only False \ + --image_aug True \ + --lora_rank 32 \ + --wandb_entity "YOUR_WANDB_ENTITY" \ + --wandb_project "YOUR_WANDB_PROJECT" \ + --run_id_note parallel_dec--8_acts_chunk--continuous_acts--L1_regression--3rd_person_img--wrist_img--proprio_state +``` + +The above training command should reproduce our OpenVLA-OFT results if `X = 8` and the 150K step checkpoint is evaluated. + +You can replace `libero_spatial_no_noops` with `libero_object_no_noops`, `libero_goal_no_noops`, or `libero_10_no_noops`. You can also modify other args — e.g., if you want to train with just one input image from the third-person camera and disable proprio state input, you can set `--num_images_in_input 1` and `--use_proprio False`. + +In general, we recommend fine-tuning until training L1 loss goes below 0.01 and starts to plateau (with the above configuration, it should reach ~0.006 L1 loss on LIBERO-Spatial after 150K gradient steps with 10x LR decay after 100K steps). However, for LIBERO-Goal only, we found that the 50K checkpoint (which was at ~0.02 L1 loss) performed best for unknown reasons. For all other task suites though, we found that the 150K checkpoint performed best. + +Please be sure to test your policy with the same device/GPU used to train it! Otherwise, performance may drop substantially. You may be able to avoid the performance drop if you merge the LoRA weights into the base model on the downstream device used for testing (e.g., if you train on H100 and then merge on A100 before testing on A100). You can see our script [vla-scripts/merge_lora_weights_and_save.py](vla-scripts/merge_lora_weights_and_save.py) for merging the LoRA adapter into the base model offline. It's okay if you already merged LoRA weights into the base OpenVLA model during fine-tuning; you can always redownload the base model and merge again as long as you still have the LoRA adapter (`merge_lora_weights_and_save.py` will handle this for you). + +If you run into any issues, please open a new GitHub issue. If you do not receive a response within 2 business days, please email Moo Jin Kim (moojink@cs.stanford.edu) to bring the issue to his attention. diff --git a/LICENSE b/LICENSE index 04f26a7d0..b2c22d519 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Moo Jin Kim, Karl Pertsch, Siddharth Karamcheti. +Copyright (c) 2025 Moo Jin Kim, Chelsea Finn, Percy Liang. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 59e0c3130..0017aa13a 100644 --- a/README.md +++ b/README.md @@ -1,635 +1,90 @@ -# OpenVLA: An Open-Source Vision-Language-Action Model +# Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success -[![arXiv](https://img.shields.io/badge/arXiv-2406.09246-df2a2a.svg?style=for-the-badge)](https://arxiv.org/abs/2406.09246) -[![HF Models](https://img.shields.io/badge/%F0%9F%A4%97-Models-yellow?style=for-the-badge)](https://huggingface.co/openvla/openvla-7b) -[![PyTorch](https://img.shields.io/badge/PyTorch-2.2.0-EE4C2C.svg?style=for-the-badge&logo=pytorch)](https://pytorch.org/get-started/locally/) -[![Python](https://img.shields.io/badge/python-3.10-blue?style=for-the-badge)](https://www.python.org) -[![License](https://img.shields.io/github/license/TRI-ML/prismatic-vlms?style=for-the-badge)](LICENSE) - -[**Getting Started**](#getting-started) | [**Pretrained VLAs**](#pretrained-vlas) | [**Installation**](#installation) | [**Fine-Tuning OpenVLA via LoRA**](#fine-tuning-openvla-via-lora) | [**Fully Fine-Tuning OpenVLA**](#fully-fine-tuning-openvla) | -[**Training VLAs from Scratch**](#training-vlas-from-scratch) | [**Evaluating OpenVLA**](#evaluating-openvla) | [**Project Website**](https://openvla.github.io/) +**Project website: https://openvla-oft.github.io/** +**Paper: TODO** +**Summary video: https://youtu.be/T3Zkkr_NTSA** +## System Requirements -
+Inference: +* 1 GPU with ~16 GB VRAM for LIBERO sim benchmark tasks +* 1 GPU with ~20 GB VRAM for ALOHA robot tasks -## Latest Updates -- [2024-10-15] Added a [VLA Performance Troubleshooting](#vla-performance-troubleshooting) section to the README with best practices for debugging poor VLA performance after fine-tuning. -- [2024-09-04] Added LIBERO simulation benchmark fine-tuning experiments to paper (see v2 on [arXiv](https://arxiv.org/abs/2406.09246)); - added instructions for reproducing OpenVLA results in [LIBERO Simulation Benchmark Evaluations](#libero-simulation-benchmark-evaluations) section -- [2024-08-14] Added new section, [Evaluating OpenVLA](#evaluating-openvla), with instructions for running BridgeData V2 WidowX robot evals -- [2024-07-08] Added new sections: [Fine-Tuning OpenVLA via LoRA](#fine-tuning-openvla-via-lora), [Fully Fine-Tuning OpenVLA](#fully-fine-tuning-openvla) -- [2024-06-13] Initial release +Training: +* Between 1-8 GPUs with 27-80 GB, depending on the desired training setup (with default bfloat16 data type). See [this FAQ on our project website](https://openvla-oft.github.io/#train-compute) for details. -
+## Quick Start -A simple and scalable codebase for training and fine-tuning vision-language-action models (VLAs) for generalist robotic -manipulation: +First, set up a conda environment (see instructions in [SETUP.md](SETUP.md)). -- **Different Dataset Mixtures**: We natively support arbitrary datasets in RLDS format, including arbitrary mixtures of - data from the [Open X-Embodiment Dataset](https://robotics-transformer-x.github.io/). -- **Easy Scaling**: Powered by PyTorch FSDP and Flash-Attention, we can quickly and efficiently train models from 1B - - 34B parameters, with easily adaptable model architectures. -- **Native Fine-Tuning Support**: Built-in support (with examples) for various forms of fine-tuning (full, - partial, LoRA). - -Built on top of [Prismatic VLMs](https://github.com/TRI-ML/prismatic-vlms). - -## Getting Started - -To get started with loading and running OpenVLA models for inference, we provide a lightweight interface that leverages -HuggingFace `transformers` AutoClasses, with minimal dependencies. - -For example, to load `openvla-7b` for zero-shot instruction following in the -[BridgeData V2 environments](https://rail-berkeley.github.io/bridgedata/) with a WidowX robot: +Then, run the Python script below to download a pretrained OpenVLA-OFT checkpoint and run inference to generate an action chunk: ```python -# Install minimal dependencies (`torch`, `transformers`, `timm`, `tokenizers`, ...) -# > pip install -r https://raw.githubusercontent.com/openvla/openvla/main/requirements-min.txt -from transformers import AutoModelForVision2Seq, AutoProcessor -from PIL import Image - -import torch - -# Load Processor & VLA -processor = AutoProcessor.from_pretrained("openvla/openvla-7b", trust_remote_code=True) -vla = AutoModelForVision2Seq.from_pretrained( - "openvla/openvla-7b", - attn_implementation="flash_attention_2", # [Optional] Requires `flash_attn` - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - trust_remote_code=True -).to("cuda:0") - -# Grab image input & format prompt -image: Image.Image = get_from_camera(...) -prompt = "In: What action should the robot take to {}?\nOut:" - -# Predict Action (7-DoF; un-normalize for BridgeData V2) -inputs = processor(prompt, image).to("cuda:0", dtype=torch.bfloat16) -action = vla.predict_action(**inputs, unnorm_key="bridge_orig", do_sample=False) - -# Execute... -robot.act(action, ...) +import pickle +from experiments.robot.libero.run_libero_eval import GenerateConfig +from experiments.robot.openvla_utils import get_action_head, get_processor, get_proprio_projector, get_vla, get_vla_action +from prismatic.vla.constants import NUM_ACTIONS_CHUNK, PROPRIO_DIM + +# Instantiate config (see class GenerateConfig in experiments/robot/libero/run_libero_eval.py for definitions) +cfg = GenerateConfig( + pretrained_checkpoint = "moojink/openvla-7b-oft-finetuned-libero-spatial", + use_l1_regression = True, + use_diffusion = False, + use_film = False, + num_images_in_input = 2, + use_proprio = True, + load_in_8bit = False, + load_in_4bit = False, + center_crop = True, + num_open_loop_steps = NUM_ACTIONS_CHUNK, + unnorm_key = "libero_spatial_no_noops", +) + +# Load OpenVLA-OFT policy and inputs processor +vla = get_vla(cfg) +processor = get_processor(cfg) + +# Load MLP action head to generate continuous actions (via L1 regression) +action_head = get_action_head(cfg, llm_dim=vla.llm_dim) + +# Load proprio projector to map proprio to language embedding space +proprio_projector = get_proprio_projector(cfg, llm_dim=vla.llm_dim, proprio_dim=PROPRIO_DIM) + +# Load sample observation: +# observation (dict): { +# "full_image": primary third-person image, +# "wrist_image": wrist-mounted camera image, +# "state": robot proprioceptive state, +# "task_description": task description, +# } +with open("experiments/robot/libero/sample_libero_spatial_observation.pkl", "rb") as file: + observation = pickle.load(file) + +# Generate robot action chunk (sequence of future actions) +actions = get_vla_action(cfg, vla, processor, observation, observation["task_description"], action_head, proprio_projector) +print("Generated action chunk:") +for act in actions: + print(act) ``` -We also provide an [example script for fine-tuning OpenVLA models for new tasks and -embodiments](./vla-scripts/finetune.py); this script supports different fine-tuning modes -- including (quantized) -low-rank adaptation (LoRA) supported by [HuggingFace's PEFT library](https://huggingface.co/docs/peft/en/index). - -For deployment, we provide a lightweight script for [serving OpenVLA models over a REST API](./vla-scripts/deploy.py), -providing an easy way to integrate OpenVLA models into existing robot control stacks, -removing any requirement for powerful on-device compute. - -## Pretrained VLAs - -We release two OpenVLA models trained as part of our work, with checkpoints, configs, and model cards available [on our -HuggingFace page](https://huggingface.co/openvla): -- [`openvla-7b`](https://huggingface.co/openvla/openvla-7b): The flagship model from our paper, trained from - the Prismatic `prism-dinosiglip-224px` VLM (based on a fused DINOv2 and SigLIP vision backbone, and Llama-2 LLM). - Trained on a large mixture of datasets from Open X-Embodiment spanning 970K trajectories - ([mixture details - see "Open-X Magic Soup++"](./prismatic/vla/datasets/rlds/oxe/mixtures.py)). -- [`openvla-v01-7b`](https://huggingface.co/openvla/openvla-7b-v01): An early model used during development, trained from - the Prismatic `siglip-224px` VLM (singular SigLIP vision backbone, and a Vicuña v1.5 LLM). Trained on the same mixture - of datasets as [Octo](https://github.com/octo-models/octo), but for significantly fewer GPU hours than our final model - ([mixture details - see "Open-X Magic Soup"](./prismatic/vla/datasets/rlds/oxe/mixtures.py)). - -**Explicit Notes on Model Licensing & Commercial Use**: While all code in this repository is released under an MIT -License, our pretrained models may inherit restrictions from the underlying base models we use. Specifically, both the -above models are derived from Llama-2, and as such are subject to the -[Llama Community License](https://ai.meta.com/llama/license/). - ---- - ## Installation -> **Note**: These installation instructions are for full-scale pretraining (and distributed fine-tuning); if looking to - just run inference with OpenVLA models (or perform lightweight fine-tuning), see instructions above! - -This repository was built using Python 3.10, but should be backwards compatible with any Python >= 3.8. We require -PyTorch 2.2.* -- installation instructions [can be found here](https://pytorch.org/get-started/locally/). The latest -version of this repository was developed and thoroughly tested with: - - PyTorch 2.2.0, torchvision 0.17.0, transformers 4.40.1, tokenizers 0.19.1, timm 0.9.10, and flash-attn 2.5.5 - -**[5/21/24] Note**: Following reported regressions and breaking changes in later versions of `transformers`, `timm`, and -`tokenizers` we explicitly pin the above versions of the dependencies. We are working on implementing thorough tests, -and plan on relaxing these constraints as soon as we can. - -Use the setup commands below to get started: - -```bash -# Create and activate conda environment -conda create -n openvla python=3.10 -y -conda activate openvla - -# Install PyTorch. Below is a sample command to do this, but you should check the following link -# to find installation instructions that are specific to your compute platform: -# https://pytorch.org/get-started/locally/ -conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch -c nvidia -y # UPDATE ME! - -# Clone and install the openvla repo -git clone https://github.com/openvla/openvla.git -cd openvla -pip install -e . - -# Install Flash Attention 2 for training (https://github.com/Dao-AILab/flash-attention) -# =>> If you run into difficulty, try `pip cache remove flash_attn` first -pip install packaging ninja -ninja --version; echo $? # Verify Ninja --> should return exit code "0" -pip install "flash-attn==2.5.5" --no-build-isolation -``` - -If you run into any problems during the installation process, please file a GitHub Issue. - -**Note:** See `vla-scripts/` for full training and verification scripts for OpenVLA models. Note that `scripts/` is -mostly a holdover from the original (base) `prismatic-vlms` repository, with support for training and evaluating -visually-conditioned language models; while you can use this repo to train VLMs AND VLAs, note that trying to generate -language (via `scripts/generate.py`) with existing OpenVLA models will not work (as we only train current OpenVLA models -to generate actions, and actions alone). - -## Fine-Tuning OpenVLA via LoRA - -In this section, we discuss fine-tuning OpenVLA using Low-Rank Adaptation (LoRA) via the Hugging Face `transformers` library, -which is recommended if you do not have sufficient compute to fully fine-tune a 7B-parameter model. The main script for LoRA -fine-tuning is `vla-scripts/finetune.py`. (If you instead wish to do full fine-tuning, please see the -[Fully Fine-Tuning OpenVLA](#fully-fine-tuning-openvla) section.) - -Below we show an example of how you can fine-tune the main OpenVLA checkpoint ([`openvla-7b`](https://huggingface.co/openvla/openvla-7b)) -via LoRA. Here we fine-tune on [BridgeData V2](https://rail-berkeley.github.io/bridgedata/) using a single A100 -GPU with 80 GB VRAM. (You can also fine-tune with a smaller GPU, as long as it has at least ~27 GB of memory, -by modifying the batch size.) - -First, download the BridgeData V2 dataset: - -```bash -# Change directory to your base datasets folder -cd - -# Download the full dataset (124 GB) -wget -r -nH --cut-dirs=4 --reject="index.html*" https://rail.eecs.berkeley.edu/datasets/bridge_release/data/tfds/bridge_dataset/ - -# Rename the dataset to `bridge_orig` (NOTE: Omitting this step may lead to runtime errors later) -mv bridge_dataset bridge_orig -``` - -Now, launch the LoRA fine-tuning script, as shown below. Note that `--batch_size==16` with `--grad_accumulation_steps==1` -requires ~72 GB GPU memory. If you have a smaller GPU, you should reduce `--batch_size` and increase `--grad_accumulation_steps` -to maintain an effective batch size that is large enough for stable training. If you have multiple GPUs and wish to train via -PyTorch Distributed Data Parallel (DDP), simply set `--nproc-per-node` in the `torchrun` command below to the number of available GPUs. - -```bash -torchrun --standalone --nnodes 1 --nproc-per-node 1 vla-scripts/finetune.py \ - --vla_path "openvla/openvla-7b" \ - --data_root_dir \ - --dataset_name bridge_orig \ - --run_root_dir \ - --adapter_tmp_dir \ - --lora_rank 32 \ - --batch_size 16 \ - --grad_accumulation_steps 1 \ - --learning_rate 5e-4 \ - --image_aug \ - --wandb_project \ - --wandb_entity \ - --save_steps -``` - -Note: If you set `--image_aug==False` in the command above, you will observe nearly 100% `action_accuracy` in the training logs, -since the [`openvla-7b`](https://huggingface.co/openvla/openvla-7b) model is already pretrained (without augmentations) on a -superset of datasets that includes BridgeData V2. - -To LoRA fine-tune on a different dataset, you can download the dataset from the [Open X-Embodiment (OXE)](https://robotics-transformer-x.github.io/) -mixture (see [this custom script](https://github.com/moojink/rlds_dataset_mod/blob/main/prepare_open_x.sh) for an example of how to download datasets -from OXE). Alternatively, if you have a custom dataset that is not part of OXE, you can either (a) convert the dataset to the RLDS format which is -compatible with our fine-tuning script (see [this repo](https://github.com/kpertsch/rlds_dataset_builder) for instructions on this), or (b) use your own -custom PyTorch Dataset wrapper (see comments in `vla-scripts/finetune.py` for instructions). We recommend option (a) for most users; the RLDS dataset and -dataloader are tested more extensively since we used these for all of our pretraining and fine-tuning experiments. - -For option (a), after you converted your dataset to RLDS, you need to register it with our data loader, by registering a dataset -config [here](prismatic/vla/datasets/rlds/oxe/configs.py#L54) and a dataset transform function [here](prismatic/vla/datasets/rlds/oxe/transforms.py#L828). - -Once you have integrated your new dataset, you can launch LoRA fine-tuning with the same `vla-scripts/finetune.py` script above. If you run into any issues, -please visit the [VLA Troubleshooting](#vla-troubleshooting) section or search for a similar issue in the [OpenVLA GitHub Issues page](https://github.com/openvla/openvla/issues?q=) -(including "Closed" issues). If you cannot find a similar issue there, feel free to create a new issue. - -## Fully Fine-Tuning OpenVLA - -In this section, we discuss fully fine-tuning OpenVLA (all 7.5 billion parameters) via native PyTorch Fully Sharded Data Parallel (FSDP) -using the [Prismatic VLMs](https://github.com/TRI-ML/prismatic-vlms) training script. Full fine-tuning is more advanced/involved and is only recommended -if you have sufficient compute (e.g., a full node of 8 A100 GPUs) and if LoRA fine-tuning is insufficient for your use case (e.g., if the fine-tuning distribution -varies drastically from the pretraining distribution). Otherwise, we recommend that you try parameter-efficient fine-tuning via LoRA, which is described in the -[Fine-Tuning OpenVLA via LoRA](#fine-tuning-openvla-via-lora) section. - -For full fine-tuning, you will need to download [a different version of the OpenVLA model checkpoint](https://huggingface.co/openvla/openvla-7b-prismatic) that is compatible -with the Prismatic VLMs codebase, which we built on top of to develop the OpenVLA model. You can download this Prismatic-compatible OpenVLA checkpoint using the git commands below -(alternatively, you can download via the [Hugging Face CLI](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli)): - -```bash -# Change directory to your base model checkpoints folder -cd - -# Download checkpoint (30 GB) -- may take a few minutes -git clone git@hf.co:openvla/openvla-7b-prismatic - -# If the command above did not download the full checkpoint, -# manually fetch it via git Large File Storage (LFS) -# Note: You may have to configure an SSH key for this to work -cd openvla-7b-prismatic -git lfs fetch --all -``` - -We show how you can fully fine-tune OpenVLA on [BridgeData V2](https://rail-berkeley.github.io/bridgedata/) using a single node with 8 GPUs. If you wish to -use a different number of GPUs (or nodes), you can modify the VLA training configuration in [`prismatic/conf/vla.py`](prismatic/conf/vla.py). - -Download the BridgeData V2 dataset: - -```bash -# Change directory to your base datasets folder -cd - -# Download the full dataset (124 GB) -wget -r -nH --cut-dirs=4 --reject="index.html*" https://rail.eecs.berkeley.edu/datasets/bridge_release/data/tfds/bridge_dataset/ - -# Rename the dataset to `bridge_orig` (NOTE: Omitting this step may lead to runtime errors later) -mv bridge_dataset bridge_orig -``` - -Next, create a [Hugging Face user access token](https://huggingface.co/docs/hub/en/security-tokens) and copy the token value (a string that starts with -`hf_...`) into a file named `.hf_token` at the root directory of this repo (`openvla/.hf_token`). - -```bash -# Go to openvla root directory -cd openvla - -# Copy HF token value into token file. Replace "hf_..." with your own token value! -# See: https://huggingface.co/docs/hub/en/security-tokens -echo hf_... >>> .hf_token -``` - -Now, launch the training script. If you wish to use a different number of nodes or GPUs, modify the VLA training configuration in -[`prismatic/conf/vla.py`](prismatic/conf/vla.py) and then change the `--nnodes` and `--nproc-per-node` arguments below accordingly. - -```bash -torchrun --standalone --nnodes 1 --nproc-per-node 8 vla-scripts/train.py \ - --pretrained_checkpoint \ - --vla.type prism-dinosiglip-224px+mx-bridge \ - --data_root_dir \ - --run_root_dir \ - --run_id \ - --image_aug \ - --wandb_project \ - --wandb_entity \ - --save_interval \ - --is_resume False -``` - -Note that the `--is_resume` argument is set to `False` above since we are fine-tuning a pretrained checkpoint rather than resuming a paused training run. - -If your training run gets paused and you wish to resume from the latest checkpoint, change `--pretrained_checkpoint` to the latest checkpoint path, -and then set `--is_resume==True` and specify `--resume_step` and `--resume_epoch` as the step and epoch number, respectively. For example, if you wish to -resume training from a checkpoint named `step-010000-epoch-20-loss=0.0160.pt`, you would set `is_resume==True`, `resume_step==10000`, and `resume_epoch==20`. - -Note: If you run the BridgeData V2 fine-tuning command above, you should observe nearly 100% Action Token Accuracy in the training logs, since the -[`openvla-7b`](https://huggingface.co/openvla/openvla-7b) model is already pretrained on a superset of datasets that includes BridgeData V2. - -To fully fine-tune OpenVLA on a different dataset, you can download the dataset from the [Open X-Embodiment (OXE)](https://robotics-transformer-x.github.io/) -mixture (see [this custom script](https://github.com/moojink/rlds_dataset_mod/blob/main/prepare_open_x.sh) for an example of how to download datasets from OXE). -Alternatively, if you have a custom dataset that is not part of OXE, you can convert the dataset to the RLDS format, which is compatible with our fine-tuning script -(see [this repo](https://github.com/kpertsch/rlds_dataset_builder) for instructions on this). After downloading/converting the dataset, you will need to modify the following files: - -* [`prismatic/conf/vla.py`](prismatic/conf/vla.py): Add a new training configuration by creating an experiment class, and then register it in the `VLARegistry` at the bottom of the file. - * Make sure to create a new unique `vla_id` for your fine-tuning run, and adjust some configuration variables as needed – e.g., `expected_world_size` (number of GPUs), - `per_device_batch_size` (batch size per GPU), `global_batch_size` (total batch size), `shuffle_buffer_size` (number of samples in shuffle buffer per GPU), etc. See comments - under the `VLAConfig` class at the top of the file to understand the purpose of each variable. -* [`prismatic/vla/datasets/rlds/oxe/mixtures.py`](prismatic/vla/datasets/rlds/oxe/mixtures.py): Define a new mixture for your fine-tuning mixture in the `OXE_NAMED_MIXTURES` dictionary. -* [`prismatic/vla/datasets/rlds/oxe/transforms.py`](prismatic/vla/datasets/rlds/oxe/transforms.py): Define a new dataset transform function for your fine-tuning dataset, and add it to the -`OXE_STANDARDIZATION_TRANSFORMS` registry at the bottom of the file. -* [`prismatic/vla/datasets/rlds/oxe/configs.py`](prismatic/vla/datasets/rlds/oxe/configs.py): Add a new configuration specifying your fine-tuning dataset's observation and action spaces -to the `OXE_DATASET_CONFIGS` dictionary. - -After completing the steps above, you can start full fine-tuning using the `vla-scripts/train.py` script. Make sure to set the `--vla.type` argument to the new `vla_id` that you added in `prismatic/conf/vla.py`. - -When you are finished with fine-tuning, you will need to convert the final model checkpoint to a version that is -compatible with the Hugging Face `transformers` library. See the [Converting Prismatic Models to Hugging Face](#converting-prismatic-models-to-hugging-face) section for instructions. - -If you run into any issues, please visit the [VLA Troubleshooting](#vla-troubleshooting) section or search for a similar issue in the -[OpenVLA GitHub Issues page](https://github.com/openvla/openvla/issues?q=) (including "Closed" issues). If you cannot find a similar issue there, feel free to create a new issue. - -### Converting Prismatic Models to Hugging Face - -If you have used the Prismatic VLMs codebase to train your model (e.g., if you did full fine-tuning of OpenVLA on a -new dataset), you will need to convert the final checkpoint to a version that is compatible with Hugging Face -`transformers` AutoClasses. We discuss how to do so in this section. - -Let's say your training run directory is `PRISMATIC_RUN_DIR` (e.g., `prism-dinosiglip-224px+mx-oxe-magic-soup-plus+n8+b32+x7`). -Inside this directory, there should be a directory called `checkpoints` which contains saved model checkpoints (e.g., -`step-295000-epoch-40-loss=0.2200.pt`). The Prismatic-to-Hugging-Face conversion script -([convert_openvla_weights_to_hf.py](vla-scripts/extern/convert_openvla_weights_to_hf.py)) expects a checkpoint file -named `latest-checkpoint.pt`. Therefore, you should first create a symbolic link called `latest-checkpoint.pt` that -points to the checkpoint file that you wish to convert: - -```bash -# Go to your Prismatic training run's `checkpoints` directory -cd PRISMATIC_RUN_DIR/checkpoints - -# Create symbolic link pointing to your checkpoint file -ln -s latest-checkpoint.pt -``` - -Then, launch the conversion script to convert the checkpoint from the Prismatic VLMs format to the Hugging Face format: - -```bash -python vla-scripts/extern/convert_openvla_weights_to_hf.py \ - --openvla_model_path_or_id \ - --output_hf_model_local_path -``` - -The command above will save the HF-compatible checkpoint in `output_hf_model_local_path`. Now you can load the checkpoint -with HF AutoClasses as normal, as shown below. Note that there is an additional necessary step to register the OpenVLA model -to HF AutoClasses before loading it because you are loading a locally saved checkpoint rather than one that is pushed to the -HF Hub (see [here](https://huggingface.co/docs/transformers/en/custom_models#registering-a-model-with-custom-code-to-the-auto-classes) -for details). - -```python -import torch -from transformers import AutoConfig, AutoImageProcessor, AutoModelForVision2Seq, AutoProcessor - -from prismatic.extern.hf.configuration_prismatic import OpenVLAConfig -from prismatic.extern.hf.modeling_prismatic import OpenVLAForActionPrediction -from prismatic.extern.hf.processing_prismatic import PrismaticImageProcessor, PrismaticProcessor - -# Register OpenVLA model to HF AutoClasses (not needed if you pushed model to HF Hub) -AutoConfig.register("openvla", OpenVLAConfig) -AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) -AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) -AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) - -# Load Processor & VLA -processor = AutoProcessor.from_pretrained("", trust_remote_code=True) -vla = AutoModelForVision2Seq.from_pretrained( - "", - attn_implementation="flash_attention_2", # [Optional] Requires `flash_attn` - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - trust_remote_code=True, -).to("cuda:0") - -... -``` - -## Training VLAs from Scratch - -We provide full instructions and configurations for training VLA models on (arbitrary subsets of) the -[Open X-Embodiment (OXE) Dataset](https://robotics-transformer-x.github.io/). If you run in to any issues with -the following, see [VLA Troubleshooting](#vla-troubleshooting) below (or file a GitHub Issue). - -### VLA Pretraining Datasets - -We download and preprocess individual datasets from Open X-Embodiment in [RLDS format](https://github.com/google-research/rlds) following -[this custom script](https://github.com/moojink/rlds_dataset_mod/blob/main/prepare_open_x.sh). See -[mixtures.py](./prismatic/vla/datasets/rlds/oxe/mixtures.py) for the full list of component datasets (and mixture -weights) we use to train `openvla-7b`. -- **Important**: For the BridgeData V2 component, the version in OXE is out of date (as of 12/20/2023). Instead, - you should download the dataset from the [official website](https://rail.eecs.berkeley.edu/datasets/bridge_release/data/tfds/bridge_dataset/) and place it under the subdirectory `bridge_orig/`. - Replace any reference to `bridge` in the OXE code with `bridge_orig`. - -### VLA Configuration & Training Script - -The entry point for VLA training is [`vla-scripts/train.py`](vla-scripts/train.py). We use -[`draccus`](https://pypi.org/project/draccus) to provide a modular, dataclass-based interface for specifying VLA -training configurations; existing VLA configurations are in [`prismatic/conf/vla.py`](prismatic/conf/vla.py). You can -add your own training configuration and refer to it using the `--vla.type` command line argument. - -We use PyTorch Fully Sharded Data Parallel (FSDP) to distribute training across GPUs. Launch training via `torchrun`: - -```bash -# Train VLA on BridgeData V2 with the Prismatic DINO-SigLIP 224px Backbone on a Single Node (w/ 8 GPUs) -torchrun --standalone --nnodes 1 --nproc-per-node 8 vla-scripts/train.py \ - --vla.type "prism-dinosiglip-224px+mx-bridge" \ - --data_root_dir \ - --run_root_dir \ - --wandb_project "" \ - --wandb_entity "" -``` - -### VLA Troubleshooting - -The following are a list of known problems and corresponding fixes: - -```bash -FileNotFoundError: Failed to construct dataset "fractal20220817_data", builder_kwargs "{'data_dir': '/path/to/processed/datasets/'}": Could not load dataset info from fractal20220817_data/0.1.0/dataset_info.json -``` -- **Fix**: Downgrade `tensorflow-datasets` via `pip install tensorflow-datasets==4.9.3`. - - -```bash -AttributeError: 'DLataset' object has no attribute 'traj_map'. Did you mean: 'flat_map'? -``` -- **Fix**: Upgrade `dlimp` to the newest version. You may have to `--force-reinstall` like so: -`pip install --no-deps --force-reinstall git+https://github.com/moojink/dlimp_openvla` - ---- - -## Evaluating OpenVLA - -### BridgeData V2 WidowX Evaluations - -#### Setup - -Clone the [BridgeData V2 WidowX controller repo](https://github.com/rail-berkeley/bridge_data_robot) and install the `widowx_envs` package: - -```bash -git clone https://github.com/rail-berkeley/bridge_data_robot.git -cd bridge_data_robot -pip install -e widowx_envs -``` - -Additionally, install the [`edgeml`](https://github.com/youliangtan/edgeml) library: -```bash -git clone https://github.com/youliangtan/edgeml.git -cd edgeml -pip install -e . -``` - -Follow the instructions in the `bridge_data_robot` README to create the Bridge WidowX Docker container. - -#### Launching BridgeData V2 Evaluations - -There are multiple ways to run BridgeData V2 evaluations. We describe the server-client method below. - -In one Terminal window (e.g., in tmux), start the WidowX Docker container: - -```bash -cd bridge_data_robot -./generate_usb_config.sh -USB_CONNECTOR_CHART=$(pwd)/usb_connector_chart.yml docker compose up --build robonet -``` - -In a second Terminal window, run the WidowX robot server: - -```bash -cd bridge_data_robot -docker compose exec robonet bash -lic "widowx_env_service --server" -``` - -In a third Terminal window, run the OpenVLA policy evaluation script: - -```bash -cd openvla -python experiments/robot/bridge/run_bridgev2_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b -``` - -If you run into any problems with evaluations, please file a GitHub Issue. - - -### LIBERO Simulation Benchmark Evaluations - -In the [updated OpenVLA paper (v2)](https://arxiv.org/abs/2406.09246), we discuss fine-tuning OpenVLA -on a simulated benchmark, [LIBERO](https://libero-project.github.io/main.html), in Appendix E. -Please see the paper for details, such as how we modify the provided demonstration datasets to -improve the overall performance of all methods. - -We copy the results to the section below and then discuss how to reproduce the results for OpenVLA. - -#### OpenVLA Fine-Tuning Results - -| Method | LIBERO-Spatial | LIBERO-Object | LIBERO-Goal | LIBERO-Long | Average | -|--------|----------------|---------------|-------------|-------------|---------| -| Diffusion Policy from scratch | 78.3 ± 1.1% | **92.5 ± 0.7%** | 68.3 ± 1.2% | 50.5 ± 1.3% | 72.4 ± 0.7% | -| Octo fine-tuned | 78.9 ± 1.0% | 85.7 ± 0.9% | **84.6 ± 0.9%** | 51.1 ± 1.3% | 75.1 ± 0.6% | -| OpenVLA fine-tuned (ours) | **84.7 ± 0.9%** | 88.4 ± 0.8% | 79.2 ± 1.0% | **53.7 ± 1.3%** | **76.5 ± 0.6%** | - -Each success rate is the average over 3 random seeds x 500 rollouts each (10 tasks x 50 rollouts per task). - -#### LIBERO Setup - -Clone and install the [LIBERO repo](https://github.com/Lifelong-Robot-Learning/LIBERO): - -```bash -git clone https://github.com/Lifelong-Robot-Learning/LIBERO.git -cd LIBERO -pip install -e . -``` - -Additionally, install other required packages: -```bash -cd openvla -pip install -r experiments/robot/libero/libero_requirements.txt -``` - -(Optional) To download the modified versions of the LIBERO datasets that we used in our fine-tuning -experiments, run the command below. This will download the LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, -and LIBERO-10 datasets in RLDS data format (~10 GB total). You can use these to fine-tune OpenVLA or -train other methods. This step is optional since we provide pretrained OpenVLA checkpoints below. -(Also, you can find the script we used to generate the modified datasets in raw HDF5 format -[here](experiments/robot/libero/regenerate_libero_dataset.py) and the code we used to convert these -datasets to the RLDS format [here](https://github.com/moojink/rlds_dataset_builder).) -```bash -git clone git@hf.co:datasets/openvla/modified_libero_rlds -``` - -#### Launching LIBERO Evaluations - -We fine-tuned OpenVLA via LoRA (r=32) on four LIBERO task suites independently: LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, and LIBERO-10 (also called LIBERO-Long). -The four checkpoints are available on Hugging Face: -* [openvla/openvla-7b-finetuned-libero-spatial](https://huggingface.co/openvla/openvla-7b-finetuned-libero-spatial) -* [openvla/openvla-7b-finetuned-libero-object](https://huggingface.co/openvla/openvla-7b-finetuned-libero-object) -* [openvla/openvla-7b-finetuned-libero-goal](https://huggingface.co/openvla/openvla-7b-finetuned-libero-goal) -* [openvla/openvla-7b-finetuned-libero-10](https://huggingface.co/openvla/openvla-7b-finetuned-libero-10) - -To start evaluation with one of these checkpoints, run one of the commands below. Each will automatically download the appropriate checkpoint listed above. - -```bash -# Launch LIBERO-Spatial evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-spatial \ - --task_suite_name libero_spatial \ - --center_crop True - -# Launch LIBERO-Object evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-object \ - --task_suite_name libero_object \ - --center_crop True - -# Launch LIBERO-Goal evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-goal \ - --task_suite_name libero_goal \ - --center_crop True - -# Launch LIBERO-10 (LIBERO-Long) evals -python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint openvla/openvla-7b-finetuned-libero-10 \ - --task_suite_name libero_10 \ - --center_crop True -``` - -Notes: -* The evaluation script will run 500 trials by default (10 tasks x 50 episodes each). You can modify the number of - trials per task by setting `--num_trials_per_task`. You can also change the random seed via `--seed`. -* **NOTE: Setting `--center_crop True` is important** because we fine-tuned OpenVLA with random crop augmentations - (we took a random crop with 90% area in every training sample, so at test time we simply take the center 90% crop). -* The evaluation script logs results locally. You can also log results in Weights & Biases - by setting `--use_wandb True` and specifying `--wandb_project ` and `--wandb_entity `. -* The results reported in our paper were obtained using **Python 3.10.13, PyTorch 2.2.0, transformers 4.40.1, and - flash-attn 2.5.5** on an **NVIDIA A100 GPU**, averaged over three random seeds. Please stick to these package versions. - Note that results may vary slightly if you use a different GPU for evaluation due to GPU nondeterminism in large models - (though we have tested that results were consistent across different machines with A100 GPUs). - -Please file a GitHub Issue if you run into any problems. - ---- - -## Repository Structure - -High-level overview of repository/project file-tree: - -+ `prismatic` - Package source; provides core utilities for model loading, training, data preprocessing, etc. -+ `vla-scripts/` - Core scripts for training, fine-tuning, and deploying VLAs. -+ `experiments/` - Code for evaluating OpenVLA policies in robot environments. -+ `LICENSE` - All code is made available under the MIT License; happy hacking! -+ `Makefile` - Top-level Makefile (by default, supports linting - checking & auto-fix); extend as needed. -+ `pyproject.toml` - Full project configuration details (including dependencies), as well as tool configurations. -+ `README.md` - You are here! - ---- - - -# VLA Performance Troubleshooting - -In this section we cover best practices for debugging poor VLA performance after fine-tuning on your target domain robot dataset. +See [SETUP.md](SETUP.md) for instructions on setting up the conda environment. -**Note**: OpenVLA typically requires fine-tuning on a small demonstration dataset (~100 demos) from your target domain robot. Out-of-the-box, it only works well on domains from the training dataset. +## Training and Evaluation -**Sanity checks**: -- replay the actions from a demonstration from your fine-tuning dataset and make sure that the robot can execute the task successfully (this ensures that your data collection pipeline is correct) -- once you fine-tuned a model, load the model in your inference pipeline (as if you would run it to control the robot), but feed images from the fine-tuning dataset into the model (pretending they come from the robot) and verify that you can reproduce the token accuracies / L1 errors from training (this ensures that your inference pipeline is correct) +See [LIBERO.md](LIBERO.md) for fine-tuning/evaluating on LIBERO simulation benchmark task suites. -**Best practices for fine-tuning data collection**: -If your setup passed the above two sanity checks, the issue may not be in model training, but in the data you fine-tuned the model with. Some best practices for data collection: -- *Collect at a control frequency around 5-10Hz.* OpenVLA is not trained with action chunking, empirically the model struggles with high-frequency data. If your robot setup uses a high-frequency controller (eg 50 Hz), consider downsampling your actions to 5Hz. Verify first that your robot can still solve the task when using 5Hz actions (ie repeat sanity check (1) above with 5Hz actions) -- *Avoid pauses / small actions during data collection.* Because OpenVLA is trained without action chunking, the model can be sensitive to idle actions in the fine-tuning data. If your data contains steps in which the robot barely moves, the model may "get stuck" in these steps at inference time. Try to collect fine-tuning demonstrations with continuous, slow movement. -- *Ensure sufficient data coverage.* If you plan to test the model with some variation, e.g. different initial positions of objects, make sure that your fine-tuning data contains sufficient diversity of such conditions as well, e.g. shows demonstrations with diverse initial conditions. -- *Use consistent task strategies during data collection.* This is not a hard constraint, but may make your life easier. Try to demonstrate tasks in consistent ways, e.g. approach objects from the same side, perform sub-steps in the same order even if they could be performed in arbitrary sequences. Being consistent gives you a less multi-modal fine-tuning dataset, which makes the modeling problem easier. +See [ALOHA.md](ALOHA.md) for fine-tuning/evaluating on real-world ALOHA robot tasks. +## Support ---- +If you run into any issues, please open a new GitHub issue. If you do not receive a response within 2 business days, please email Moo Jin Kim (moojink@cs.stanford.edu) to bring the issue to his attention. -#### Citation +## Citation -If you find our code or models useful in your work, please cite [our paper](https://arxiv.org/abs/2406.09246): +If you use our code in your work, please cite [our paper](TODO): ```bibtex -@article{kim24openvla, - title={OpenVLA: An Open-Source Vision-Language-Action Model}, - author={{Moo Jin} Kim and Karl Pertsch and Siddharth Karamcheti and Ted Xiao and Ashwin Balakrishna and Suraj Nair and Rafael Rafailov and Ethan Foster and Grace Lam and Pannag Sanketi and Quan Vuong and Thomas Kollar and Benjamin Burchfiel and Russ Tedrake and Dorsa Sadigh and Sergey Levine and Percy Liang and Chelsea Finn}, - journal = {arXiv preprint arXiv:2406.09246}, - year={2024} -} +TODO ``` diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 000000000..d4d7c72c7 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,24 @@ +# Setup Instructions + +## Set Up Conda Environment + +```bash +# Create and activate conda environment +conda create -n openvla-oft python=3.10 -y +conda activate openvla-oft + +# Install PyTorch +# Use a command specific to your machine: https://pytorch.org/get-started/locally/ +pip3 install torch torchvision torchaudio + +# Clone openvla-oft repo and pip install to download dependencies +git clone https://github.com/moojink/openvla-oft.git +cd openvla-oft +pip install -e . + +# Install Flash Attention 2 for training (https://github.com/Dao-AILab/flash-attention) +# =>> If you run into difficulty, try `pip cache remove flash_attn` first +pip install packaging ninja +ninja --version; echo $? # Verify Ninja --> should return exit code "0" +pip install "flash-attn==2.5.5" --no-build-isolation +``` \ No newline at end of file diff --git a/experiments/robot/aloha/aloha_utils.py b/experiments/robot/aloha/aloha_utils.py new file mode 100644 index 000000000..7acb0f24e --- /dev/null +++ b/experiments/robot/aloha/aloha_utils.py @@ -0,0 +1,85 @@ +"""Utils for evaluating policies in real-world ALOHA environments.""" + +import os + +import imageio +import numpy as np +from PIL import Image + +from experiments.robot.aloha.real_env import make_real_env +from experiments.robot.robot_utils import ( + DATE, + DATE_TIME, +) + + +def get_next_task_label(task_label): + """Prompt the user to input the next task.""" + if task_label == "": + user_input = "" + while user_input == "": + user_input = input("Enter the task name: ") + task_label = user_input + else: + user_input = input("Enter the task name (or leave blank to repeat the previous task): ") + if user_input == "": + pass # Do nothing -> Let task_label be the same + else: + task_label = user_input + print(f"Task: {task_label}") + return task_label + + +def get_aloha_env(): + """Initializes and returns the ALOHA environment.""" + env = make_real_env(init_node=True) + return env + + +def resize_image_for_preprocessing(img): + """ + Takes numpy array corresponding to a single image and resizes to 256x256, exactly as done + in the ALOHA data preprocessing script, which is used before converting the dataset to RLDS. + """ + ALOHA_PREPROCESS_SIZE = 256 + img = np.array( + Image.fromarray(img).resize((ALOHA_PREPROCESS_SIZE, ALOHA_PREPROCESS_SIZE), resample=Image.BICUBIC) + ) # BICUBIC is default; specify explicitly to make it clear + return img + + +def get_aloha_image(obs): + """Extracts third-person image from observations and preprocesses it.""" + # obs: dm_env._environment.TimeStep + img = obs.observation["images"]["cam_high"] + img = resize_image_for_preprocessing(img) + return img + + +def get_aloha_wrist_images(obs): + """Extracts both wrist camera images from observations and preprocesses them.""" + # obs: dm_env._environment.TimeStep + left_wrist_img = obs.observation["images"]["cam_left_wrist"] + right_wrist_img = obs.observation["images"]["cam_right_wrist"] + left_wrist_img = resize_image_for_preprocessing(left_wrist_img) + right_wrist_img = resize_image_for_preprocessing(right_wrist_img) + return left_wrist_img, right_wrist_img + + +def save_rollout_video(rollout_images, idx, success, task_description, log_file=None, notes=None): + """Saves an MP4 replay of an episode.""" + rollout_dir = f"./rollouts/{DATE}" + os.makedirs(rollout_dir, exist_ok=True) + processed_task_description = task_description.lower().replace(" ", "_").replace("\n", "_").replace(".", "_")[:50] + filetag = f"{rollout_dir}/{DATE_TIME}--openvla--episode={idx}--success={success}--task={processed_task_description}" + if notes is not None: + filetag += f"--{notes}" + mp4_path = f"{filetag}.mp4" + video_writer = imageio.get_writer(mp4_path, fps=25) + for img in rollout_images: + video_writer.append_data(img) + video_writer.close() + print(f"Saved rollout MP4 at path {mp4_path}") + if log_file is not None: + log_file.write(f"Saved rollout MP4 at path {mp4_path}\n") + return mp4_path diff --git a/experiments/robot/aloha/constants.py b/experiments/robot/aloha/constants.py new file mode 100644 index 000000000..20cf90099 --- /dev/null +++ b/experiments/robot/aloha/constants.py @@ -0,0 +1,52 @@ +### Task parameters + +DATA_DIR = '' +TASK_CONFIGS = { + 'aloha_wear_shoe':{ + 'dataset_dir': DATA_DIR + '/aloha_wear_shoe', + 'num_episodes': 50, + 'episode_len': 1000, + 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + }, +} + +### ALOHA fixed constants +DT = 0.02 +JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] +START_ARM_POSE = [0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239, 0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239] + +# Left finger position limits (qpos[7]), right_finger = -1 * left_finger +MASTER_GRIPPER_POSITION_OPEN = 0.02417 +MASTER_GRIPPER_POSITION_CLOSE = 0.01244 +PUPPET_GRIPPER_POSITION_OPEN = 0.05800 +PUPPET_GRIPPER_POSITION_CLOSE = 0.01844 + +# Gripper joint limits (qpos[6]) +MASTER_GRIPPER_JOINT_OPEN = 0.3083 +MASTER_GRIPPER_JOINT_CLOSE = -0.6842 +PUPPET_GRIPPER_JOINT_OPEN = 1.4910 +PUPPET_GRIPPER_JOINT_CLOSE = -0.6213 + +############################ Helper functions ############################ + +MASTER_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_POSITION_CLOSE) / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_POSITION_CLOSE) / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) +MASTER_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) + MASTER_GRIPPER_POSITION_CLOSE +PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + PUPPET_GRIPPER_POSITION_CLOSE +MASTER2PUPPET_POSITION_FN = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(MASTER_GRIPPER_POSITION_NORMALIZE_FN(x)) + +MASTER_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) +PUPPET_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) +MASTER_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +MASTER2PUPPET_JOINT_FN = lambda x: PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(MASTER_GRIPPER_JOINT_NORMALIZE_FN(x)) + +MASTER_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + +MASTER_POS2JOINT = lambda x: MASTER_GRIPPER_POSITION_NORMALIZE_FN(x) * (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +MASTER_JOINT2POS = lambda x: MASTER_GRIPPER_POSITION_UNNORMALIZE_FN((x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE)) +PUPPET_POS2JOINT = lambda x: PUPPET_GRIPPER_POSITION_NORMALIZE_FN(x) * (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +PUPPET_JOINT2POS = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN((x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE)) + +MASTER_GRIPPER_JOINT_MID = (MASTER_GRIPPER_JOINT_OPEN + MASTER_GRIPPER_JOINT_CLOSE)/2 \ No newline at end of file diff --git a/experiments/robot/aloha/preprocess_split_aloha_data.py b/experiments/robot/aloha/preprocess_split_aloha_data.py new file mode 100644 index 000000000..8de07f232 --- /dev/null +++ b/experiments/robot/aloha/preprocess_split_aloha_data.py @@ -0,0 +1,260 @@ +""" +Preprocesses ALOHA dataset(s) and splits them into train/val sets. + +Preprocessing includes downsizing images from 480x640 to 256x256. +Splits happen at the episode level (not step level), which means that +an episode is treated as an atomic unit that entirely goes to either +the train set or val set. + +Original ALOHA data layout: + /PATH/TO/DATASET/dataset_name/ + - episode_0.hdf5 + - episode_1.hdf5 + - ... + - episode_N.hdf5 + +Preprocessed data layout (after running this script): + /PATH/TO/PREPROCESSED_DATASETS/dataset_name/ + - train/ + - episode_0.hdf5 + - episode_1.hdf5 + - ... + - episode_M.hdf5 + - val/ + - episode_0.hdf5 + - episode_1.hdf5 + - ... + - episode_K.hdf5 + + where N > M > K + +Example usage: + # "put X into pot" task + python experiments/robot/aloha/preprocess_split_aloha_data.py \ + --dataset_path /scr/moojink/data/aloha1_raw/put_green_pepper_into_pot/ \ + --out_base_dir /scr/moojink/data/aloha1_preprocessed/ \ + --percent_val 0.05 && \ + python experiments/robot/aloha/preprocess_split_aloha_data.py \ + --dataset_path /scr/moojink/data/aloha1_raw/put_red_pepper_into_pot/ \ + --out_base_dir /scr/moojink/data/aloha1_preprocessed/ \ + --percent_val 0.05 && \ + python experiments/robot/aloha/preprocess_split_aloha_data.py \ + --dataset_path /scr/moojink/data/aloha1_raw/put_yellow_corn_into_pot/ \ + --out_base_dir /scr/moojink/data/aloha1_preprocessed/ \ + --percent_val 0.05 +""" + +import argparse +import glob +import os +import random + +import h5py +import numpy as np +from PIL import Image +from tqdm import tqdm + + +def load_hdf5(demo_path): + """Loads single episode.""" + if not os.path.isfile(demo_path): + print(f"Dataset does not exist at \n{demo_path}\n") + exit() + + print(f"Loading {demo_path}...") + with h5py.File(demo_path, "r") as root: + is_sim = root.attrs["sim"] + qpos = root["/observations/qpos"][()] + qvel = root["/observations/qvel"][()] + effort = root["/observations/effort"][()] + action = root["/action"][()] + image_dict = dict() + for cam_name in root["/observations/images/"].keys(): + image_dict[cam_name] = root[f"/observations/images/{cam_name}"][()] + print(f"Loading episode complete: {demo_path}") + + return qpos, qvel, effort, action, image_dict, is_sim + + +def load_and_preprocess_all_episodes(demo_paths, out_dataset_dir): + """ + Loads and preprocesses all episodes. + Resizes all images in one episode before loading the next, to reduce memory usage. + """ + cam_names = ["cam_high", "cam_left_wrist", "cam_right_wrist"] + idx = 0 + for demo in tqdm(demo_paths): + qpos, qvel, effort, action, image_dict, is_sim = load_hdf5(demo) + # Save non-image info + episode_len = image_dict["cam_high"].shape[0] + # Resize all images + print("Resizing images in episode...") + for k in cam_names: + resized_images = [] + for i in range(episode_len): + resized_images.append( + np.array( + Image.fromarray(image_dict[k][i]).resize( + (args.img_resize_size, args.img_resize_size), resample=Image.BICUBIC + ) + ) # BICUBIC is default; specify explicitly to make it clear + ) + image_dict[k] = np.stack(resized_images) + print("Resizing images in episode complete!") + # Save preprocessed episode + data_dict = dict( + qpos=qpos, + qvel=qvel, + effort=effort, + action=action, + image_dict=image_dict, + is_sim=is_sim, + ) + save_new_hdf5(out_dataset_dir, data_dict, idx) + idx += 1 + + +def randomly_split(full_qpos, full_qvel, full_effort, full_action, full_image_dict, percent_val): + """Randomly splits dataset into train and validation sets.""" + # Create a list of episode indices + num_episodes_total = len(full_qpos) + indices = list(range(num_episodes_total)) + # Shuffle the episode indices + random.shuffle(indices) + # Create new lists using the shuffled indices + shuffled_qpos = [full_qpos[idx] for idx in indices] + shuffled_qvel = [full_qvel[idx] for idx in indices] + shuffled_effort = [full_effort[idx] for idx in indices] + shuffled_action = [full_action[idx] for idx in indices] + shuffled_image_dict = { + "cam_high": [], + "cam_left_wrist": [], + "cam_right_wrist": [], + } + for k in full_image_dict.keys(): + shuffled_image_dict[k] = [full_image_dict[k][idx] for idx in indices] + # Split into train and val sets + num_episodes_val = int(num_episodes_total * percent_val) + print(f"Total # steps: {num_episodes_total}; using {num_episodes_val} ({percent_val:.2f}%) for val set") + num_episodes_train = num_episodes_total - num_episodes_val + train_dict = dict( + qpos=shuffled_qpos[:num_episodes_train], + qvel=shuffled_qvel[:num_episodes_train], + effort=shuffled_effort[:num_episodes_train], + action=shuffled_action[:num_episodes_train], + image_dict=dict( + cam_high=shuffled_image_dict["cam_high"][:num_episodes_train], + cam_left_wrist=shuffled_image_dict["cam_left_wrist"][:num_episodes_train], + cam_right_wrist=shuffled_image_dict["cam_right_wrist"][:num_episodes_train], + ), + ) + val_dict = dict( + qpos=shuffled_qpos[num_episodes_train:], + qvel=shuffled_qvel[num_episodes_train:], + effort=shuffled_effort[num_episodes_train:], + action=shuffled_action[num_episodes_train:], + image_dict=dict( + cam_high=shuffled_image_dict["cam_high"][num_episodes_train:], + cam_left_wrist=shuffled_image_dict["cam_left_wrist"][num_episodes_train:], + cam_right_wrist=shuffled_image_dict["cam_right_wrist"][num_episodes_train:], + ), + ) + return train_dict, val_dict + + +def save_new_hdf5(out_dataset_dir, data_dict, episode_idx): + """Saves an HDF5 file for a new episode.""" + camera_names = data_dict["image_dict"].keys() + H, W, C = data_dict["image_dict"]["cam_high"][0].shape + out_path = os.path.join(out_dataset_dir, f"episode_{episode_idx}.hdf5") + # Save HDF5 with same structure as original demos (except that now we combine all episodes into one HDF5 file) + with h5py.File( + out_path, "w", rdcc_nbytes=1024**2 * 2 + ) as root: # Magic constant for rdcc_nbytes comes from ALOHA codebase + episode_len = data_dict["qpos"].shape[0] + root.attrs["sim"] = data_dict["is_sim"] + obs = root.create_group("observations") + _ = obs.create_dataset("qpos", (episode_len, 14)) + _ = obs.create_dataset("qvel", (episode_len, 14)) + _ = obs.create_dataset("effort", (episode_len, 14)) + root["/observations/qpos"][...] = data_dict["qpos"] + root["/observations/qvel"][...] = data_dict["qvel"] + root["/observations/effort"][...] = data_dict["effort"] + image = obs.create_group("images") + for cam_name in camera_names: + _ = image.create_dataset( + cam_name, + (episode_len, H, W, C), + dtype="uint8", + chunks=(1, H, W, C), + ) + root[f"/observations/images/{cam_name}"][...] = data_dict["image_dict"][cam_name] + _ = root.create_dataset("action", (episode_len, 14)) + root["/action"][...] = data_dict["action"] + # Compute and save *relative* actions as well + actions = data_dict["action"] + relative_actions = np.zeros_like(actions) + relative_actions[:-1] = actions[1:] - actions[:-1] # Relative actions are the changes in joint pos + relative_actions[-1] = relative_actions[-2] # Just copy the second-to-last action for the last action + _ = root.create_dataset("relative_action", (episode_len, 14)) + root["/relative_action"][...] = relative_actions + print(f"Saved dataset: {out_path}") + + +def main(args): + # Create directory to save preprocessed dataset (if it doesn't exist already) + os.makedirs(args.out_base_dir, exist_ok=True) + out_dataset_dir = os.path.join(args.out_base_dir, os.path.basename(args.dataset_path.rstrip("/"))) + os.makedirs(out_dataset_dir, exist_ok=True) + # Get list of filepaths of all episodes + all_demo_paths = glob.glob(os.path.join(args.dataset_path, "*.hdf5")) # List of HDF5 filepaths + all_demo_paths.sort() + # Create a list of episode indices + num_episodes_total = len(all_demo_paths) + indices = list(range(num_episodes_total)) + # Shuffle the episode indices + random.shuffle(indices) + # Split into train and val sets + num_episodes_val = int(num_episodes_total * args.percent_val) + print(f"Total # episodes: {num_episodes_total}; using {num_episodes_val} ({args.percent_val:.2f}%) for val set") + num_episodes_train = num_episodes_total - num_episodes_val + train_indices = indices[:num_episodes_train] + val_indices = indices[num_episodes_train:] + train_demo_paths = [all_demo_paths[i] for i in train_indices] + val_demo_paths = [all_demo_paths[i] for i in val_indices] + # Preprocess all episodes and save the result + out_dataset_dir_train = os.path.join(out_dataset_dir, "train") + out_dataset_dir_val = os.path.join(out_dataset_dir, "val") + os.makedirs(out_dataset_dir_train, exist_ok=True) + os.makedirs(out_dataset_dir_val, exist_ok=True) + load_and_preprocess_all_episodes(train_demo_paths, out_dataset_dir_train) + load_and_preprocess_all_episodes(val_demo_paths, out_dataset_dir_val) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_path", + required=True, + help="Path to raw ALOHA dataset directory. Example: /PATH/TO/USER/data/aloha_raw/put_green_pepper_into_pot/", + ) + parser.add_argument( + "--out_base_dir", + required=True, + help="Path to directory in which to save preprocessed dataset. Example: /PATH/TO/USER/data/aloha_preprocessed/", + ) + parser.add_argument( + "--percent_val", + type=float, + help="Percent of dataset to use as validation set (measured in episodes, not steps).", + default=0.05, + ) + parser.add_argument( + "--img_resize_size", + type=int, + help="Size to resize images to. Final images will be square (img_resize_size x img_resize_size pixels).", + default=256, + ) + args = parser.parse_args() + + main(args) diff --git a/experiments/robot/aloha/real_env.py b/experiments/robot/aloha/real_env.py new file mode 100644 index 000000000..f3f6c8f54 --- /dev/null +++ b/experiments/robot/aloha/real_env.py @@ -0,0 +1,213 @@ +import time +import numpy as np +import collections +import matplotlib.pyplot as plt +import dm_env + +from experiments.robot.aloha.constants import DT, START_ARM_POSE, MASTER_GRIPPER_JOINT_NORMALIZE_FN, PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN +from experiments.robot.aloha.constants import PUPPET_GRIPPER_POSITION_NORMALIZE_FN, PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN +from experiments.robot.aloha.constants import PUPPET_GRIPPER_JOINT_OPEN, PUPPET_GRIPPER_JOINT_CLOSE +from experiments.robot.aloha.robot_utils import Recorder, ImageRecorder +from experiments.robot.aloha.robot_utils import setup_master_bot, setup_puppet_bot, move_arms, move_grippers +from interbotix_xs_modules.arm import InterbotixManipulatorXS +from interbotix_xs_msgs.msg import JointSingleCommand + +import IPython +e = IPython.embed + +class RealEnv: + """ + Environment for real robot bi-manual manipulation + Action space: [left_arm_qpos (6), # absolute joint position + left_gripper_positions (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_positions (1),] # normalized gripper position (0: close, 1: open) + + Observation space: {"qpos": Concat[ left_arm_qpos (6), # absolute joint position + left_gripper_position (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_qpos (1)] # normalized gripper position (0: close, 1: open) + "qvel": Concat[ left_arm_qvel (6), # absolute joint velocity (rad) + left_gripper_velocity (1), # normalized gripper velocity (pos: opening, neg: closing) + right_arm_qvel (6), # absolute joint velocity (rad) + right_gripper_qvel (1)] # normalized gripper velocity (pos: opening, neg: closing) + "images": {"cam_high": (480x640x3), # h, w, c, dtype='uint8' + "cam_low": (480x640x3), # h, w, c, dtype='uint8' + "cam_left_wrist": (480x640x3), # h, w, c, dtype='uint8' + "cam_right_wrist": (480x640x3)} # h, w, c, dtype='uint8' + """ + + def __init__(self, init_node, setup_robots=True): + self.puppet_bot_left = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", + robot_name=f'puppet_left', init_node=init_node) + self.puppet_bot_right = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", + robot_name=f'puppet_right', init_node=False) + if setup_robots: + self.setup_robots() + + self.recorder_left = Recorder('left', init_node=False) + self.recorder_right = Recorder('right', init_node=False) + self.image_recorder = ImageRecorder(init_node=False) + self.gripper_command = JointSingleCommand(name="gripper") + + def setup_robots(self): + setup_puppet_bot(self.puppet_bot_left) + setup_puppet_bot(self.puppet_bot_right) + + def get_qpos(self): + left_qpos_raw = self.recorder_left.qpos + right_qpos_raw = self.recorder_right.qpos + left_arm_qpos = left_qpos_raw[:6] + right_arm_qpos = right_qpos_raw[:6] + left_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(left_qpos_raw[7])] # this is position not joint + right_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(right_qpos_raw[7])] # this is position not joint + return np.concatenate([left_arm_qpos, left_gripper_qpos, right_arm_qpos, right_gripper_qpos]) + + def get_qvel(self): + left_qvel_raw = self.recorder_left.qvel + right_qvel_raw = self.recorder_right.qvel + left_arm_qvel = left_qvel_raw[:6] + right_arm_qvel = right_qvel_raw[:6] + left_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(left_qvel_raw[7])] + right_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(right_qvel_raw[7])] + return np.concatenate([left_arm_qvel, left_gripper_qvel, right_arm_qvel, right_gripper_qvel]) + + def get_effort(self): + left_effort_raw = self.recorder_left.effort + right_effort_raw = self.recorder_right.effort + left_robot_effort = left_effort_raw[:7] + right_robot_effort = right_effort_raw[:7] + return np.concatenate([left_robot_effort, right_robot_effort]) + + def get_images(self): + return self.image_recorder.get_images() + + def set_gripper_pose(self, left_gripper_desired_pos_normalized, right_gripper_desired_pos_normalized): + left_gripper_desired_joint = PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(left_gripper_desired_pos_normalized) + self.gripper_command.cmd = left_gripper_desired_joint + self.puppet_bot_left.gripper.core.pub_single.publish(self.gripper_command) + + right_gripper_desired_joint = PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(right_gripper_desired_pos_normalized) + self.gripper_command.cmd = right_gripper_desired_joint + self.puppet_bot_right.gripper.core.pub_single.publish(self.gripper_command) + + def _reset_joints(self): + reset_position = START_ARM_POSE[:6] + move_arms([self.puppet_bot_left, self.puppet_bot_right], [reset_position, reset_position], move_time=1) + + def _reset_gripper(self): + """Set to position mode and do position resets: first open then close. Then change back to PWM mode""" + move_grippers([self.puppet_bot_left, self.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) + move_grippers([self.puppet_bot_left, self.puppet_bot_right], [PUPPET_GRIPPER_JOINT_CLOSE] * 2, move_time=1) + + def _get_obs(self): + obs = collections.OrderedDict() + obs['qpos'] = self.get_qpos() + obs['qvel'] = self.get_qvel() + obs['effort'] = self.get_effort() + obs['images'] = self.get_images() + return obs + + def get_observation(self, t=0): + step_type = dm_env.StepType.FIRST if t == 0 else dm_env.StepType.MID + return dm_env.TimeStep( + step_type=step_type, + reward=self.get_reward(), + discount=None, + observation=self._get_obs() + ) + + def get_reward(self): + return 0 + + def reset(self, fake=False): + if not fake: + # Reboot puppet robot gripper motors + self.puppet_bot_left.dxl.robot_reboot_motors("single", "gripper", True) + self.puppet_bot_right.dxl.robot_reboot_motors("single", "gripper", True) + self._reset_joints() + self._reset_gripper() + return dm_env.TimeStep( + step_type=dm_env.StepType.FIRST, + reward=self.get_reward(), + discount=None, + observation=self._get_obs()) + + def step(self, action): + state_len = int(len(action) / 2) + left_action = action[:state_len] + right_action = action[state_len:] + self.puppet_bot_left.arm.set_joint_positions(left_action[:6], blocking=False) + self.puppet_bot_right.arm.set_joint_positions(right_action[:6], blocking=False) + self.set_gripper_pose(left_action[-1], right_action[-1]) + time.sleep(DT) + return dm_env.TimeStep( + step_type=dm_env.StepType.MID, + reward=self.get_reward(), + discount=None, + observation=self._get_obs()) + + +def get_action(master_bot_left, master_bot_right): + action = np.zeros(14) # 6 joint + 1 gripper, for two arms + # Arm actions + action[:6] = master_bot_left.dxl.joint_states.position[:6] + action[7:7+6] = master_bot_right.dxl.joint_states.position[:6] + # Gripper actions + action[6] = MASTER_GRIPPER_JOINT_NORMALIZE_FN(master_bot_left.dxl.joint_states.position[6]) + action[7+6] = MASTER_GRIPPER_JOINT_NORMALIZE_FN(master_bot_right.dxl.joint_states.position[6]) + + return action + + +def make_real_env(init_node, setup_robots=True): + env = RealEnv(init_node, setup_robots) + return env + + +def test_real_teleop(): + """ + Test bimanual teleoperation and show image observations onscreen. + It first reads joint poses from both master arms. + Then use it as actions to step the environment. + The environment returns full observations including images. + + An alternative approach is to have separate scripts for teleoperation and observation recording. + This script will result in higher fidelity (obs, action) pairs + """ + + onscreen_render = True + render_cam = 'cam_left_wrist' + + # source of data + master_bot_left = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", + robot_name=f'master_left', init_node=True) + master_bot_right = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", + robot_name=f'master_right', init_node=False) + setup_master_bot(master_bot_left) + setup_master_bot(master_bot_right) + + # setup the environment + env = make_real_env(init_node=False) + ts = env.reset(fake=True) + episode = [ts] + # setup visualization + if onscreen_render: + ax = plt.subplot() + plt_img = ax.imshow(ts.observation['images'][render_cam]) + plt.ion() + + for t in range(1000): + action = get_action(master_bot_left, master_bot_right) + ts = env.step(action) + episode.append(ts) + + if onscreen_render: + plt_img.set_data(ts.observation['images'][render_cam]) + plt.pause(DT) + else: + time.sleep(DT) + + +if __name__ == '__main__': + test_real_teleop() diff --git a/experiments/robot/aloha/robot_utils.py b/experiments/robot/aloha/robot_utils.py new file mode 100644 index 000000000..82f080cc9 --- /dev/null +++ b/experiments/robot/aloha/robot_utils.py @@ -0,0 +1,187 @@ +import numpy as np +import time +from experiments.robot.aloha.constants import DT +from interbotix_xs_msgs.msg import JointSingleCommand + +import IPython +e = IPython.embed + +class ImageRecorder: + def __init__(self, init_node=True, is_debug=False): + from collections import deque + import rospy + from cv_bridge import CvBridge + from sensor_msgs.msg import Image + self.is_debug = is_debug + self.bridge = CvBridge() + self.camera_names = ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + if init_node: + rospy.init_node('image_recorder', anonymous=True) + for cam_name in self.camera_names: + setattr(self, f'{cam_name}_image', None) + setattr(self, f'{cam_name}_secs', None) + setattr(self, f'{cam_name}_nsecs', None) + if cam_name == 'cam_high': + callback_func = self.image_cb_cam_high + elif cam_name == 'cam_low': + callback_func = self.image_cb_cam_low + elif cam_name == 'cam_left_wrist': + callback_func = self.image_cb_cam_left_wrist + elif cam_name == 'cam_right_wrist': + callback_func = self.image_cb_cam_right_wrist + else: + raise NotImplementedError + rospy.Subscriber(f"/usb_{cam_name}/image_raw", Image, callback_func) + if self.is_debug: + setattr(self, f'{cam_name}_timestamps', deque(maxlen=50)) + time.sleep(0.5) + + def image_cb(self, cam_name, data): + setattr(self, f'{cam_name}_image', self.bridge.imgmsg_to_cv2(data, desired_encoding='passthrough')) + setattr(self, f'{cam_name}_secs', data.header.stamp.secs) + setattr(self, f'{cam_name}_nsecs', data.header.stamp.nsecs) + # cv2.imwrite('/home/tonyzhao/Desktop/sample.jpg', cv_image) + if self.is_debug: + getattr(self, f'{cam_name}_timestamps').append(data.header.stamp.secs + data.header.stamp.secs * 1e-9) + + def image_cb_cam_high(self, data): + cam_name = 'cam_high' + return self.image_cb(cam_name, data) + + def image_cb_cam_low(self, data): + cam_name = 'cam_low' + return self.image_cb(cam_name, data) + + def image_cb_cam_left_wrist(self, data): + cam_name = 'cam_left_wrist' + return self.image_cb(cam_name, data) + + def image_cb_cam_right_wrist(self, data): + cam_name = 'cam_right_wrist' + return self.image_cb(cam_name, data) + + def get_images(self): + image_dict = dict() + for cam_name in self.camera_names: + image_dict[cam_name] = getattr(self, f'{cam_name}_image') + return image_dict + + def print_diagnostics(self): + def dt_helper(l): + l = np.array(l) + diff = l[1:] - l[:-1] + return np.mean(diff) + for cam_name in self.camera_names: + image_freq = 1 / dt_helper(getattr(self, f'{cam_name}_timestamps')) + print(f'{cam_name} {image_freq=:.2f}') + print() + +class Recorder: + def __init__(self, side, init_node=True, is_debug=False): + from collections import deque + import rospy + from sensor_msgs.msg import JointState + from interbotix_xs_msgs.msg import JointGroupCommand, JointSingleCommand + + self.secs = None + self.nsecs = None + self.qpos = None + self.effort = None + self.arm_command = None + self.gripper_command = None + self.is_debug = is_debug + + if init_node: + rospy.init_node('recorder', anonymous=True) + rospy.Subscriber(f"/puppet_{side}/joint_states", JointState, self.puppet_state_cb) + rospy.Subscriber(f"/puppet_{side}/commands/joint_group", JointGroupCommand, self.puppet_arm_commands_cb) + rospy.Subscriber(f"/puppet_{side}/commands/joint_single", JointSingleCommand, self.puppet_gripper_commands_cb) + if self.is_debug: + self.joint_timestamps = deque(maxlen=50) + self.arm_command_timestamps = deque(maxlen=50) + self.gripper_command_timestamps = deque(maxlen=50) + time.sleep(0.1) + + def puppet_state_cb(self, data): + self.qpos = data.position + self.qvel = data.velocity + self.effort = data.effort + self.data = data + if self.is_debug: + self.joint_timestamps.append(time.time()) + + def puppet_arm_commands_cb(self, data): + self.arm_command = data.cmd + if self.is_debug: + self.arm_command_timestamps.append(time.time()) + + def puppet_gripper_commands_cb(self, data): + self.gripper_command = data.cmd + if self.is_debug: + self.gripper_command_timestamps.append(time.time()) + + def print_diagnostics(self): + def dt_helper(l): + l = np.array(l) + diff = l[1:] - l[:-1] + return np.mean(diff) + + joint_freq = 1 / dt_helper(self.joint_timestamps) + arm_command_freq = 1 / dt_helper(self.arm_command_timestamps) + gripper_command_freq = 1 / dt_helper(self.gripper_command_timestamps) + + print(f'{joint_freq=:.2f}\n{arm_command_freq=:.2f}\n{gripper_command_freq=:.2f}\n') + +def get_arm_joint_positions(bot): + return bot.arm.core.joint_states.position[:6] + +def get_arm_gripper_positions(bot): + joint_position = bot.gripper.core.joint_states.position[6] + return joint_position + +def move_arms(bot_list, target_pose_list, move_time=1): + num_steps = int(move_time / DT) + curr_pose_list = [get_arm_joint_positions(bot) for bot in bot_list] + traj_list = [np.linspace(curr_pose, target_pose, num_steps) for curr_pose, target_pose in zip(curr_pose_list, target_pose_list)] + for t in range(num_steps): + for bot_id, bot in enumerate(bot_list): + bot.arm.set_joint_positions(traj_list[bot_id][t], blocking=False) + time.sleep(DT) + +def move_grippers(bot_list, target_pose_list, move_time): + gripper_command = JointSingleCommand(name="gripper") + num_steps = int(move_time / DT) + curr_pose_list = [get_arm_gripper_positions(bot) for bot in bot_list] + traj_list = [np.linspace(curr_pose, target_pose, num_steps) for curr_pose, target_pose in zip(curr_pose_list, target_pose_list)] + for t in range(num_steps): + for bot_id, bot in enumerate(bot_list): + gripper_command.cmd = traj_list[bot_id][t] + bot.gripper.core.pub_single.publish(gripper_command) + time.sleep(DT) + +def setup_puppet_bot(bot): + bot.dxl.robot_reboot_motors("single", "gripper", True) + bot.dxl.robot_set_operating_modes("group", "arm", "position") + bot.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + torque_on(bot) + +def setup_master_bot(bot): + bot.dxl.robot_set_operating_modes("group", "arm", "pwm") + bot.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + torque_off(bot) + +def set_standard_pid_gains(bot): + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_P_Gain', 800) + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_I_Gain', 0) + +def set_low_pid_gains(bot): + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_P_Gain', 100) + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_I_Gain', 0) + +def torque_off(bot): + bot.dxl.robot_torque_enable("group", "arm", False) + bot.dxl.robot_torque_enable("single", "gripper", False) + +def torque_on(bot): + bot.dxl.robot_torque_enable("group", "arm", True) + bot.dxl.robot_torque_enable("single", "gripper", True) \ No newline at end of file diff --git a/experiments/robot/aloha/run_aloha_eval.py b/experiments/robot/aloha/run_aloha_eval.py new file mode 100644 index 000000000..177883487 --- /dev/null +++ b/experiments/robot/aloha/run_aloha_eval.py @@ -0,0 +1,384 @@ +""" +run_aloha_eval.py + +Evaluates a model in a real-world ALOHA environment. +""" + +import logging +import os +import socket +import sys +import time +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Union + +import draccus +import tqdm + +# Append current directory so that interpreter can find experiments.robot +sys.path.append(".") +from experiments.robot.aloha.aloha_utils import ( + get_aloha_env, + get_aloha_image, + get_aloha_wrist_images, + get_next_task_label, + save_rollout_video, +) +from experiments.robot.openvla_utils import ( + get_action_from_server, + resize_image_for_policy, +) +from experiments.robot.robot_utils import ( + DATE_TIME, + get_image_resize_size, + set_seed_everywhere, +) + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger(__name__) + + +@dataclass +class GenerateConfig: + # fmt: off + + ################################################################################################################# + # Model-specific parameters + ################################################################################################################# + model_family: str = "openvla" # Model family + + center_crop: bool = True # Center crop? (if trained w/ random crop image aug) + num_open_loop_steps: int = 25 # Number of actions to execute open-loop before requerying policy + + use_vla_server: bool = True # Whether to query remote VLA server for actions + vla_server_url: Union[str, Path] = "" # Remote VLA server URL (set to 127.0.0.1 if on same machine) + + ################################################################################################################# + # ALOHA environment-specific parameters + ################################################################################################################# + num_rollouts_planned: int = 50 # Number of test rollouts + max_steps: int = 1500 # Max number of steps per rollout + use_relative_actions: bool = False # Whether to use relative actions (delta joint angles) + + ################################################################################################################# + # Utils + ################################################################################################################# + run_id_note: Optional[str] = None # Extra note to add to end of run ID for logging + local_log_dir: str = "./experiments/logs" # Local directory for eval logs + + seed: int = 7 # Random Seed (for reproducibility) + + # fmt: on + + +def validate_config(cfg: GenerateConfig) -> None: + """Validate configuration parameters.""" + assert cfg.use_vla_server, ( + "Must use VLA server (server-client interface) to query model and get actions! Please set --use_vla_server=True" + ) + + +def setup_logging(cfg: GenerateConfig): + """Set up logging to file.""" + # Create run ID + run_id = f"EVAL-{cfg.model_family}-{DATE_TIME}" + if cfg.run_id_note is not None: + run_id += f"--{cfg.run_id_note}" + + # Set up local logging + os.makedirs(cfg.local_log_dir, exist_ok=True) + local_log_filepath = os.path.join(cfg.local_log_dir, run_id + ".txt") + log_file = open(local_log_filepath, "w") + logger.info(f"Logging to local log file: {local_log_filepath}") + + return log_file, local_log_filepath, run_id + + +def log_message(message: str, log_file=None): + """Log a message to console and optionally to a log file.""" + logger.info(message) + if log_file: + log_file.write(message + "\n") + log_file.flush() + + +def get_server_endpoint(cfg: GenerateConfig): + """Get the server endpoint for remote inference.""" + ip_address = socket.gethostbyname(cfg.vla_server_url) + return f"http://{ip_address}:8777/act" + + +def prepare_observation(obs, resize_size): + """Prepare observation for policy input.""" + # Get preprocessed images + img = get_aloha_image(obs) + left_wrist_img, right_wrist_img = get_aloha_wrist_images(obs) + + # Resize images to size expected by model + img_resized = resize_image_for_policy(img, resize_size) + left_wrist_img_resized = resize_image_for_policy(left_wrist_img, resize_size) + right_wrist_img_resized = resize_image_for_policy(right_wrist_img, resize_size) + + # Prepare observations dict + observation = { + "full_image": img_resized, + "left_wrist_image": left_wrist_img_resized, + "right_wrist_image": right_wrist_img_resized, + "state": obs.observation["qpos"], + } + + return observation, img_resized, left_wrist_img_resized, right_wrist_img_resized + + +def run_episode( + cfg: GenerateConfig, + env, + task_description: str, + server_endpoint: str, + resize_size, + log_file=None, +): + """Run a single episode in the ALOHA environment.""" + # Define control frequency + STEP_DURATION_IN_SEC = 1.0 / 25.0 + + # Reset environment + obs = env.reset() + + # Initialize action queue + action_queue = deque(maxlen=cfg.num_open_loop_steps) + + # Setup + t = 0 + curr_state = None + replay_images = [] + replay_images_resized = [] + replay_images_left_wrist_resized = [] + replay_images_right_wrist_resized = [] + + log_message("Prepare the scene, and then press Enter to begin...", log_file) + input() + + # Reset environment again to fetch first timestep observation + obs = env.reset() + + # Fetch initial robot state (but sleep first so that robot stops moving) + time.sleep(2) + curr_state = env.get_qpos() + + episode_start_time = time.time() + total_model_query_time = 0.0 + + try: + while t < cfg.max_steps: + # Get step start time (used to compute how much to sleep between steps) + step_start_time = time.time() + + # Get observation + obs = env.get_observation(t=t) + + # Save raw high camera image for replay video + replay_images.append(obs.observation["images"]["cam_high"]) + + # If action queue is empty, requery model + if len(action_queue) == 0: + # Prepare observation + observation, img_resized, left_wrist_resized, right_wrist_resized = prepare_observation(obs, resize_size) + observation["instruction"] = task_description + + # Save processed images for replay + replay_images_resized.append(img_resized) + replay_images_left_wrist_resized.append(left_wrist_resized) + replay_images_right_wrist_resized.append(right_wrist_resized) + + # Query model to get action + log_message("Requerying model...", log_file) + model_query_start_time = time.time() + actions = get_action_from_server(observation, server_endpoint) + actions = actions[: cfg.num_open_loop_steps] + total_model_query_time += time.time() - model_query_start_time + action_queue.extend(actions) + + # Get action from queue + action = action_queue.popleft() + log_message("-----------------------------------------------------", log_file) + log_message(f"t: {t}", log_file) + log_message(f"action: {action}", log_file) + + # Execute action in environment + if cfg.use_relative_actions: + # Get absolute joint angles from relative action + rel_action = action + target_state = curr_state + rel_action + obs = env.step(target_state.tolist()) + # Update current state (assume it is the commanded target state) + curr_state = target_state + else: + obs = env.step(action.tolist()) + t += 1 + + # Sleep until next timestep + step_elapsed_time = time.time() - step_start_time + if step_elapsed_time < STEP_DURATION_IN_SEC: + time_to_sleep = STEP_DURATION_IN_SEC - step_elapsed_time + log_message(f"Sleeping {time_to_sleep} sec...", log_file) + time.sleep(time_to_sleep) + + except (KeyboardInterrupt, Exception) as e: + if isinstance(e, KeyboardInterrupt): + log_message("\nCaught KeyboardInterrupt: Terminating episode early.", log_file) + else: + log_message(f"\nCaught exception: {e}", log_file) + + episode_end_time = time.time() + + # Get success feedback from user + user_input = input("Success? Enter 'y' or 'n': ") + success = True if user_input.lower() == "y" else False + + # Calculate episode statistics + episode_stats = { + "success": success, + "total_steps": t, + "model_query_time": total_model_query_time, + "episode_duration": episode_end_time - episode_start_time, + } + + return ( + episode_stats, + replay_images, + replay_images_resized, + replay_images_left_wrist_resized, + replay_images_right_wrist_resized, + ) + + +def save_episode_videos( + replay_images, + replay_images_resized, + replay_images_left_wrist, + replay_images_right_wrist, + episode_idx, + success, + task_description, + log_file=None, +): + """Save videos of the episode from different camera angles.""" + # Save main replay video + save_rollout_video(replay_images, episode_idx, success=success, task_description=task_description, log_file=log_file) + + # Save processed view videos + save_rollout_video( + replay_images_resized, + episode_idx, + success=success, + task_description=task_description, + log_file=log_file, + notes="resized", + ) + save_rollout_video( + replay_images_left_wrist, + episode_idx, + success=success, + task_description=task_description, + log_file=log_file, + notes="left_wrist_resized", + ) + save_rollout_video( + replay_images_right_wrist, + episode_idx, + success=success, + task_description=task_description, + log_file=log_file, + notes="right_wrist_resized", + ) + + +@draccus.wrap() +def eval_aloha(cfg: GenerateConfig) -> None: + """Main function to evaluate a trained policy in a real-world ALOHA environment.""" + # Validate configuration + validate_config(cfg) + + # Set random seed + set_seed_everywhere(cfg.seed) + + # Setup logging + log_file, local_log_filepath, run_id = setup_logging(cfg) + + # Get expected image dimensions + resize_size = get_image_resize_size(cfg) + + # Get ALOHA environment + env = get_aloha_env() + + # Get server endpoint for remote inference + server_endpoint = get_server_endpoint(cfg) + + # Initialize task description + task_description = "" + + # Start evaluation + num_rollouts_completed, total_successes = 0, 0 + + for episode_idx in tqdm.tqdm(range(cfg.num_rollouts_planned)): + # Get task description from user + task_description = get_next_task_label(task_description) + log_message(f"\nTask: {task_description}", log_file) + + log_message(f"Starting episode {num_rollouts_completed + 1}...", log_file) + + # Run episode + episode_stats, replay_images, replay_images_resized, replay_images_left_wrist, replay_images_right_wrist = ( + run_episode(cfg, env, task_description, server_endpoint, resize_size, log_file) + ) + + # Update counters + num_rollouts_completed += 1 + if episode_stats["success"]: + total_successes += 1 + + # Save videos + save_episode_videos( + replay_images, + replay_images_resized, + replay_images_left_wrist, + replay_images_right_wrist, + num_rollouts_completed, + episode_stats["success"], + task_description, + log_file, + ) + + # Log results + log_message(f"Success: {episode_stats['success']}", log_file) + log_message(f"# episodes completed so far: {num_rollouts_completed}", log_file) + log_message(f"# successes: {total_successes} ({total_successes / num_rollouts_completed * 100:.1f}%)", log_file) + log_message(f"Total model query time: {episode_stats['model_query_time']:.2f} sec", log_file) + log_message(f"Total episode elapsed time: {episode_stats['episode_duration']:.2f} sec", log_file) + + # Calculate final success rate + final_success_rate = float(total_successes) / float(num_rollouts_completed) if num_rollouts_completed > 0 else 0 + + # Log final results + log_message("\nFinal results:", log_file) + log_message(f"Total episodes: {num_rollouts_completed}", log_file) + log_message(f"Total successes: {total_successes}", log_file) + log_message(f"Overall success rate: {final_success_rate:.4f} ({final_success_rate * 100:.1f}%)", log_file) + + # Close log file + if log_file: + log_file.close() + + return final_success_rate + + +if __name__ == "__main__": + eval_aloha() diff --git a/experiments/robot/libero/libero_utils.py b/experiments/robot/libero/libero_utils.py index 70a5d7074..9d1f3fa1a 100644 --- a/experiments/robot/libero/libero_utils.py +++ b/experiments/robot/libero/libero_utils.py @@ -30,31 +30,17 @@ def get_libero_dummy_action(model_family: str): return [0, 0, 0, 0, 0, 0, -1] -def resize_image(img, resize_size): - """ - Takes numpy array corresponding to a single image and returns resized image as numpy array. - - NOTE (Moo Jin): To make input images in distribution with respect to the inputs seen at training time, we follow - the same resizing scheme used in the Octo dataloader, which OpenVLA uses for training. - """ - assert isinstance(resize_size, tuple) - # Resize to image size expected by model - img = tf.image.encode_jpeg(img) # Encode as JPEG, as done in RLDS dataset builder - img = tf.io.decode_image(img, expand_animations=False, dtype=tf.uint8) # Immediately decode back - img = tf.image.resize(img, resize_size, method="lanczos3", antialias=True) - img = tf.cast(tf.clip_by_value(tf.round(img), 0, 255), tf.uint8) - img = img.numpy() +def get_libero_image(obs): + """Extracts third-person image from observations and preprocesses it.""" + img = obs["agentview_image"] + img = img[::-1, ::-1] # IMPORTANT: rotate 180 degrees to match train preprocessing return img -def get_libero_image(obs, resize_size): - """Extracts image from observations and preprocesses it.""" - assert isinstance(resize_size, int) or isinstance(resize_size, tuple) - if isinstance(resize_size, int): - resize_size = (resize_size, resize_size) - img = obs["agentview_image"] +def get_libero_wrist_image(obs): + """Extracts wrist camera image from observations and preprocesses it.""" + img = obs["robot0_eye_in_hand_image"] img = img[::-1, ::-1] # IMPORTANT: rotate 180 degrees to match train preprocessing - img = resize_image(img, resize_size) return img diff --git a/experiments/robot/libero/run_libero_eval.py b/experiments/robot/libero/run_libero_eval.py index 5c3f58178..ed370adaa 100644 --- a/experiments/robot/libero/run_libero_eval.py +++ b/experiments/robot/libero/run_libero_eval.py @@ -1,25 +1,16 @@ """ run_libero_eval.py -Runs a model in a LIBERO simulation environment. - -Usage: - # OpenVLA: - # IMPORTANT: Set `center_crop=True` if model is fine-tuned with augmentations - python experiments/robot/libero/run_libero_eval.py \ - --model_family openvla \ - --pretrained_checkpoint \ - --task_suite_name [ libero_spatial | libero_object | libero_goal | libero_10 | libero_90 ] \ - --center_crop [ True | False ] \ - --run_id_note \ - --use_wandb [ True | False ] \ - --wandb_project \ - --wandb_entity +Evaluates a trained policy in a LIBERO simulation benchmark task suite. """ +import json +import logging import os import sys +from collections import deque from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import Optional, Union @@ -36,10 +27,17 @@ get_libero_dummy_action, get_libero_env, get_libero_image, + get_libero_wrist_image, quat2axisangle, save_rollout_video, ) -from experiments.robot.openvla_utils import get_processor +from experiments.robot.openvla_utils import ( + get_action_head, + get_noisy_action_projector, + get_processor, + get_proprio_projector, + resize_image_for_policy, +) from experiments.robot.robot_utils import ( DATE_TIME, get_action, @@ -49,6 +47,35 @@ normalize_gripper_action, set_seed_everywhere, ) +from prismatic.vla.constants import NUM_ACTIONS_CHUNK + + +# Define task suite constants +class TaskSuite(str, Enum): + LIBERO_SPATIAL = "libero_spatial" + LIBERO_OBJECT = "libero_object" + LIBERO_GOAL = "libero_goal" + LIBERO_10 = "libero_10" + LIBERO_90 = "libero_90" + + +# Define max steps for each task suite +TASK_MAX_STEPS = { + TaskSuite.LIBERO_SPATIAL: 220, # longest training demo has 193 steps + TaskSuite.LIBERO_OBJECT: 280, # longest training demo has 254 steps + TaskSuite.LIBERO_GOAL: 300, # longest training demo has 270 steps + TaskSuite.LIBERO_10: 520, # longest training demo has 505 steps + TaskSuite.LIBERO_90: 400, # longest training demo has 373 steps +} + + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger(__name__) @dataclass @@ -60,72 +87,122 @@ class GenerateConfig: ################################################################################################################# model_family: str = "openvla" # Model family pretrained_checkpoint: Union[str, Path] = "" # Pretrained checkpoint path - load_in_8bit: bool = False # (For OpenVLA only) Load with 8-bit quantization - load_in_4bit: bool = False # (For OpenVLA only) Load with 4-bit quantization + + use_l1_regression: bool = True # If True, uses continuous action head with L1 regression objective + use_diffusion: bool = False # If True, uses continuous action head with diffusion modeling objective (DDIM) + num_diffusion_steps: int = 50 # (When `diffusion==True`) Number of diffusion steps for inference + use_film: bool = False # If True, uses FiLM to infuse language inputs into visual features + num_images_in_input: int = 2 # Number of images in the VLA input (default: 1) + use_proprio: bool = True # Whether to include proprio state in input center_crop: bool = True # Center crop? (if trained w/ random crop image aug) + num_open_loop_steps: int = 8 # Number of actions to execute open-loop before requerying policy + + unnorm_key: Union[str, Path] = "" # Action un-normalization key + + load_in_8bit: bool = False # (For OpenVLA only) Load with 8-bit quantization + load_in_4bit: bool = False # (For OpenVLA only) Load with 4-bit quantization ################################################################################################################# # LIBERO environment-specific parameters ################################################################################################################# - task_suite_name: str = "libero_spatial" # Task suite. Options: libero_spatial, libero_object, libero_goal, libero_10, libero_90 + task_suite_name: str = TaskSuite.LIBERO_SPATIAL # Task suite num_steps_wait: int = 10 # Number of steps to wait for objects to stabilize in sim num_trials_per_task: int = 50 # Number of rollouts per task + initial_states_path: str = "DEFAULT" # "DEFAULT", or path to initial states JSON file + env_img_res: int = 256 # Resolution for environment images (not policy input resolution) ################################################################################################################# # Utils ################################################################################################################# - run_id_note: Optional[str] = None # Extra note to add in run ID for logging + run_id_note: Optional[str] = None # Extra note to add to end of run ID for logging local_log_dir: str = "./experiments/logs" # Local directory for eval logs use_wandb: bool = False # Whether to also log results in Weights & Biases - wandb_project: str = "YOUR_WANDB_PROJECT" # Name of W&B project to log to (use default!) - wandb_entity: str = "YOUR_WANDB_ENTITY" # Name of entity to log under + wandb_entity: str = "your-wandb-entity" # Name of WandB entity + wandb_project: str = "your-wandb-project" # Name of WandB project seed: int = 7 # Random Seed (for reproducibility) # fmt: on -@draccus.wrap() -def eval_libero(cfg: GenerateConfig) -> None: - assert cfg.pretrained_checkpoint is not None, "cfg.pretrained_checkpoint must not be None!" - if "image_aug" in cfg.pretrained_checkpoint: +def validate_config(cfg: GenerateConfig) -> None: + """Validate configuration parameters.""" + assert cfg.pretrained_checkpoint is not None, "pretrained_checkpoint must not be None!" + + if "image_aug" in str(cfg.pretrained_checkpoint): assert cfg.center_crop, "Expecting `center_crop==True` because model was trained with image augmentations!" + assert not (cfg.load_in_8bit and cfg.load_in_4bit), "Cannot use both 8-bit and 4-bit quantization!" - # Set random seed - set_seed_everywhere(cfg.seed) + # Validate task suite + assert cfg.task_suite_name in [suite.value for suite in TaskSuite], f"Invalid task suite: {cfg.task_suite_name}" - # [OpenVLA] Set action un-normalization key - cfg.unnorm_key = cfg.task_suite_name +def initialize_model(cfg: GenerateConfig): + """Initialize model and associated components.""" # Load model model = get_model(cfg) - # [OpenVLA] Check that the model contains the action un-normalization key - if cfg.model_family == "openvla": - # In some cases, the key must be manually modified (e.g. after training on a modified version of the dataset - # with the suffix "_no_noops" in the dataset name) - if cfg.unnorm_key not in model.norm_stats and f"{cfg.unnorm_key}_no_noops" in model.norm_stats: - cfg.unnorm_key = f"{cfg.unnorm_key}_no_noops" - assert cfg.unnorm_key in model.norm_stats, f"Action un-norm key {cfg.unnorm_key} not found in VLA `norm_stats`!" + # Load proprio projector if needed + proprio_projector = None + if cfg.use_proprio: + proprio_projector = get_proprio_projector( + cfg, + model.llm_dim, + proprio_dim=8, # 8-dimensional proprio for LIBERO + ) + + # Load action head if needed + action_head = None + if cfg.use_l1_regression or cfg.use_diffusion: + action_head = get_action_head(cfg, model.llm_dim) - # [OpenVLA] Get Hugging Face processor + # Load noisy action projector if using diffusion + noisy_action_projector = None + if cfg.use_diffusion: + noisy_action_projector = get_noisy_action_projector(cfg, model.llm_dim) + + # Get OpenVLA processor if needed processor = None if cfg.model_family == "openvla": processor = get_processor(cfg) + check_unnorm_key(cfg, model) + + return model, action_head, proprio_projector, noisy_action_projector, processor + + +def check_unnorm_key(cfg: GenerateConfig, model) -> None: + """Check that the model contains the action un-normalization key.""" + # Initialize unnorm_key + unnorm_key = cfg.task_suite_name - # Initialize local logging + # In some cases, the key must be manually modified (e.g. after training on a modified version of the dataset + # with the suffix "_no_noops" in the dataset name) + if unnorm_key not in model.norm_stats and f"{unnorm_key}_no_noops" in model.norm_stats: + unnorm_key = f"{unnorm_key}_no_noops" + + assert unnorm_key in model.norm_stats, f"Action un-norm key {unnorm_key} not found in VLA `norm_stats`!" + + # Set the unnorm_key in cfg + cfg.unnorm_key = unnorm_key + + +def setup_logging(cfg: GenerateConfig): + """Set up logging to file and optionally to wandb.""" + # Create run ID run_id = f"EVAL-{cfg.task_suite_name}-{cfg.model_family}-{DATE_TIME}" if cfg.run_id_note is not None: run_id += f"--{cfg.run_id_note}" + + # Set up local logging os.makedirs(cfg.local_log_dir, exist_ok=True) local_log_filepath = os.path.join(cfg.local_log_dir, run_id + ".txt") log_file = open(local_log_filepath, "w") - print(f"Logging to local log file: {local_log_filepath}") + logger.info(f"Logging to local log file: {local_log_filepath}") - # Initialize Weights & Biases logging as well + # Initialize Weights & Biases logging if enabled if cfg.use_wandb: wandb.init( entity=cfg.wandb_entity, @@ -133,154 +210,319 @@ def eval_libero(cfg: GenerateConfig) -> None: name=run_id, ) + return log_file, local_log_filepath, run_id + + +def log_message(message: str, log_file=None): + """Log a message to console and optionally to a log file.""" + logger.info(message) + if log_file: + log_file.write(message + "\n") + log_file.flush() + + +def load_initial_states(cfg: GenerateConfig, task_suite, task_id: int, log_file=None): + """Load initial states for the given task.""" + # Get default initial states + initial_states = task_suite.get_task_init_states(task_id) + + # If using custom initial states, load them from file + if cfg.initial_states_path != "DEFAULT": + with open(cfg.initial_states_path, "r") as f: + all_initial_states = json.load(f) + log_message(f"Using initial states from {cfg.initial_states_path}", log_file) + return initial_states, all_initial_states + else: + log_message("Using default initial states", log_file) + return initial_states, None + + +def prepare_observation(obs, resize_size): + """Prepare observation for policy input.""" + # Get preprocessed images + img = get_libero_image(obs) + wrist_img = get_libero_wrist_image(obs) + + # Resize images to size expected by model + img_resized = resize_image_for_policy(img, resize_size) + wrist_img_resized = resize_image_for_policy(wrist_img, resize_size) + + # Prepare observations dict + observation = { + "full_image": img_resized, + "wrist_image": wrist_img_resized, + "state": np.concatenate( + (obs["robot0_eef_pos"], quat2axisangle(obs["robot0_eef_quat"]), obs["robot0_gripper_qpos"]) + ), + } + + return observation, img # Return both processed observation and original image for replay + + +def process_action(action, model_family): + """Process action before sending to environment.""" + # Normalize gripper action [0,1] -> [-1,+1] because the environment expects the latter + action = normalize_gripper_action(action, binarize=True) + + # [OpenVLA] The dataloader flips the sign of the gripper action to align with other datasets + # (0 = close, 1 = open), so flip it back (-1 = open, +1 = close) before executing the action + if model_family == "openvla": + action = invert_gripper_action(action) + + return action + + +def run_episode( + cfg: GenerateConfig, + env, + task_description: str, + model, + resize_size, + processor=None, + action_head=None, + proprio_projector=None, + noisy_action_projector=None, + initial_state=None, + log_file=None, +): + """Run a single episode in the environment.""" + # Reset environment + env.reset() + + # Set initial state if provided + if initial_state is not None: + obs = env.set_init_state(initial_state) + else: + obs = env.get_observation() + + # Initialize action queue + if cfg.num_open_loop_steps != NUM_ACTIONS_CHUNK: + print(f"WARNING: cfg.num_open_loop_steps ({cfg.num_open_loop_steps}) does not match the NUM_ACTIONS_CHUNK " + "{NUM_ACTIONS_CHUNK} constant defined in prismatic.vla.constants! For best performance (in terms of " + "both speed and success rate), we recommend executing the full action chunk.") + action_queue = deque(maxlen=cfg.num_open_loop_steps) + + # Setup + t = 0 + replay_images = [] + max_steps = TASK_MAX_STEPS[cfg.task_suite_name] + + # Run episode + success = False + try: + while t < max_steps + cfg.num_steps_wait: + # Do nothing for the first few timesteps to let objects stabilize + if t < cfg.num_steps_wait: + obs, reward, done, info = env.step(get_libero_dummy_action(cfg.model_family)) + t += 1 + continue + + # Prepare observation + observation, img = prepare_observation(obs, resize_size) + replay_images.append(img) + + # If action queue is empty, requery model + if len(action_queue) == 0: + # Query model to get action + actions = get_action( + cfg, + model, + observation, + task_description, + processor=processor, + action_head=action_head, + proprio_projector=proprio_projector, + noisy_action_projector=noisy_action_projector, + use_film=cfg.use_film, + ) + action_queue.extend(actions) + + # Get action from queue + action = action_queue.popleft() + + # Process action + action = process_action(action, cfg.model_family) + + # Execute action in environment + obs, reward, done, info = env.step(action.tolist()) + if done: + success = True + break + t += 1 + + except Exception as e: + log_message(f"Episode error: {e}", log_file) + + return success, replay_images + + +def run_task( + cfg: GenerateConfig, + task_suite, + task_id: int, + model, + resize_size, + processor=None, + action_head=None, + proprio_projector=None, + noisy_action_projector=None, + total_episodes=0, + total_successes=0, + log_file=None, +): + """Run evaluation for a single task.""" + # Get task + task = task_suite.get_task(task_id) + + # Get initial states + initial_states, all_initial_states = load_initial_states(cfg, task_suite, task_id, log_file) + + # Initialize environment and get task description + env, task_description = get_libero_env(task, cfg.model_family, resolution=cfg.env_img_res) + + # Start episodes + task_episodes, task_successes = 0, 0 + for episode_idx in tqdm.tqdm(range(cfg.num_trials_per_task)): + log_message(f"\nTask: {task_description}", log_file) + + # Handle initial state + if cfg.initial_states_path == "DEFAULT": + # Use default initial state + initial_state = initial_states[episode_idx] + else: + # Get keys for fetching initial episode state from JSON + initial_states_task_key = task_description.replace(" ", "_") + episode_key = f"demo_{episode_idx}" + + # Skip episode if expert demonstration failed to complete the task + if not all_initial_states[initial_states_task_key][episode_key]["success"]: + log_message(f"Skipping task {task_id} episode {episode_idx} due to failed expert demo!", log_file) + continue + + # Get initial state + initial_state = np.array(all_initial_states[initial_states_task_key][episode_key]["initial_state"]) + + log_message(f"Starting episode {task_episodes + 1}...", log_file) + + # Run episode + success, replay_images = run_episode( + cfg, + env, + task_description, + model, + resize_size, + processor, + action_head, + proprio_projector, + noisy_action_projector, + initial_state, + log_file, + ) + + # Update counters + task_episodes += 1 + total_episodes += 1 + if success: + task_successes += 1 + total_successes += 1 + + # Save replay video + save_rollout_video( + replay_images, total_episodes, success=success, task_description=task_description, log_file=log_file + ) + + # Log results + log_message(f"Success: {success}", log_file) + log_message(f"# episodes completed so far: {total_episodes}", log_file) + log_message(f"# successes: {total_successes} ({total_successes / total_episodes * 100:.1f}%)", log_file) + + # Log task results + task_success_rate = float(task_successes) / float(task_episodes) if task_episodes > 0 else 0 + total_success_rate = float(total_successes) / float(total_episodes) if total_episodes > 0 else 0 + + log_message(f"Current task success rate: {task_success_rate}", log_file) + log_message(f"Current total success rate: {total_success_rate}", log_file) + + # Log to wandb if enabled + if cfg.use_wandb: + wandb.log( + { + f"success_rate/{task_description}": task_success_rate, + f"num_episodes/{task_description}": task_episodes, + } + ) + + return total_episodes, total_successes + + +@draccus.wrap() +def eval_libero(cfg: GenerateConfig) -> float: + """Main function to evaluate a trained policy on LIBERO benchmark tasks.""" + # Validate configuration + validate_config(cfg) + + # Set random seed + set_seed_everywhere(cfg.seed) + + # Initialize model and components + model, action_head, proprio_projector, noisy_action_projector, processor = initialize_model(cfg) + + # Get expected image dimensions + resize_size = get_image_resize_size(cfg) + + # Setup logging + log_file, local_log_filepath, run_id = setup_logging(cfg) + # Initialize LIBERO task suite benchmark_dict = benchmark.get_benchmark_dict() task_suite = benchmark_dict[cfg.task_suite_name]() - num_tasks_in_suite = task_suite.n_tasks - print(f"Task suite: {cfg.task_suite_name}") - log_file.write(f"Task suite: {cfg.task_suite_name}\n") + num_tasks = task_suite.n_tasks - # Get expected image dimensions - resize_size = get_image_resize_size(cfg) + log_message(f"Task suite: {cfg.task_suite_name}", log_file) # Start evaluation total_episodes, total_successes = 0, 0 - for task_id in tqdm.tqdm(range(num_tasks_in_suite)): - # Get task - task = task_suite.get_task(task_id) - - # Get default LIBERO initial states - initial_states = task_suite.get_task_init_states(task_id) - - # Initialize LIBERO environment and task description - env, task_description = get_libero_env(task, cfg.model_family, resolution=256) - - # Start episodes - task_episodes, task_successes = 0, 0 - for episode_idx in tqdm.tqdm(range(cfg.num_trials_per_task)): - print(f"\nTask: {task_description}") - log_file.write(f"\nTask: {task_description}\n") - - # Reset environment - env.reset() - - # Set initial states - obs = env.set_init_state(initial_states[episode_idx]) - - # Setup - t = 0 - replay_images = [] - if cfg.task_suite_name == "libero_spatial": - max_steps = 220 # longest training demo has 193 steps - elif cfg.task_suite_name == "libero_object": - max_steps = 280 # longest training demo has 254 steps - elif cfg.task_suite_name == "libero_goal": - max_steps = 300 # longest training demo has 270 steps - elif cfg.task_suite_name == "libero_10": - max_steps = 520 # longest training demo has 505 steps - elif cfg.task_suite_name == "libero_90": - max_steps = 400 # longest training demo has 373 steps - - print(f"Starting episode {task_episodes+1}...") - log_file.write(f"Starting episode {task_episodes+1}...\n") - while t < max_steps + cfg.num_steps_wait: - try: - # IMPORTANT: Do nothing for the first few timesteps because the simulator drops objects - # and we need to wait for them to fall - if t < cfg.num_steps_wait: - obs, reward, done, info = env.step(get_libero_dummy_action(cfg.model_family)) - t += 1 - continue - - # Get preprocessed image - img = get_libero_image(obs, resize_size) - - # Save preprocessed image for replay video - replay_images.append(img) - - # Prepare observations dict - # Note: OpenVLA does not take proprio state as input - observation = { - "full_image": img, - "state": np.concatenate( - (obs["robot0_eef_pos"], quat2axisangle(obs["robot0_eef_quat"]), obs["robot0_gripper_qpos"]) - ), - } - - # Query model to get action - action = get_action( - cfg, - model, - observation, - task_description, - processor=processor, - ) - - # Normalize gripper action [0,1] -> [-1,+1] because the environment expects the latter - action = normalize_gripper_action(action, binarize=True) - - # [OpenVLA] The dataloader flips the sign of the gripper action to align with other datasets - # (0 = close, 1 = open), so flip it back (-1 = open, +1 = close) before executing the action - if cfg.model_family == "openvla": - action = invert_gripper_action(action) - - # Execute action in environment - obs, reward, done, info = env.step(action.tolist()) - if done: - task_successes += 1 - total_successes += 1 - break - t += 1 - - except Exception as e: - print(f"Caught exception: {e}") - log_file.write(f"Caught exception: {e}\n") - break - - task_episodes += 1 - total_episodes += 1 - - # Save a replay video of the episode - save_rollout_video( - replay_images, total_episodes, success=done, task_description=task_description, log_file=log_file - ) - - # Log current results - print(f"Success: {done}") - print(f"# episodes completed so far: {total_episodes}") - print(f"# successes: {total_successes} ({total_successes / total_episodes * 100:.1f}%)") - log_file.write(f"Success: {done}\n") - log_file.write(f"# episodes completed so far: {total_episodes}\n") - log_file.write(f"# successes: {total_successes} ({total_successes / total_episodes * 100:.1f}%)\n") - log_file.flush() - - # Log final results - print(f"Current task success rate: {float(task_successes) / float(task_episodes)}") - print(f"Current total success rate: {float(total_successes) / float(total_episodes)}") - log_file.write(f"Current task success rate: {float(task_successes) / float(task_episodes)}\n") - log_file.write(f"Current total success rate: {float(total_successes) / float(total_episodes)}\n") - log_file.flush() - if cfg.use_wandb: - wandb.log( - { - f"success_rate/{task_description}": float(task_successes) / float(task_episodes), - f"num_episodes/{task_description}": task_episodes, - } - ) - - # Save local log file - log_file.close() - - # Push total metrics and local log file to wandb + for task_id in tqdm.tqdm(range(num_tasks)): + total_episodes, total_successes = run_task( + cfg, + task_suite, + task_id, + model, + resize_size, + processor, + action_head, + proprio_projector, + noisy_action_projector, + total_episodes, + total_successes, + log_file, + ) + + # Calculate final success rate + final_success_rate = float(total_successes) / float(total_episodes) if total_episodes > 0 else 0 + + # Log final results + log_message("Final results:", log_file) + log_message(f"Total episodes: {total_episodes}", log_file) + log_message(f"Total successes: {total_successes}", log_file) + log_message(f"Overall success rate: {final_success_rate:.4f} ({final_success_rate * 100:.1f}%)", log_file) + + # Log to wandb if enabled if cfg.use_wandb: wandb.log( { - "success_rate/total": float(total_successes) / float(total_episodes), + "success_rate/total": final_success_rate, "num_episodes/total": total_episodes, } ) wandb.save(local_log_filepath) + # Close log file + if log_file: + log_file.close() + + return final_success_rate + if __name__ == "__main__": eval_libero() diff --git a/experiments/robot/libero/sample_libero_spatial_observation.pkl b/experiments/robot/libero/sample_libero_spatial_observation.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8863226be214ba60f8be5b3674d272024df93439 GIT binary patch literal 301501 zcma&P_j6rGmglLKp=d=9O8_M3y%DVpjVMZ_Xe7})cn`dX_weAo_aF>ONtE}ts=B+X zt7JrmNg|j~ zj7j`dpC_*cm26p5LVcn(Q5UtKYblF_mYpbhi9(fh>(;dm8*&#fS~zds(+d~QCuC-( zuU(tHar36*ubg=M(#5=j{8(MQHXb9yS#1tmq@&F>ebzQcbz=-%K10XUA=y# zxU?t|t*NW4YiMX_ZfzkT-X*^?(v z9(nQb_MO{y?AmrK?-tc6=T}yq7yQsUe*Ea}-8)addOQ}dp~BSMZxPM6run?A=UYi? zNH8FpQV?Th5>+97MhT^YSzbM>i&6p%^(1d&1E2v^wO6lR z+P81d(W5V2y7czNi*H@IatXBD%DbIkSU@N&E-ET1uBfUkE-nT>`T6+=&GvL zh1y7SWQVy_O*ZOV4_^AD%03pITMiQGs|2Gy1|&3zaH6QNVG0HnF-CoS3Yi8}!zap* z<1LB2=n%EaST$g*^43gAh?=2{F&!~zN)0-!)8kd-lw^^Jgz!zC^fu^~&{|H?H2ee);m{D`5B1r7Kr2zkcS_ ziIc})Kl9q@H(oz;_VgR)&z!$-4p5N_2(MkchLeOU3z&TK%{R}VKY!-Tnb%)`{nb}r zJ#pg1%P+qSJ`Wx~ux-cIV<%3OmX~pr>o;y(ym%3vS6(^s!V8DDZ{PaH+1G1ps~Qrq zP@~L-)-Gdc<0EY@In;u(P%?~!Vk9@Al0rQ~)CWK)N|GT&j42S6M2t}ZPxOC@&ybp5 zh~d+Os;AXoj2Qt23DnxF=1cA?`)ZQ=seXaJ0S#}aK#x6t!+iC%lW&|mbMf-q zSFT-Ve7|<%%C+m@@tVLXWP(+|251?|FI>0)K4IQ?;|<`4|737KcI?>EV=wR8v-_o& zU%H)__tx8QojZT-wbx!de*DwE)OZD}&F8p{7BSiT|pe?=~u$z2^*+h(= z#dsJ}&EO_i7L%(?iuL8fq3H+-KS2#MVoa0h?A?O(4h_Wu|$0>5Up>hjn-BjdEr2M zddh|k>$hy#v}@N6=6?I1-+SP|^90a)(Q5cs@)^IBfPZ9+j|UQtOwNh#xV zX-V)aC@5g0W~#|V^7if9_);7%&=q3A>bZ0004gXwcH;Q{g9i>DJ{%0_ry2BLdg;ae z`}eL}mviLse(K32BvHo<{L!P2+F}+k+?sr@iBOh(U`$OVJd#RBU2MHzd?Z8-pFVo+TA`~GwMvSSf_Q5V(YZzl(2T6?auiat zB-z|7PpU5w&xK&riOLS&)SReqh~Yw^0F}karcJpisVmdcS7)wCUAs05s4(nVAm1Y14*lS1;GbBDL{o zqM^Q_sj<1WrKPR4rMaLE_d?PW6@}3Fs4U9W_eciZO^PONjctqoklr)(cd1);k4`)WNke4ps>)pb83aoq6Nb=`*js z_WDVtpsaORoi{c&HMKN%banRh^#-A*Z=k2QKM1|O-QC??U0od=9ol7T41T~Btb$m8 z${YY+$^;cJ3PcYcJg{-&`n`L1U%!48XU=sQ)(L=<>cwiSteMo=%pztk4cR|vX0m~n z*NijQEF?*d?)JL|q2UvrBvXJVkWu7+6u}5@A=DZCd<2>hW#h#tyNWfQ zi>f}>9IK2ZKS7mO*^M_w85Q-g^l&dl3MBc)m!>Z?iI`A*0-@`DPf#IS4!GYn?k&&^n(ecsoJ7dW(K0X@!hK7a)2M7E6`+;j` zXQyq4%A?{-Z{5Dl8voq6^SgKNV)9T~S>D)GuWJEI@EMI%i1ATmZ5Jaenb2%#ZMVGA zfm#xb34KO|(3=G0VvJgpyBQwrn~GuM$mvJQbVCZ!dw7buF1F2tj8=6Hc0 zgW^J+Lkl2US061Z%8y2>f#v8pfSj1VJ2P|d?%n%$@4tKR-aC%I^Ugc>@87?B_wLlx z)Sci5TES{xUtdp84<5CtsR>^iiA0LaO4w*+UH`%hN2;nS>JqhXE56>*crBqi;jN6+ zG{2>4lBtNeYOXGzrqlQlyfCsIt zD6v$Zp@dMBx=APu!>hJpR1?)u7Ent?&8n7Unn=X^5(z0Ep=F^Ecm)+GaEh8-QM;lT zi>frsj&}+!k(`$*FLOWe$tnjB)d5$)$v7KlCRxSM&w{4zVE5(J_jee~gnA3k{S;N5rM1+FtQGt<-4Ky_?vY+zsjcZx6Nxl(0y z6_2RcU&_nB-PzU7PAoTqyB%=gcyESvoVm9$qS}7_)nrMGH#7Q8wp4@^=r!Ms$OK>6 zN+6ebQFRA=sNW?qO{Mx#DS#8h3_9>Wz{JZL3Sh!#MR~CaB~jGZhgab~wMt4zR8^9> zti4uG7_*WIB_SHh9#SwvSqZYUnVP7QbY1nSDZ^NOjF-eRDvKXAU&}?T!+qU7?JU`Fq3m}7)JSb5 z6{9AqHWbUK<}#7r9HmMw9h1OFQ_X}vQbGFaS)Z50Iz>_IG5}x-#>kSU=_5xxpaN-v znW%iG357!whN3D&!Vo?~QLCOREK?Y@M#0P#BB3E69W~9@M=caZ#}EbY2bHN*%cvev z)K`W_s4U{4rp(48zLe#&!zTm1+bNBK&zm=|SJhM{8XE?Oho)wx-hJ@ihaZ3V*%zOJ z&u_l_8f5*%Lijqi2663`{k?Ycu2umRfiUdW;;Tg&eZ=@PB;T4oBY9%YG zLJ-18*)AtdqpBNGRbEjw7MsW{N65 zk>>2e1h~4OXguK_HeJ4Q3D5vec9wek`k07LPEXx`_x^kDzsnH+$!DJuKKuN$k3V6& z{|IEh_uhL^=cb9y#5-eG7bU-M%HiAn*N;9!yP7Ub}XQ=bx2TrIV8r z0|R{{BSW2??JX@$b@6ch)fV44s=vuC3Hykbp-k;@jQY-$)w3!AY8?##;6-&&@yd?G zQpH#*48c%DDuJi~X;1{Nz63c?R9)Kf;|E_EK2k%YIHOg;*eza6RVCAMtihd2^Bn^|*_LEQpI>;JXYSW; z-6$$8?dtE3H#GJR4t4hS4UUd7BfT>-J$ZKqdS`lSa%OsBYI1ORbZ}^dji=78?zWCL zE=gd+{lZ&s@(78|Po8Nq){l>mGY2?z>eZB#6qfVsO-)STNryo-+W;-iO_7Lu6q}r& z#<3pu@k4U1NTX!B#~IKpAcI>}8BK`wOd<%81S5n9Li#OjR8f#%p9g% zq9p$M2#FPFXhdje6mL3WNVN+YV|7+lp@tdXAn>w-nZnO#ni5J=m3lCYJi`-924>1EY8D z433WU4D^pojCb|*cK7$Sc6NZ??*4(M=GLyBUi@uMG{S)>j-9-I<}?RIc&5pzCl*aO zSSF~PecG{O$EsDUc!QMAUJ!{dB0+2u=eZz^cBux|gQ51mI+yG(;2ERu#elA@|b)bA*OBWJV!{L1kh^OH~ zi*EsVQbqkNb9G3?j8e8#!O4$u2@T~+lI4w>rWoWXt4#5tbum|nVVc`J>VoGz4K2;>-Ca#>t!-VM_05fqEY+J^$}6f0 zi%U4IS5j8WQ6*-c1x1DI)w7Cc(RA+~TU2+UI81h*z%8fph~h%oq?(?dVjDG)a8F=y zp-|_^-JY03=`7>SNc}@?d|Z^)(M`%iuDKaekdAs&{i`gNWQF&&j-Q8TQp2Mr}jZfar>23QLF&qpeg zr53M3@38BbsMW3xTRMRx($vJ$OLwr16GWu)G~B7x*A^7zojrHv8t${Gu%x20q9)SV z+J*~l>F8|Xfl+rC`0N`ROf)s#D#(X+^z>A*8n5ONe`9%71?NZzY@dGj!}ovnSAX@* zH{ZPf{`+`N(1^o?vL!_np9v3Wh1)xK#@pIjYH;jq+Rr*qO-lkNpY)A>GtFEuAuA~= zxu3=4mRFsk?hbX3>A)z-r;#5+(Uj2}JJ;0E#B?&w zCOj*YNVF1KRa3^4j{{Ckjdg9UjbN~(tnl20vsbQPV|*{Itip2wKbNhck%#4NT^)_B zEe!6Ulo?f`siCQ@ttQwis;Y_P7Zm``Z@>TU_ka5b)=-SyfqR_({PWMh`sypK^ns;$ zRItjAqJw2V(HRV*sZ9N4O62m8RV{4<|WE!iWhFeq&Hb-ve3h~w>^envP74H{l znqmwIAtt0|V1of&3h+Mg1U5_cAu3HNAP?aaxvFg!Rb?z{4DEu4T8oI9DI`~tvLL}? zP)j0J!Iw+HsOi9%!C~kuMWJGpMNw!pN?^FGZf|e>_PekD_HTdxo8SD+&wu{IH{X2q z#TTElfBB1F{P3IK{OY&=;n%Zr#i)EGQ}~uda>pOq3xW zd?uQj>ly)SJp(;7(d4G40&sanb$Mmg?fg6**5f%@``58`#fu~HI1lF;)7i0S10NU4 zt|{A7Uw-)|yQYAXT`BN6Jw3^CKH{oc6|1XZr;8<<2KRUji*esMD_OhLSl=vaxcXu) zkHj|}JQ9@EtKfU}@PU31LQ2l3;``!B6-UPZB{Ap)@2{AqYeGv^t2fny8)# zt-@+1)Ne9T8Bk4@8I{E}mE=>)B7I9z)Tb&$oh9D9il0Ol@Y&Fip!dJ}>We@A@ehCh z_y6g?{kQ+}KmN!6_3!`wfB*T<{|f(a|MsW<{@?#kqW|;%{rCUpfByTQ{`4>96{WAf ze)87sTfF^IR#i;^oIrFIpUsTv_03H{lxYEnq5v>+fXb?xqLNac7PWP>^Xxu(9xoPS z{s~0cablJ9{PUcz+j-!?0gl`QTE=~LP}xoWsW6}XlgKYu`g7TQA34kA`#e>fHs&8zpXQmg6F$;}0H#ZUxq9z_U``YWLUOf8pm1|ddbY5Cj&ESrc1c!Bv zP54g&;3P1z#~a+Cy+}On=ASjytc+N>GpXe;9^0pcdba*+xo8t7ML{OZ_Qk<(#flYK zSy>x5ZsgQG=kyr;c^Gx~?lcETii-1@18ctvWXgrk(&T6^6Zw`gn~i93Qqn6YPn>$~)Eno{vOxqsODZaGk4!J+KO0(F3H8km zq73xluQmZb-4M@s%*>PZI}b)%+uLe*-?QGm?}?vetCN>LmMvSBmX@}B`SR4@$1Z3_ zM#h>oYc_4##AZJ0cko%pd6HY#-3&Q+*a|-VKy#4$Hj%}A39VMwsQEGoJUUV{Wicwj zvY4hMP`~S~_f04TDN7j2VxThk!AO%N6Z*JCNK{F7Au9*QE~M+uO+rCQs*HG=X2IGH zV?n7(G9ip?Xa!m!a~b7AZrP2ekWgQs!}9!t4<3B@0S|dTVw%VH_$NH_`Q(%L-gD2& zHSv?2XF%_N_yNx`KmGi(k+D&BhxGPwSw$IRt^6nBJNrn<(A0uEMZ(Yzdch}0LF7L< zILdGyPsEyA-KkIxGlNfFom(CJz$Tc)c|vn?a@MU|2gAYpj*hmD4vv)6RaLq58Y?5h zkHN|I9ouDYzKyLy#_=U)Ni5*2Nj_JIj*`rn(lKoKj1_J;@l)sW0*~R)u!JVO%95Oe znJsNCEgXIa0G;iPO%2e&p@IH^e!{?@qx}PYeVq0j8YH-5pF@KzbATw$u&koInsaiz zAkxrS$A(lez9*VnI6V(O1OHhs|5;vHfif<%p^0Pqq_AtitE2VIJUJ`EOP#RmSmo z^^}=Wn$~^N^l7N0NivBTqXrB?#Q=i>yznM!Q8DH=jX|zr(U%uRV!5mk2~Am9LY4QU z@YZml#VB&w$`qt*LO*J`jImt41RnS)NG@=i4h%2&aCVMUZC$;c-MwAC{oQ>7z3)7@ z|KTSeeERt(97YQ0=O2Ii5zOabe){>BUwrn(=WGgn`Sn*gLtZ@wpP&wWMx*fr_zdQs zjPD8F?;tcb<39nS@af*555{)~&c+1O2IjYn?<|gbdwanr<2#t-2R`}b=H>!Url5e9 zJAk93wY3r6R4%V5Vf_v)wYdai;GD&3a!6A1Wgly}IwfRt8An2Anc0y}%9aJW-_h{C zf_NzyMHo`EB$&C-LTJDfBV-yys9ef!Q5Xe7rm5K#{n{cVN{g(9YPA{4Y+}|hMj0(? z8DT8D3C*5LQm4R+vHc~f>7vqfwXLmk2ivUKmX@{{>Oj)^Pm3oZ-4%$fBENs{HOo?5C8bTv&`XiM=9ew zFM!5)LA^c}>@P9CC-K?j@X7ctd;)2`zM8U|1RVpCK}%qR$}yT;;Iyb1viq98j!GugE+Ix0GfHKsm7?NRJ@K-M zNkR-6B~iN+HQpEs0dh+9i$$U&q14w`u}THKUEQ6$G}PA7Ixy7#mw)|>-~a7@{M~>2 z55NBH-~9Ej{_6XmefJx-J%9T*zxwqr3BUav^56XG*ChQC=7*pEti7{?FO5`ISF^t# zt8@FGJV*-GM-HMI-|?TCf8sxxe-bXddEvzu4s(K(#W<5sj??2mIoHIyNWAM2Woj9T z@OCFJ&=Fib(VQJ&o`UMAhg;{AqI(rVq_dBy(Vgm zB^hHK?K2dM0(d~^^QL2%F_<9{FDj&&kSNn4O+J<6Ly;^*Eu#wOT9Ux1G8VOJC`m>z zp`u3BQrX8^O{yePvs4NRNfb4gXsI%FC@L76P`QY*cxP(Ih7!9>-+ue`dk@~>9Ssi4 zajG+TabpIjIyEy%(2p&2o{4kTh^RXcKQRvQaHP0XmL+Vt^FWmSPaeMOSv=Ie;YRJN zD%mf+dYunCT!CSGpGho_ir9_kJTpJmXn**_AO7JV{^9Tc{_p?z$3Onv-~Ao*_rL%B zuYUC_o-2L*?KdBN@^MFd>*dQAOUsJb0%gfX!8W+7)lgZIrpa`nCX=eJf$yO3R-%B#+*l|?1= zsWnqNvYUjK%a~9X7}8WAsdIyWE%D0b%B+p_^te|odB8n0Gj)g4!BZ2Ujw79{(}zcg zhekLY7lff9P8spC8@>*Y$AAm%?(Jc@#KUy<+u1b&b?k8B4(I^)*|KG#@BMs$S6$O+fF^0~P1jdT89<(H1O#mRtFT~F>7aPhVIH5@t-XJI5 zqCbsKl>J~On@|#{s-$Zvs^)~KTA>CCOV%zK#J3x2p;W&#lB@*f4M_~W zMFp&jfhdoVctA5U<^VZ3JjjcgEY*8>p^aBA`rPbP5CY1)iOG*iBR__CHrsKJ(D&Yd z09?sMm$Uzgxj7Ne+wlaD$>dW{j^4WXHVZ4>O+NqT1x9er*RlD@2bvh)c_W_hitxrU zpKE~f0i45!5A%*ZugsIZCQ`$FR8dhzoms-yM9OEu38EaR%mT)!M`S~5a!8Y?Nnq43 z5}Mq~v0j#%C`tFiEJP87#Ji7#+yvsQ}Hu}zk_gZ=Py|V*9%n{|j zfqnoC#yFFF_WZePH?E(*aAC{Vtt^6Ap|eik!Y6e$ZCby6Jv5vJQ4Q|+Pc>GC@=YNL)P$xvmoXifP^#G)~_Nk~~y zWt4)TB@n_tG6+}DP-;j{)Jj(2l5kP0q$&_0s^!g9_7PH%sJVOM+mx@ox&hQzoos6bIAJP8#(I96Qa`6QW$=-DV6Yi z!+}Sf;FIkv9#?7Mj|*jR=PZ-9m;x{A)z`IQeKQ~@=y;d8WO4> z5n6ViQI$yb(E=}h$%Pz46pRIfmU&J~g7G2h%Z(3R)UK;KE7fYQYMW(XEGP*oiA61o zX~M{QGPPxqf+c~&5EMfSeu2C+p(&^XClk?bz6a8c-wVL$@X78IgS!il`nf@$|NDB$ z!pII#8O`B=C?(W4HUMCFgq=NIU|17ZCJCB$GMnWC4)Bzr7vwd+6=m89RzWEr8RM)J z{<*r^O_+;0*v$SCr?NY*GBN7g_XvJw(Alr*JdX18+ayT=7nh7rspk-m79V+uZ6 ztXPBwrw}83gy0Z-5~M&h#3-twLItW2OR}i1Hn+}Z0X4*|WQHvuk4GAF|nl(&rxlAJSLg`H_$7j$~DhQE|0Dw@1_{u(SYA`|<)c1!z2nEiN z3pHh3+AqfnR1(*fL>GmsCW^vKq6*6_@MaAdgbMJML`Lz-=uzSKlU zsFqzuL8+GWEa6#F@%cuL@AywXi5H7I{|P>|N|L#(X}-mbS1R^_3F(J2Dr&u;J`GV7 zi_rR73M$!@iNc$*HQMKxmBV7=V#HGOu4H%Vih!>!ymS&azC%I0h%lr70>a zSqWk!3{fj2#K@hRQ&lQ?O%4C~5TE$H-Y({!9bFv(ID7c25<8z`6Qcw^Wy8MH!@wTs z_&D#cJ?h5?nMOw0oeF~67El-+8W|n-!Vqs8jm-Xdp@R)oj?-(S6^0b{vVbd_^K3@3 zyaJ!pHZSj1MJ4!jPdT+jf<_|_a%MNBI8Z4-)s;}GNur)r=Sso!#VE;m>6lPa0S2nz3w_5Z z9MW#VQUkq#)1rnw7)$lbp?tfdt|bZdk(IahD%Ce&NZ&W$TpnYBJwJO$hte!{uKkq? zJ~?~*5TC4Kg7rHSNJ22ay9e~a63I;?L*e1j?s1IObAx-t-@|MH|NJm8e+7ZIXab7* zF)wA8Ks*ZYEd1F-1)sc4!nf`5pSC^t>a&p<_ESFtNjPS|ghyy}yP8f{uAX6?BEFs}s6Lc1_KZ2--%7-k7 z3VshIaP02~fonKWDj-KwGf$}h6KDldlFN~5@2-MfHBnng$tV$-a=BNo~95b4-kV=we54Bg(5Z-)Y zC@B=Ri;^X{M5T@IxHmu{d;%dhGoj#yQ=`^LoP8=QFVgs~Q%p3I=3r{t1#it1Bm2P18j3=FH3=yM#*8m`s0YOZ z1L;^HQkIStB4sPbbmqdzBtn!$?Q%#${?ouF#Xw zi}Za;ffn^M8ZYCAdgqZs%CTIv9AdAp3&wZ1XT)NKr3w$UW~L7;0T0#&Y@cxk8mjc9kO_0T>tw1s-tdW6XGY zLd#{!Vvs*hLXuq+9g9i`RaJc}`3*7SSYs{KGAb_=iKR+t*#$!8qJ~CkngWcU#ZQey zs(2#9$PO@>3<6FE!sI9AIhV;HCDt!402o0Fh99s2rBIgiIYn?S2vzT#Y z3h77zDr?Ka#*2|Ay!GZI6|6v&CIFbGpe1-1FGiXIm>?8VAR5A_qH~2z15^%Kj8XYe zD^?PsFjhmVjg{saM-Vb2z(_|KCAVA%C6Su)LJgM2gcST-el^I;=2CHSA%WM2gB48R zB!l%4<1(NGGqbzv4m^Sx6mSZzCNaj;0sY4&PmIf5Pq}=R=atbM|~)FcE!-Pc>P_0i41oQ8R^^2nNH_S(5l& zX38d%W~gUPO+}fGkN_209v3y__>7NLgCausN8(eLfN~Ye@uQOXg{vlhqu{L-#$zs7 z%elc zH#NIQjV(M(Z{?tp`@^rIFg%C^Y%u)T4)T8iwA^3rP^zdOHD!)P5!@Tc92((o)U>q) z$D%lhPpIcnXLEA{Z#A`Wda8wQe@FB3Z@qlkYI?P^_f`sp^?oLW*oDd z&yXrjnOBzajgK->I#fdOM3I|N1%}j2-?B(TF5w3%-l9H)eBRE_3!Fv`gfP;lKn0a) z3Vvy(>DLh6S|PbCgF(bVg(3rw)RYTqC71-p@`|w}l>Gw5ha}4GzP!Zikg=*8H~IT+ z7x@FNyhFj08lKniNRcCL{Kubx<82FmoF?L^2H#T1&(G)Fk(h$nA5jUR`XS**IwkNU z7d%Ol!|;cX`OC2GawWwS&Lx6Bl6W~U@8<2>H+j|l`i;xC^RM5yd4<1m^Y+E_r(Qd8 zJMVh5wvzK9eD;Yy)f9Zl?nhoxMhF>1LNlpSA4$#Ev5RmJ>DA~CegwUTd4G!=a$otqblG(+X< zM$j#wT*<32VVmu4F!0h#M{;tqvU0L>)~#K)K6fqepRZfDYu7GL5x?}(OFMV&7YOIhy}_)Q_c+++|YR01={0K0}4LPACiyF*?Pu!__RAsjU#R!we z+l8zSetjggh8TlQ=AtEJDDt9@(WLCB%HT-ENXMem*St%)girC7#T8!5N9s!W;z3^i zjrALHR<7hV-{+PtU$$haqf3cA_Z)9l@?|96qvW(O$BDrh&&@SmWD3dwIUd3DXp;Am z?Cm5TP4c{wRImz$d4LWeIV@Mr%XDmMRaf)k29nIIH47InUcPGOw(Z*}pFbjZ=n#LX zYs2#8OZV*A%}1NuCj9I-KVdi%!m;__hAOYFDPv?0!ozuvS-{%pH(AD%tz-~W6Jxz& zy(VQSN$L(Ii4W=fxh#u>1`@#rY7#3D#!`JI zRm$qsDJi`3%co{I&CaV9FTVI9C);tP;0?#f{1XJSCB-9n0BOfXIOzdsk%M8pD_)fE z&;m%o5|nae4sh}Vj$5{E-Mx3ui!brF6#3(`M|SVtK?RmCe{R{brEk76JUw7oo zwCpB1S5IyEqaL%!bo=wc7U;ufhU}oIEN@9>95Hiy&3Y=7k*Fn6TBw(mYSfA~i7*3N zkQkz*CJ72A7lXtHq|X}y{an6K{3EYrH7C_7AYSc4_&9T6G-X#t(+^qIvOkhR+2q0` zQ9P~KoNHeCJmo3z!f>$P%L>+1I$2_pCD?d<#`3^(v?f6-#ByN;DHxjc=7Pz zL;RIboadS~84DNAKXhn+@Fzjt*KvhRM`OXXGdZ$5d}>)_#<5l#V`h{@QK+K6L6Fc# z)@A4zPm*OcM#?`yRe~u1Ibj9{p+V8-O=!wKCLhNOqI1EC#4p@xVilX)7L%B{#Q4o&48|77yX z@8qcyZ(n+guZ0Nr=ig zYO+za)S_Y#(o-R!pQK#Yn=odv5EIf-R2Vbf*Ab8mgrTU(eQ?@@GXzfZB7x2gDo2G| zDnhH0T?mP+;a5^Z41(E74H-&(Av=&{jHyXt)Z{)R97+POAY_WC+LX}P=)NbvVdJ{= zbfB>+Jv}Wmlh4LwX0Of3-ME2e3va98L$6=I#_|V%oeX}wWy!zzMx3NZc4nS>Bc5}c z@E{ha%5FbHJiz81B4J)KF`9~P1S3|gABBs#t+%uv4E=T)rKd|q9K zJj5spwIu6$74kTJnTs1R$wGX;~VaEroN4OKU*hN7ahJrtESpnd>9 zzMLi#MF_eqNp_H$6s!bGGKtj4u0S2A9z1WTzxCFeygHegnNHvjuY(O%L*O$#KOc-o1+b;Cg22_`oS~AgHJ7ySj#g76-2?OUMT{f ztl$qE+W+#gmkz&h==h0aOdG&wc6K%ufCdbL^j*7lT)Ol&f00(rgi>=}7?tzHiw0Q5 zV#z!Gam^3O6@8>oePjyQT)gZS>gc&blqFO-zP_QtgeXf0gdXP%4SvQ5pIXJ6(9cep zJ~$tTPu0+A{PByc_ii&a`EiqCvO!JpD4jw!RN*VD1o;gpD$`XyfDB(HT1b`koa`4EDhe7F!FTKDLjH|O8gHjcK0F~LriWSQ< zGSV)-^(NmYQPZ$q&=3sbM)@FsPq$F=d};EQmwo0^tYQ5EFI%BzLd&8glN;kJo6wLT zcnEva6lNq5^>c;bjMCIaecmbn<2MK1S}o&^Qxrz@Lv%v&Dm#pcLZv0n8YcAkS&JNENTE z_$8Z4nR{=!2=&IwjX<*0989nmoHyt6A1^%G?D=wn6WPe%owQ|+;Ni3 zHu0JEUWOoQL%epG$V&<2#f56;6aUFi3wgXTbrLst2vCdPC`XxK%c60rtGW1`A+NnBKURaL8lUvsK$CPFUD=q~onWl2&X z$r$7Pq7cd(Jah=|@wD@zweH16_jPT2DIbI9k6++|($Z5hvoqK}0-sBlE!CeS0F!c? z=CuGSi*y_#-~HxKy#P)i3Y=IdF>o{U1c5fP>to>jAs85bAQp()5D%&~J>Y$I)<@uz z$DMq&o08FFXMoKm04Wy=P?rWj_b4m4WgJ{rucE>%o|Y_Gv}ez*C~Lhq+o?bDin5YX z8g2ohDN8{~l9OaXsfvm>9b;6KY5M(cVd8g$?A=2R6ni32J!rb0L)H6cye~ITW>`RDwmtSotQm3#l^JDAhk?9HNHCxn@)r zqoy2^Aa|0$f8sWof4Zyirv!onX3B9lyhI8RMA!KZE$7!+{q+&+tzEOuK88ENb@Fv`Dv`Xo<0UDJXfBf!gUW)rig zX@>M4DOYyC@8GRW$%&E|YEkh5Oh_3WLBbe8B?K6v<{{Bns!1N1iqOC|mr>p3%E{$d z+i!($9~pBl6w#4k{Uo1PA<_{q`p{hjgS(q{!qk9L9$VLN{wFncB|~IJW_oIR3iy2T z>8GA~<{8FuTqQq1Vz87D&?1C_6Bg_oegk9R6X(ge4+0s=@tI)q!w=mb0pZ6?018}z zs9Y%1R9q+!Wr*j8!(`gY=Mw>;#&j;hyVSlwYK6{%g$ouxw`9dimTb<2g5@1M zw{W6~_C~6ceh(u*(rCWj#0xyW-HcaHL6L}8Kl%#FZq)B+c!5DeKMA=|VQ3*LbV*GL zCNwq46*U(sH9x!RBQ?q4RkB4*A~jOywn8mMZpJi5Hj%dm*#GGMw*_^eF)=hEd%pL*tLa41Yd#efRXi09OZ@3wf@wGZ%v7iHea5KrgdUTqvs~TrTUQ>({U2fsY@3jP$g{OXe39=jn-U$YPjfu8CwT@a!NRxLm2JgG`?& zeFv4q??GuA4JDbrAOZC;pr{WgBt(4)`X*6cD?zY)B=t-1uR>nG4t{|?kB(a8D@$lL z@nPl*jW=^W&hom1={v^vpR@Qpf8osPloe}pva_<+q-UlvqodO^*!v82nE(T*aFM`~ z;toOwLKVSwWYOwT$zP>bl zHTcKXN*R?(HT-h?EGmsugjQRu0WX_iE`_-gxw>|=>0X})#@$!|o zmMveBo4Yn=ZT8yS96e+Q2><{j7(~S&#D9Vujq+g9z$v(bF|ZBgAQNZp*s*77%A+Ej zq;`{7s_T^yydQ<2tgDcnOECH55|l{=Sf1lNxgAs!QB@2DGExmaZv~%>?>tLdwtR7D zd^e;$G{Wqi`x6r7QU@6|p>-%pK97R0tRy79+_DG*K2G2b9wFYYB1XA$y20w4GRjU zmUMB>+N{j1jJ3JzIKIvg49HUnKZ=41@X7WL&;m|?3OUL!TH~|s2DD&FPES4%Hwf7H zkrAG}FzAp=8%I!n6i8`M3gn{bF!>~xFo~Qj{79l2M9D%`nRjY@XNzpx_DwVg|LNQ6 zCwMcy-xp>*6Ix$jQt{S<*0a_lC|DOO)s(G3p~a{a6a{r5#xKpX_~29)zYwX-&1e!U zC#30D-fF9+S=%KyD_N>v~u7oIf zAOyyQ762({C5AvhCT1bWP?oBESzoQ?7Duu1jkWKj?eMiy?Zxn z6Mighwr$%cO|pP$<3TmqiSp+*_)!QYpbR5<0Q_7OV91DEYUpg=woR9V=SM!GY_xJY zbaWxzFMyNxO&Hl1FJ8ED<9fai=ALf`&P`_VPcC7HSz!{iW=Sy?HIrJ`h(T@=(^0X= zO(F(~f#;#b1Ky9|ROuWe0n}M8;Y4x?C4mN-MU@rB)s-dH)umOlfwvkJs;jAwn@OTc zDz6PHNE6=AV!U37WekGRY>@8>h=EoIbHXHw0wW8nfGvbK-<}WoU{jiCD3`Lk5_p5m zz4_=~)p76P25*)T<;>5m+t<=F-1>sGJbsQxHMl9jBiH~8wvTuk&f9Rj2M0Z{e?L(m zx?}qe@FvUvvq%0Y2l;x)ri}!k51+eXJ+CT*X&3;yZO8WAd-w8A+`fJLcI*I@n|JTp zv2X7l-fTL2#7BcQ>MF>BS3NtFt9*5k>=oZ%aZVN8tEtS*ZLFaQY(cA_r= zk^{_86Y4|fs_UUHc%S8@o(^Lmt)jT9ys!qok|}67IJF>vNmAi8zQc2J!C(nP`GCq@ zH^8TZsKZ+_!28PLNp%);pjB}A3?${o(1ZXp$zw%-MVG# zwk@08A0yhZY4gV1_3PYXpFPuThWL#1j5Gp|p8z)dKn&f1guNdp4pJiJNaK`GQI8Wz49Uh#mYJerL8`qRMAnCT{P7mH%(*16z3#{!BNN*BgloX04 z6&{`&DU+MBe!<%Ax^ylb9iYPcJphL5cLIoF%mSqp=$v7B5#X$;ERIw;e@Lp~(_krv zGQp=5D$89V<&`d|a6StVKrsok3b0y5i^^SU5_n}WM4hH;B0g}KpxVK;+WA|k7y~v9 zNg6h|{c9p+)e%r#3NssMT}gFKiJ~M0p`<2S9*H^~r|$y1GA4OaBhlpR;egZJwH!4* zhi`6FRk}~O7Z(>Y(DNXh2b~#NX_;B+gf-deIqR}=*0FS2lbMyahQ(A?dd3>1rln`5 zre~%QC8V!`=YV}`x{Ja%i5MUUdR<6QRRD|#)6-ItlG!A;S;V3|C?|U@Te8eYUEy3E z%B5GX=7hkrOP4QMymS%6I^p>7qugKgJJV^#g1~zOE)<{Opn@wS%AcgKsBr^_+g_6$ zkW@Qs2T5-Ha_)kFFc{8eqk)1_UH=9_nn~*Xl`F(iSBT4KYDkih2w*@G$SDVA6(s>a zgCQ7TI02}u?7bR z>Ksi2S8P?X1H_xhnIOmEM|?nmG!lWzbigx^b3x8l)cITIcwK-=OHU=Fa|zVeWTYa& zsZw}GN@hTL(;3_@U-8_$XP;j3+``367A{;cpJma31AFW0B5nYxE{oSya|^rsIl!S8%>z@o12aw-bb<4wE=m_TlR6{2-U<9+5&(gVqaR)2_5Ar%4mp7SR#Rfo1zF%p>bVYfQ3xN+5wbEda_yI``7s3fUKkP!a<_s0V z2fZtpGZ4xGA(ID`%OX|fF3{wv@?@0uR+d!-$}lKHYxpavE|gbSqJ~YdY9;3xB2_Rh zM5}ogTuBL-gl)Dg2iekZEqacdhT_^}qa=E!skk?J0ToU?$n$v_)M&@HqGYC!? ztl|@#Imhkrmn(V2EY9DsJ_|lM3!0La!pOTSWff!YDo!k?^4dEeXW#++s#G2+I-~&} zw`Hj@+U@f&+OKijJ&rmipaJ9bfw%Lh?7tHn@>8AfO@k&SE|pt=(;lQc_saG=*IMn~ z@8DEF72&Bt3dcX^&wrNPq$SKg7cF@DsV6pX%8ju*QB|6VRVCt8(Hdr+1ZO~p2f&t` zSO$i26CoE|YH}9jCY*}G)VeR^U?{RP{<=_IT@sn)U9_qMY`6{zyrrD80WWYnE=w}I zn>3bCnFzOx>r-IhTqT~3$pdeuh^ee_t}i*6bUx4v!A*g`HP`CkT~$(6bgQiRc6sq_ zXeoTrt$^m073Y-~=a-idV2GmQq7@|t6;7C62~(OsD}>2IlcciZK&P~@vb3n8w2&aC zvLxWkijvXNqRO(OD$%mSsB?UxDa>7!l7U)ov(@6@;i>p}` zmXlOaSw?QT|A&! zwR-t7K60~+Q$tHvu2{yBpS_)wRE`k46GZ3}c;~_)91J@+xrgIOz!*paZwjl^f=`j8 zQg{j{PTd-Om3#jVo|F3Q$uDwK%Y~dmUAbZ@$E!%sKsjXmMpL+V~#qLu(OPPKy zS~Q;rq36%O7Gs`SRa{$LTve7=T}~)qxxv(qZq?99Gn31Z3TMU&OyMSeMb(wXRY;sQ zipWw`0h5nqD$DXK%knBq^D0VjSBWpnhfyk;lu%w)P(e`@g~95^8Md+Cro1fQ(Le~}lKA9Qc|rcQ z3kBCNvVv<;4kTBhfx^X-+muG2@V5(Zz6CA5 zh0sL{ZoKKr42qqt36)XZsL!RMWKFKjR+0+4yebpEyzpvS;niZAlYgb8;M&zoZ=5=D z`1;j1Uq5y9`90eXJilY#?yb-R`?eGI@7um>`^KHyHtgBCdH0S@dvP?AX5-p$jksB!s=Yx9!`#eb3IVyLWCu zZRhq)JGO4zwmEm(mi3#~XYbj*`H<7!ad`h;vK&5ifGyW2o_K>lH&ZU<;6FOZ(*OeOY<(3s$_6R(PYV@MdKRu+Imagwmqh*RQ;J{p#Bc?xn@IDoRPZRaJI7xW(L!Mzo^D z4>!3MVWg+th2Hw%9zw%#cf(Lm!$?o#NMF-vZ_{W%$NHMb`&%XkS{3dLwoVSUO%AnB z4R;V=ED)U@>6{p7CfpfpnH(at-Weo2r8QU>>uVV6Yb5CEqrHi-enP#%XkXoMcWkgL zI?x&EZ?Ea^hzxW_2RdRyUGb6b1X)IV3CR*v4h144mHJb8DoiB_69X+=+Er$#?apvJ zbi#$!agv7Grbas^huak$A8Z*%d7x#&CAC6L;+VmtoQz`w&7=KIBhglper%Z5vQtyopHi&cVe`sVT^`5BPW8&CxXhmCQJ^tjbYyYrs3YkzV`SK z*?XFs8={4{jPs}Qs82ri7mJoGbYCjNoiFE!>*94enG9r|t#yOl%>(U;o|b4|Tdco5 zP8jLI0!@8w@t)Qgp|>q=A&AEM+Uxo|>bMXLVW6vih`#PpsCP_v{jh6A{Xi$JNKj@^ zOKo>^Ey@(t-(E*KeH|2%=;5N2PqZ}-gBsQ7ZLJ;b1W3&kJJ#Pe+Sk_65NTa=IELEwS>(Xh}m&VS*5Gp*~tP z-kTWjjgNtlJ{QJ=exK-T9Cx8%qQ7x+plNEbX=;l`<}0Zzix6M8SV_! zjrT-HyK9C!tA;x(hdQc;JE}&yYR0-FV?EJ{-uOgc9T_JH1NBn__2TdJC7j%~B{A-J zu1Ia0riY*|%#5_mjJ6VJ#guDEt7{9)Fgmo3N)jC7^i4R}Fge(ur0F0DQ%nvtO!UdJ<#Z^&?$%!=1Gw?a`5r$XI9XcvozqJ2u%{H`$lC(@WiA6TLC$WMBMF z5bg~(-RZBx_#@qMZnuHXx+`y=TAQ_M;ew}@KDThulKD@~dvd{|1uWt;D`3N6$>N0z zo_%u1=G@An8|{gzvEHVEw%Xy2I2D-asT=D~40hD^w?+HgYWv&4EwF(>QajKd8|;XC zft#KCrJmad>KN|pBuxx=C5AgGD%Rgx+uIr$Xd|!Vhq~&(VRutaPjjTV6^{|`YmW9d zNBdi2{Vh@M!OZ_ioG0N?$?9PFuNl&FZyTDI0S$@Wx=Wr?s}d zzOu8Siqd+TYAC9`uDrFjs3nr$RFmHj$!o01Z>-Iy#{<*F9YYsm%)a_5VCez?XuLbv zG&9tA*M%m+y`koN!_D_cTJDduzBAei760y7`+H;UNJw>IqzU`q9d;R~2Pp?c1vu=l zgP$I#S9A(tU)@w+e5x;|Fxf}XR*!a8j&@Xzv{#IDRE~C5jdw@JyCQdbYNt@{C%X$X zgY`25^%OM?RslIk=uHR@Kw_%DVS1n;sCFYYxj))`F9>&sn`c}@nuE4b$)IV24R`&Z z_6|185C)+2cL$ReKtcs{p{YKuo45luy5r+r@zIXj;kM{VTXeKNHqli_xYM1O?5UsX zZG_(Gsk_q?pX^aR;bSv>iF<<$W1W%Vj!17y&9Ngp7tVWP`E&DEEML5wcjJ~W;RH2@ zRCyVn!*xtqmo9$}r^;JX%NEaD`1BKpcW-J>R19_24Yk*fb=6My*Wc-l4|PNb+9UmK zk%4wl>X`nv8byI_5Tas6y5eKqb>rRKSvZ6QVXVhRN4x9By6Q$cV?*uHf!3P-*6RM2 z>Ol%@sqSv79B7U9wM4oaDtnr$dzx#&72qH1X#l;^vK!UKH!hw%@%r(D>oS(5uUL?q zxnf=B3O1Y9X0BYHowhb}_4=HY^*O84S1&$(WOrMlf{_lKQ(AX@MPFl8S6x|0ZBbh! zza?_JspeKoB(FJ|ccjmkC>CahVsXg$w_YyPFvT#}8z<1$ zg4slO6ex~$){J&kLkR&g<%{CzfXM(hjdur{C~Bs^(N)-$=8(XJ9HKxbh#~+#YCh6@ zf0SysK!fTza3a62TPeV5px-1)OP#_X7I3#GAa{3MfNE?q-A`K@ssD7pYsj6Rh6#GP z6MV)-+G~gLpPjMs?z)Mf({btUmdC5R-p7Ro?$o*Z*u8;VpVjk~D&0oOj6Sn)82j4CWzEHY^=bWpTFJAEU6G!%Kql^1mBNILG ziJn^g^C-CLhz_<#2m@_301OI`aG2Ytvvvp?Fk@YDS4js^S5?BD-uh&<9Yo_aWT-7d z7-;480iw~~)@s~oS7UW&L*-CsqPscTQD0SCcI(QySD3|b%T3+5HYF#0S!U{DaJX?@ z#`^5k+?aTrum%_LJJ54hoJGnc-sSz zIoA6AM8^krI-w4qpm(_SouRh-LAX26GSk;IP3Uc$>1!r>Z?N@le+$t7OHK~71S;?_ z#WbTcN*L{k5X4V()jH49TQ}WTKhxg?ED84pTkZ|C+#P71p|rl{>0XK=G*9<|+eWa& z4RW6wYS4j@8!8zBZ|}h7Ld*SOyuAyw)PfqGj3$i$s@np!*6D%fDPY+TO@=$YP2-G= zor#eSLLI$7(iL+aiMI4603;I+=opizo|q;l4C`R&9y5yGx=|cpXZ^w5TNccFl0`c2 zIWwFuTh6=BD|qo}{``fTH*G#~{Mgp5n|X}I1{4b=CZY`WOBOwgBfWLyEOTskp}@(2 zE8Mt(16XT~bLs81BTg7|p);Tz+(7|8h0Fk);FJ4ESS6J#fD>2-^8{!tm|C+;XlYGU zTt0vD%&RXP+PihzhOE^~=clb)v?gs?ZZLcUzjc5xI~7>wtO1SbYqQeU@%Up+%7%3r z4x%|}fHpnlx%Aab&b)G@vw`_%tRqp;#c*F&+8!%zixvRRmYQ45k=u-qQ;g&N4Z(ok z%(RU#Gt@CN?B^gaWz=rFKZ>htbA$MJ=Yt8**!92#$Gm^1`~5qPC)H_=1R>~m z`kIm*=KFAHV8FrN`A;r~3w4g#U7CTIz_{r=Kv%>C{3l+NVcpF+nX|cB0d5m-#snid zIx|u;_h6ohi}nH%u61t+MCtwlobA*ts2(i}kj8X$gef$kWoC938q8PSbTlbp7Rn`t zo9~Xc+#79?CUa5dy>|xOw44CL6m+zw4oor+90>qG=ep4xWcDz33LIv@kN4Cv1k+4r zHEw(l+yc|K*7}OAo3fsK@(H%$7cX17kbS4;+?t);r)TFqy?y)U^XJa&-@ljBRcz0) zAIdX%7V~V1&U^ZaZJV?EIufjH#~Awt8%M#FTSY=+@|HuLxXsvbS1c(+9t0-LW8`A_ z*3&Rf=q1pWMtG7&nSyr2hnTezI_p>_w>MTd)Ru5N9N4>M_cj)4E3z_IrmS3?nYJ=# zO=?!g>df?2x!LKw-L*D5osaOrWM-^hlbM>sv$@RFHJK^o;qwET=_|8&+?KYIUFWRK z)!7*0cfIu+m)<@G+!!3duSsx+1>ynsAw>W%`wL(#L_zV_J0S)lk z@9;^dg9LmI9_&uHO)>+w-_IPz0dk_dmYEDw6lT!@KB2Yb?QE{uv_9*pr=FO<@R^0r zEt$^)cea(DV`HAjMo(_rvf+(0r}pgK!IS#sE0!-rCv2gMZNmyzzE2p|JYrtqwTkeT6_tTkyQW%Hty zldM*jv^2JwR{_yA8LNOOv(vZFp6qUp^)yGBJha72TWSlNfM_(|4UT6VtP@K3V4|Dw;hmn3CVQbDI=H&QozY(t02?#||QeRlpytDPTqvju=1)G&D=H>+{;ZEmjFn}Sw2 zH3j@O!y;p%8(2Gom3tRc!6ZI|d3Bv@Ad45KD6HE0m_IkTl~I5FbZ;$_?VB$!kH znFJxzU>RW2fm6ZZ(@j2sF%WH~L;V@u3=_^FLfdSgI$tVGf=?K?-eHd67T^qvG5Hy& zI+tbjLrezX1XMw2br=&wXE85mX(fJtxRD9-yQ7*aCzo%`mXW+0;FIhsDnPpfr-Bgf zE;{5cq&qIyqYy;F=R=4JpDZg_^Nw}`hA8VGun}w~Xx-;-LpRn3fwfZ5e79XN!!A<~ zs}=SzqnmTr0L4WM=Q+>#+;j66EL^yVGfVTIe(K2u3z&q?Bg~us%)I$94!X{{E}BP{ z1G~3Rk8})l)Q z(ub(&7&yfU{5cuS5LR8i@G9G(y-hVNmwFOqw%nk<HI3-6hmo*`@>D|j<&FlW9|0gWS6br-eWEr+(4M+ zKk)=lJ3*-1i+8_;4B`Y%e$HR$&N8tby1)(ZJQRKQnWz5(e4amZd~&#R7}O2&Or&Xy zHFzJiailB3))U*9Y-X~z2?80lUDOGg12o(l?|$bFpN)t%M2ZTpU$}bV-H7^=SNfB z1egaC?GKoUj&=y5f?tsRAwEOM)WpCo|J|0ayW@1Psf{$sjqhyHu;B^(@c%eZH;%jE zAQ%GJH{;nv1CO!p-|2b(?%;jy=(|JrCc5skqMYiU9_ySK>3nd1qVV>$Et}V8a;}N9 zBpf;6GtTLpR#`>hKolngI1jLj10;yk`9A)NWlI-5zi($(Tm9r%4-dj-#yam!^vsNP zO^)Q;y{@|{Qkr-1#--Co5AE8Jy?SHL>dchI z8`h?6+nfX4yq@7ZV?%D{x}5afwHdi}RFAB^cC&uQ=r$$ex>hm{loJ|{aR zCo2VH60+7XJ56KJl(S|vJNdb5R_11`IIump`1+e{-K&@7IW>-p0z3eT&-`G58Dx52W2^y%TsmQ6V!Jnp;Q>HnQC4 z{NvCjjLkyS4a5dfN+wVO&@y~}Jlz9C125`sKxIcE@QqBlTka1gXA6c;-50v9G}%4l zd4%14F!G-cpPejr+-fM;sY~J$lrr<28ER(yef#Xu+t<(EogDbtw;%oR&4=HA{oyaZ z`{)FS*@(3Smym{l=_8r@H?B2e6_s;Fx zw(Q@(FOi5(PK}R_4UUct4fJ<^`2PK0eE-EaUwrWM?>_nYw;%uP+mConngaMffzEHeFV*pE^@+^rtElVfB zNeC8MDOnjSnTTekEys(lS-p76+SFT@|3Ajw1G=r`%=7im%-Zc%OCqV<$~os8sZ2>! zq;k$VN0MSD0RjX80w4({L=wz7DYm-ZZik8X*lwqBJN9^Hc4pts&YrXT_U+q!=e*xn z1zbVuws-rKuL~C!ASwLt|6CKvdKF7Mu|^ zR^-5-DS^5buO@an_ouReUy+1*iK&d)258-HhHX1H7>&z|*v@aoyi51xMT z>gm%bT|M2cU7amGUCmwX^-p@+`j~Vybagg#wAOaDhvnEB_E))_Mftn8uV1=g#>^>W zCJ~8@h#xmX0&o)MWP~EhM*e5q$f=Vg@L_1=cuFL3!f0Vt@%csk^pysma=nU}^xGQ8 zSUA;u7}?yhV~4R#^;ngC0@0Bp;zx~$BR??a{R9%Eqld>$9yjcZaB4r(SEAr_=J;JX z0e4=o^fjNMQnSIQ=2?I#636YWu9?rjb+5hBO-^*7op1*wh~zCTY9@%89iOCogGcx-7Q_6&CSgq+L)1ccmABI^k$I2 z88=GOIE;y-hfS1tTtaMel^Q3Q(WC*Ve##(9!-=D)FOEggnCmyu@1RfeCi)#BKPcj4 z$`93iiZzGNv81OWgS{w*4;wssWE{Q>$t%_nfG8iaVbFoi>yQsrtBrBge1^&jBBRt4 za29~EexX521kR`+3$ZGk2OA>=gjB*I^F<}D$b6F)@QR)t!5IbSGCB$*cbo|;cUjq)j;0C4INvF|jB zDE*|iE@6>wFwjXe=2Rank8>X1TEA{-^3E+QRxY{s;C4@ccTay8_gSag*rN$`<`^Y zc+vm-X>WgD*OUJ4-rml>e&BC!Yj5i7>pXSx$cW+b2;hvV6J}1EIDN`^8I#6MnvVsjh$#*-{w39QI)g8#7l}^^QjZ35ko(E zI2)bnQB|B(LlESEOg=l&Ot1t<4W}lPV=STpHz7QJX|@!PqQQ|Uk1~-_K?Wf(kOIBI zCva92rNWrknMKM}^QmQ1W2rS$@hRe|e+LB>zo8zgVx{C-D1DvSjAUP-R7J>zD7{o4k4J#w|&ktNk^vKK}5<2hU%A@ch~H{#UP_fB4}G5bf{pYN!w9 zU~B9~1G`7~5CRx=G9W@4O2JQ;YKA^>! zWlDdJ8VRg96V8Aon#n_h&tPc*Pw^`C@rm~0&dFyT9bh9T)ev1$&LLzH_;`v^YXr{h z8fO;vz0$7B_YqU>b7td9Fqo)$t~eFZ-^A-6Sj+Rtk*`LCF(@(ytQu;p#504xOaD>& zm*>~!NAu0%9YB;{*m!MKab~PjYM6Dcx;*4c{awH!7;ztbV&bJPOm|AIG{B&rI50Ax zl+-R|FA_?H>3NUu(g+QoJ9ctIu8l`eo(MNJe)HQOfAH$%H$M9C)vFgDfAR`Rym0Xh z`#`Q%}J;% zpAm59@~Ii0AZd9LKExMDr6!Y){UAHD_EB92~f>tu$ zuyK9j`gI#OZP>DP^TtgZlajV>+Qep!P`ZB2T4-LmfB(MJv`4jd{-@7+J3CwOOWWES zvG|7%rBKq3j~l?IydVUef)=C^`$SQkD7i-&lgHC#B)vvzjFJFqyrd|#iP!PFCX=^f zqM77qfTt(1K5273upAiw(e5yYLCr665{MtO!@2SP} z4L+rxkea?!@EPbQ!)Rd8jV`JHA#H@X-2By&&zJh+-JRRkA3u6v@7`TIcPH=My>m;_ z))Oa=m6f?a{O}doq?Id|bFZ$s^XIHtw|WygxoIgVqU_(W zW%GIzH8BxY<=HKpxA1V)nw5A0-0oQuE-)SnmQjaO!^v?x zW6TJHOhurWa)xnIKLi_;Q^TX+|S%Uf+HEW+ky z)RbgaL%1``=hh-AZ(dtY@FhQjQClt#BRECAAO)nhG@=?#j>4*n?tz@&CzebDPBLTq z`^6s;`2*cjUjTL9L4U;*koct$Xy1ZU^I7enYl{vOY0%O=Fz{)(phWQuX>FzUq}r=U z#;QLY+d+#|rZx8wE1piDJbd)%p_G)pJCb*B^WyE>lTuQ4pFVwj?Yh+~SFM;aYsR7_ zi?HU28xj{US3R)^*0cWCM_exiXKMl1UjAGAI^{oBkC0x@~C;xKpH3YX9_tC5l=TU6roMTbJJvU z6gG&nilu3U09nLSw<}9Q@F5*XxSG|D^cpA5XNK36>2m{Tw!c(lQ^Q&3F~MXIeJjfg zm7D53=5R$})F>+-SY==RPRY9(Lnu{+AqACuYocUg*|ifZK7;8jPsQO-q67O!5rW^LlSWh<61SiBH@L>~3- zI*XSsg3op9*REN|F>&p>L>$w^#0{(0tev-D{*q-&5;v@c-o;B6By!o>m1obK=24?Hnl^RfpaEYU5;p)uM+_az1Vm9MLjaj*J|me=mFkSeGL}z+G$sZ@u@kL+#+m=^ z^;{{}C_Y~wqj2&RpB)%zIQay%NNQ{}mc~3#nqSRliddDX7El$rdy8}Yj@&>=j@O>W zTX~&1%D2q)y0SgCG)5(wS(fW7WR@_TZF6?yRKFXe*F+KPNexWZglZVH#1fivhS z3{_e}RYkRx);f>1w$xNBNv-@~xkx%S0)SK?L6^zzH2F$YnV>k!XP4oJG~#ZIzUXR} zuOu_z%nZ6RK~(;TUGfq5|B(P*HBm*qN$|{4*;=I%ohxV`o#_2PskDWYm^!Txql#~MpIP4|lA#7FJ+Tf+MmfU_{+QCa@=; zDFo6@N5D%=L8{yi3CbIE zR~A;fbHYBG&r{&{mOpaPqak(qWuNLQ2ki~*$ict1!x;0XEZn|;v zDmOhkeeS~PGiQz;J9g~&(Ve@Jm!W=%Yq{%|3fR{$2ULP|VjuEKIGOyXvDD!Z^Q~JY za9Xu$m7YT^xFWXP7GxEfC<3YV>y8{fNR}xx;}KCX3+IO>0H=&$34-WQ8PZy;#gik$ z2%UgaHx!YjR6}L*-^gCZh-Xy##5(_1bD5Oea?0&FZW~LU(<>b4JV~&8erI5dqy^x!G~WlGqMs&c7k4Gn zzly_<>sM|7-&TAZ^{NP^e~3>*A+>}WnkiXX8lj_n;^&~fMLGBP?$~hu{+;VLZd_nn zq6-(!o;`c&)QKHCk{}b86QjLm&8o$ib;T!$u2``g-xDUe`{C- z!6!6wQqwDwM8SB=)~$H3N!zz>+OomxtF&73M~-0A(*Z+>k8nYU3}$fiham~V=kSDq zAc}EEHnoY5(Mjpn(8EB*m>7UDnZSr5YFrRIdE??YA&N{F$L*7zv zxvk1+0%K4Nm6`l*;KZ{v`LXN*XT(gvC&TXqNJ&#@Qxc1)Vc*qame=+j{fy;PD<@}A zPL;OGt|-n$Kb47r&uqFY*{~x0@uNHUxUuUUHrcs!>Efl!7Y`jiIBoirb%|?;k4O_O zSU8_a;`+7R5q`zW734sPMOO#1)It)QxSh&mM>rAc zhRcdzSrlv(C0~av#>Ff$nm9%V3yJlWF_3?W@WwtHJO5IQxuCuh{lvZ#l1TbR8FaA3 zVT+h}4QnbIGp-C;ZQ`}gkUO1jO@JJ z0&}jZ@YekYk1`+IOP!hdxp{@=+(MJpVb99Vb(WTyEvD?8M;3FM&tq*0I78)#rzPyQ zhN}pU3c}_2bp-MrbD+ZPtFZVhY&E6UP>s8-AwbhkeaPF~5bo>g{`eaoef!(r`Rl*_ zn{R#hJD+^(TV%ZHngG|XmSC{bE^a`XB~S{J;&v*}Q=%~bOwms#-mIvljtBXM;^(M; z1j%a|#+`^s^J%!Cs~- zdGg-9n>Vjty?*`7xwGVj;iaT)m3Rwr)$}Jjtltdv}tk-?BMz(0gBFX|lpn!x`a|GpH!0A9`U}l#nL3 zXp)EA&V=^~sE5%r92@d*4a~%HHF4~yNn^;S%Fs{69*yf`PrfzRRDbD_D;ugTQ9e-> z)DkjtYs*RUXbBZc1zKlLEuW(!E1_i|IZK)KrwZ9TlFkhWc=CU;khK z_22&H^MCmLKmE&Z|Nb9;{CA)I?WaHe=C{A|;YU;rd;5A?J6b}4N=KE;94xm+;FQQH zl0(x3YEy0miLre0RK&QGTvh7<--?nnrG}95kjrBE%a3t)wxD#{mx-NhN7MHKIIZ03;8M3jmT-DsV1cz6?sq zH6`!dv0~+N{Lk-v_v2;D7Y!KjRqC5x0SHwp1-Jqg0tsNpF4Z;^2J+sZeF=| z>F)hoX^$V~Tk@?%rmT#+^l~@&O2SpvW}l7ki||>Y)XZb?d2HS)m#?}!J?&v$ZgyE& zX=77kdq-D&LvurOdqZnSQ+ro)M|XWoduvy3&y!~#e)HpReDgzU*>#~Bf3@3JVfU6= zeWit{s0!?XW6{8iHx4D8w3| zZbBb7tcOgpOiD=6(oLJUY-CGaz5z7p&73urnDWHFnKFT&ScyqJd!YD_CuEMw@&(xqH5pOA{DNK6q}5w`Ex zv2Et;smVK&FzzKz+lb)_g9d*63w(-Z!ly{+&_OcbQwe$KpKb|KSjS4zl`!fO_kx} zzFZxIL)IzGo)8G?C zYfAE}9Sq$?lLenS>38buLwtwq&Ju^+VJ$R6X;v1Vcy4-n8s`4w$>XrMkiGAg3XtGR zVw2l@EnT{V)YGa}BsrIJWN1pcfb$Hn1WpEQ&z(1O%a--@i2mhYeTO!C8r0t#@HK73 z-!$=qG@pW~VdWDBlDfyxLxm(lrJd>hDdQud{agBGpct#4#>p>E3>1yY7nwAE1TWW4 z1$tgzN$&AApUUkN{}UQDpSto0pRgLNFjbWn+`Dyt`?ihSw{InC+`4VY)+92+Nh^pR zSFcFgzWLzMgDD3OA3J#_Eh8)6Y(~>+{59U7*X?n;E8K-POP;BKdsIJ6%Q$}e{JulS zj-5Dh>f}N8g=-GEn|;p48b?Ew4L+Nzi|CcAt8x_PXYvu;zI*S;i4z;QZryue|FKi2 z*k<>BYTA9wSQTeJp6mtyoP=G`Z2?p@za*QD1uS`K z&!6>|mX%ifycOl1@-lby2HaMY2~ZzDew=px{8=(ft5*q)z-BNBd*to;N6;wJyu{FF zm>`~@%NmdwGo~FsdF08{p6~qmC*S?@uP0of;&+Z3%hyl6Y3|C>?uv4|%b8t}mv1p2KYb>7&%T2P_g}kwsxaqnYslSD zUEJb#HPzU01skg@b(NL|zne{E_wC!a|G=Szi^3@x+?`34?mX=q9 zYQt^qoo#I$wRK^4Svf{OJv#?%HeV@FUBf=+3dL5SRPk;Em za6`DEsWBJ`)`kMypp$zXS5|Vfc(>JBn4X^6-{0HW)s~Z!$=2?hH*Z+AYUT3fOLRqv zBb1Wn1Yk`j4|!HAElJcvhYrwU_sJ(8eeZjJ`SHiEe)Oa7oj88@PyggAETbbiuf{-j z$9c^sOsWuGa-q@l6hIl08XYnSHfRl<#QLK@fSEq8TA`xx`xpw>7-PC(|Rx_D;Q?b?MDffrN6?`1tnyP@0Gif=QnfX}{ z(^HS0IG&uGbmjb!2EU`3_#GG2Yp<`iHP+bbt1#~QEw!F|x3BHlvuF04d9&xtU$b_7 z(zfKICr{^@t%17wuCD&xzGtsqeY3Nx@8t*I`1qUOdGYd9|MM4>USE24*6Fin9;ZLL zcjsD}y&zOkBxQYyxvEh_=R#E?G$Nkxse^Y^R3Qp!@EPft5><>@X@bQUqpubQ>I$jX zF4=xMPhi_mZpxL9iPyDY{7-jbnkoCipMU3*XV0JZ(T3dD-_hR6J!HA}absgR81&P^ zR9#*9>tFxsKm7CWfBcgleE)C%vaPN8%9Tq{cktkW-MhHSXY%gdJL#swxFeWr*R00X z+_r5iMN)(}5b%BS$v1!g^Plh)`0Ky=?kC^;@Uvh1l+NU@eB~<%@dG0@O|&t}p#oBJ z^^uHyg38!OYM_xy)5xJLvWPZ2>6LL&P{HOkv~R}mhDdUOF^@$wRyMiDP)lgk@2L?r z-e2)a3N*r}lFfpKDl-EG43$JsgO zE}T!=y8iz4(_KxTaCKpm*VY~?ZSp!AYKoeCworwMf~l{jocaj&72v)vLZEm6WIo>M`YF z;-%2ZV4`h$w1Or@3K>!WjkHVY9KN#JqN2tSM5E1GI%cOaK-Lc`?WM-5OuW1V9VUiK ze{l6_f6rh1O?_FLcn_y<4y-Y-7;*{47L z!S}xZm!JOV`+xiWzxv+a{P~}M_nVtHt!GJn!k_^LpVDRb8c_qzh@qFDiA+A?Idm{7 zDV@C^^FCWNL`=Ma*;{<-NJ^XmwYE+lwL%(aG^=ktH28drNdrzMU*r?tQy?+P-63MU|&ETvuMJ^95ieh!nG%-{e1 zH^2SuFMs>n&zOAv`L90v?5Dr})!+T{S3m#7FMjg#zx(m0pML)bKm41&`^8Uw^3xyw z?B|~{m~iHYpMDSF{n<}{^y5#z&uD6_oi%gHfOr0E$Y3^OR6QX|9t{a)3;d{fzUC8p zmC-kfq2iZf=;7xrF*F_$MiE2lVUP)&#xR)FXY@nk<*^e@sfOP;-&AF8#F~5Z8$6~4 zyuOM;Mst<5snXh5VW}@Shs#WQ)Z-1(xlv`o#IIYuaOjYCM~qAuKWW^&Me{drOFDG) z2={5Ydi@&pqN~?#9Xo#d?!8AQv)N|16$zErLaP-aY4)m0Z#)~xf|*l?hzFo!hhg-aKT9rniNriSJge{Eeb zTpw;~t!-%R?tfC-&`{e@@2?Ftwzk#y0wqq@u@gs^Enhfg^5~u0*H*d;>MEqfAI82b z{iI7odQqgfUr@_W5l^O&bQVKS62+I9{q6z=q%m~&oj!u1W)l6>6ONpO=h`yfv(R5^ z@=D`Mz646HY=*DAfc-kW9{WcxS&rHB^5v5cUp@Qt?|$;bANpZ@BXKl{~ZKmFw|e)5Z- zfBN&E{pe@F`O_c%^v6E{#ZQ0qJqDAX{P+hy{n@8K{L%Li-o~cd*>k46_wJwJOb%mx z2Q~M>?-KhUp+kne7oYHMeEa~^l0kLe7(4_~o|1kNw!@8kZ)n`W;qe2tc)DQV0XrEKVsV~-if+&1KW>cj_jR>C& zo6~_B!Q81s65eAYi)k~aChbT%e){C)YgcaFx&0vZVP;k~;r5}U zN3Y+!!EMn^g{G|B%=FCk61TI^YG!`<%7yt0=aAfZe?(kv#@%|qv)*3}pUqxdi;QAY zpxE4o8d5`4JRO(L9yxmKP>IV`7p`w?Ztdvp>*(oy_3=0RpTFeT+1vN@9&Y{!QSC-G7A>Vk>^B;cy+yDHZ{^h^@ z*MIvT|L=eMpa1K>|F8e=zyGiQ`QQJ?|MOq}@Bj8+^x&!g@^Am~zx?Na`L}=l=YRcA z|Mai_*FXKsKmYze{)fN+XU0E${!jn#+kg1|Z+`##Un8xIzyJJKpMU<#-~R5i-~RR& zdi?HpzxnL5zr#srZmFL!bJDv5{&ZNv0QejsybT=6#cn)%eMMjjH; zi`WhxIV>LS<%PU_?5LrTIgQ1h6UI!H=te6fObVIPCXZpTIFuz$kZDk72=7~DYN<3R zH71%>Lp*sEeCp9yDdGv7E#!8pinM;hr$!V?Ip0Jug`un{@{|;0JiM_ZDRJSV*=yFV zTD5w`@smf-Ts(XI&W)SYI`7{~&rZK}@Ako?M^BzTPhBZBGcB)>nEKAmJ6G0kTs4jz zO2>~HK5XEU`IEX^su9WBYD=@fxYb|O=HsXHJ6OptHB=VWR@myR9R-i?ojY^x@R5@j zFJ3LOJMt|Se{F3S>lOR@19i1&RAX|p$%T?P*tT=$vQ=vrE?q&myJqc5KBIkmHhaqo z;WO+p*CU~-(*RR0GzyvKa8+TLTmkWs8d^UkjZaHSDZM3vXh3!E6aMJlduY^T>LU+1 z51)1A7SUdOSJj>>NdU?j?wnvnVTIGwQs@7R?|kdSk6yj{=+y@dHevY2^AA5_D+ZkC zryTYC`HQ~i&t*J&-pg8embyQA($)W@i}k|ld)eO8D}xpEUEQr6T`ldM&5X8=rndIR zuC9)jmL@vuxL@;NWRPuc2Yp?XaL|Ae+`?u^+^`{ok%ghdxa9(T(jp?p9Z)r{Wb&D) zdgx#<9-c6mK}3)6a%3$qz#xYDICb*aX#(Wf=~Ez+Maq$6Q2KyicL-Xxx`WTed!Y?y#F}$Mnk|Yo-=}s zgi0z2IfYn&lu1LiMdZ3t5EW=6_MMOrJ~_UzYl^|AMwFu-+PFvg1Y^Q0YN^r|Rn4N* zib7M)g9USEtXsR9MDg;~t5-2wyNa*29%~a<(-gmM{Te#q*Q{H~TDi4}3^}7Npw>wX zlU{L5jb07b9MSir2ASuTtCv$Al|m9d_Ul$no;=~*0biy4gIg^OjTjZ?j&vU~}7)ayW} zP8_R-$mtZeX<}8~7Z!}Xb>=NxHI{E#f3P&pFli=gRxT)fBAajV36>g8CM}gkZ8gPQ z(@~)Kj)0}jYi_ABqn}OH z#rz=5y|>hKftxyAzI6Y={j|rabg5nC9;G*~p5;Da=g;m+N!hw>`;{x#j~zcrWgvN1 z3JY~MZQZP{WO<`%ODH6X0J{6ogKEh2^xlpXO zDdaZma7n`R=rSmcrB+l!Ji{J~;Ejtbe6nT=?aiy9=vG04$h8^MCXg`Zj>y!97A{@F zIu4rUsGcubwv;1t%I>t)>4$W}bCPT5ms`4w`SPX9mP_rEI{ngRidE;WF>1VZK4Ac5@b~!a6+j*g6M00#*J2;A+m5@Ek=&)!t@rF zUwjCJoP7O0G4a$}C*e~Nt%e7&?0P6r#l*)B4JS`^2As_`xQNBndTOieClBqIG;YYE z1=HrvnKF0Yv^n!Za0a(gpE6_0)R|MJ&7MAc-mF=3XU^q*>htGLpG-k)$l|$^?S*L_ z^`2J0y)EEq^*g$2%esQ@&XA)mU~Q@?Y^t$vvWcHkQw&HA{^G|OH_lx=dh^a@+9zp$ zx^s`}X4*YGhPw|>oj$*NPs+Iq7Y!x+qg2xSkvT(Hn`%WeI z^{BGcBb(LYQ7b-+L_Z_z2(*JLm_>&yS<=G*5pE!7E-}Y^rHmzT|Va>9Kw=T8SRW?fne%7Qm_K{w+}Shc&Yr$#?#$(j=FFZs86T1jYzMygb^1djWfU*Y zB#S$!cv6+xk-UBHUY6+XOitcT!e4s$Hzaaq|A9Rz`*-i!y?yG`3Gm6A;EtlAAZk#m z-BA#gj5>ZJHa(J}kNA+9L&KCCDC+aZLp?Fn@+~a27l9+8;Z#ftpG-JvAY;XoGmX_U z^k*s`D>*2_B~hWG&N>go)27Uv&fV0fGLc_n{K#eVrf_ey z)}XUdQb|P}f#RN8cW<4WBl_7HD(kGPXbzM$hdhnJ(mG#BL#?|uSlZSac>1)Rt%{z# z=&$mYx3txF_qD(Lu)m|bvAoh@;*L;h5B4A2pK@^j>C97{dn2*bPt@#LqF ztqQSLnpnQHkk)p zK7M%T@k7ZB&ddC8@=>llxb5ivq$B&b98KAJbia%vDa^Mqj_l*SjKe8OhfpLyHe0Iu8rg7}~1 z9pRdWKxNop(NypA)|5CLd9G3m>u7p=d%JpithS;@X%7P-UuRESV@qwYuCl7e6{xN9 zRJqbKA0D9D?<679-n_@RI>MetuN`l&#cMadN0@ScG&WsX_M(Pr`77X)^NL#$SY@Ck zQ6KR_qj2hna0rdDd@|=L1I{R)QGbAr6e%akGQC<>QArxMC`6|p)hp8YEDey=0i<~g z3Gnr(tuE4ob%IQSRTiH0;Uu_ns-4-@WK&59l;--tzTCu6Z)NgHMU01q!Ae_OeRame z>)gGC#t+=cL4)1}Gm$~Uc3SD;2XbSA=`*KL$_Gv~bI+b#+qNaK#S`6n$Gq@KizCwv-%^H}OO zZ!rTh^#tkwGCBl8%^^0o(O2B$w@H=92VmratonuRK0CTZ{L2~*YUqstOM63ccXMfj z-@=;W`kKPlkfSwNBL1G@q0Lv^;j@cnCtwfRy6T;s4duan&2;vWvSs#H|BjB1@kJpTxjT=N!|Jd7n%&o!J#C>xqFKucrEM9u9AR8BG} zR+K7r<`HZ^CtZ+qk*u%)< zsa6QDGPl&2yX%TOYi*3K23K>?9;l?el)=5)A3V69MRk%^-r~Zn{4AD)WakxT<(Sjb zGw7gmmzG(qb~Cl8Vn>PFZ7M7@6&BR6HpyYKvShNlxSDm!{^D?@8DW%f7D*})^+k13 zC>1i**97qw7h3r(>bwZ4!6&L18TgY1pISl?r&@|A>u5-M)#HS)=%$i4QK1$J%$`DQm&$I^wzL}d zuySOu3d}DHsO9Cn53ST{nW}7U(cmxPrp5#Q>?`k+Q5q=zr$9B7Nc>$S6!lX*$=ozN zZq}^nn>Vk=tna}U-MyO?^ofbhH03NAz z@>D`^pr|8M)E*!+kke3+*;JX`RGG(;qK1n6R&QY&Nu)qQN1e5`&ej@sxojC(>9&6*ml}Aj6LzB7E}25w{s8vFX4GlYEt$mD3V+sLa}++PFvMm!3d#9gFuXZFQa^YNoOTwKON>&JMGOk2@bY!>Fh;OYIuL z-X=LPsUVN^{_zv*l^-k3W05`0Wbv2Zk?k$c^x85w`t7m`jJE=Kv{y?gi7X~K)pb?2 zL%TN){`$XT`7rmG(|ih+Ls-qBa`Lb@G$9`FhQ-a9J7eR8n zG^s&mY^7eY8Zq=-ph49W7TRlh)RbxtzeuB&$5^Q}&cu>wxS$4~dZMvx_W`XQBC0$U zJK7b}5lVfU{n?9GFH&>o+_7UvlD2QV za`h@Lb0^Q7r66_Y!ui|x>CJz1IkiWAS!8mHB^&fgipf})yk=zPn1*hsXPRUl8EJ{ih+>I zzEZMehU!&aIR#MhNP`etnp0bz8!G3>4l1ljUpq4Ex z*a!$`fE}w!GG$W~X{FHcQjTjJ88gWe&jE_XbpKbgpAC`0T! zL?y(-YCPG_!Epl=qO{y^f=u{CMU#`0H>_VrNXuo)1$~dBfvCYJOe)hqNF-Fbro1x| zJ#fOFfu%9g5dM)Pui{y-G|1$&cnca!j-7!Lj_?VY9e(C|2$@=1JGmx0>dU$V_VyY}m*3J_nb!`SHKsOyQG3YI9(J`icq+^5mHltfxDCgq;a@vD%J)Jc^j? zpUIIF{rQU*PMtZEn)dinTKeORoMOA%K}?l%|Ixj(olTYGPMf_Yt^N}6GoyTpsCpw| zAP$T+@m#>yP5Gk2r$$sF{V1XuOT)_ZFqTj3`xp4MkPc-F6wN0swF)E}QSeKl5&sq} z^Xn>d!j-NcSBY7tam(HF)cl_kBZeuvN89!Z(s0~7KGJOkybCi zWJz0~glJJ2q&hA=XNSjA~Q)NzbRc@!Ru(hV3wc6b3vo;58ZDD6_){VnQ_wPS)aCgey zJ^S|V*}DfL&Vp|CYTLJe@8J_ilK1YqaP>-7K|x`$-D)qU6(rbL@2c=PJ)TS=dkevl z`N6~c*REf^bmbDOIc{v>HzKpwN730YtiS zGsIh#t^gu0z8Cp*>_`BLD5q2Jc$S4jYw*b$Tm7EwQbmIzHv*eFb~p`n!{Q{rr1->R zq|+los_?Q46U#TGDkPgT;6u`iJajN=C8~SmKS7kds7l@Mpqf8r$|Pwi8T>BVNGL?o zUR!o1@ClPzSD9!&HKJ^gsIf#0wTV{^POtgYWU>wZTVzTSH43Mm7*eT0h$)~T5;uvT|ffI~zJ|(xQ_(VTJ6tmv$qXVj--DhbJ*jj6; z4OxyKN!q<MN!{s)6K~bV;cIP(7_sG?MSQ3FnG_tkk>6~cs9X;;=T^om|>-}5z} zl>XsUd9#8jYRS-W>LrSxNLUA)cp!>T-dUC|(wz&U!lY78CST5HSzcSvWzM`a_-kLG zW0Tg+c=DeECBG#7oUi$$p027w_@z8Wwv)W*ka%wUMIni;@>J>`Z2#<_=ofua=+ba1A64@Cj95OQ)F7NIwN$>v zXB5%MYD8_yHHrqGO*IHIZml31!zcQQeP`21@j|tIH}o@9p3epX*$=PAzx$`7XpbMv z&AQ3Y!>8K6F3Kmie%KIEKWa&+sFu*7!{QT$41WLpk+hDea48vgbT6%GJG6>UOh~}P zj2k&}7)QxIA;`nxSzoX9GvcOlQd;=phT^rV#(T}DK5}y<856l9!|Rs&STeO>YG}2t zg0bdPGsY3+>f~^QaUCOGrU_(PyAb^B6b#%BYDojPT9Fv*V?%DbIMD)pf_8dEL_`#hEjdZa1 zN;>dT{H`v)e5U|X5@G>+ThP%SEa_0eV0%zPJdG}VYQw5*D{LT%Qdd$OMhm0Sa7mX+ z4NrgxSfapsL=lyZgS6%S1AMZ(l&nbAeHYBc{}fSGjUVz5?-BDoEN<}N zfv6~kUa9C#_R&q=y^|b11yI^Z5)$I#;zVibdRnk>4tq5(Tgozu!F&s}mP(;tvZHas z38{3;D0xv8(d*cU`4m|X!mSx1OG08N29t)}=L{SgL&Mo$U#|I7=bb$vH)05HDrJOU zDl{reML)YkkQ{*-j}(CW8%m$lQ&lhNu0wBaZJ70%0)~>%!j7P=El}JNEV1O=Si5TG z;>B~8uUZbD(`U|HzkcJ&HEZ`CIB@pD1z9a}<+8JCk|G)3tB?X)<8+S=%=GcInt=8=%j)ZJ|R}T zmynY=LnCVNX-qVtQ5sQHX)nT2q=`!-su$qY;b$sDp)};qjNw!1aCrf}I=(XV_6^HNCA>EpItL9P zgEVl^JDR|uL*j?Yi0s8g#H}N7HNh8^IGy8Qn(apnqiKqY5}ja@7u|;gmW+X$PGj2a zSrq8mQ+M~Corez}V3YExQwZ}VBZ?qvL(dUq7L~@iRn1NW^-@=&`pJ&CGTzWn5d9*V zoYZpqML3@{c;GX3KxSXPn<0pXB76dBET39ed+J0y`@_z@a7k|%ufW#oHQ|54XNSMA z!*6NBxciG*0>!?v-1Vzw&6qJ^!J>Kd7tNnEdCH1aYglBr|L_s^UbuSm=K0H)E?>WH zF0!(|uEbqh>?*T5TqcW^?Lp8|tD_{}YBf_gvK8l=&Fs=}<@(j+o!fRLZzM9Jo0O0o z{iKT8t@+dmG58D#kZi;io>b- zWO+g?pQs`O&1_)A@ClrJU*NOZi!3AHvOp|GLUBGbWBH84SswVLlyBLYv^sv!S4T;k zC$J58Px#agNEl_BMo1Y&mUXD4kURyM7-$ANGGXQ81`irGY$%O=AWB9Q72TV%FL~Ea zb|GW;C!(j6eS652CMR#BjdAj%aXMB~h?2!pYOKVWkZJ_t9RD7BHA~Y7)lcPHzQw18 zQy&dEHEg}+jUzrM%BhEjlaoEQa=tI@Mim+GiJV3e)s`J%1!g6xghei;pRWFJNe>mw zAk8NcK3i+@(bCQ!0Tk7x;UDZ{EC{ zntDGy{mjLSC(fMBGZ)%QT&0yYj#5uqMWw^-E_Rj_mpD1kJ|Vz~anH`nNlklj`t*^T zS5LKsTphuZHiVJ%?CWfpGV4*n)Uw%G>tdjyiiZfCZ&4bHC37TE!)frT+*GTMv*1&w z0^n0$9O08K1*CQ+o1TR$b^cS3W56DR(wlHdFRl{LFXdB3@!7SdN<3W|!LqDinIu9P z665nDsx1{Id9$aE9yyd$WE?%86o~}6cxB>M=5x&G_eYQ6w#Wh`LdOxJ^pT0srNSk= z&2i(#(1VIEDybESjH)PW@Kwvx>Ur~Kvo*c6*{kG|e!0AZT2>G9@XwIIpUuaH$I(DC zMJeaB36ZcJCiTtim}nJ!i%%#8HpQX1ohT;*L}8MlDV2Fx#50!9?vNOIZQOfm<;n=3 zwNCnxp47Y0&u+@=F?{Oys3TB>U)mlh?rw0|&G#0}o6NrzJ=?SA&SDc1#PjT>D_3sc z2F|qX?2O#pi&rTXU3{4SxX@mL9k(h!?61>uVK_>SQEP;vtf#b=BZgJD)m9 z^%kFMV;0d*jiMZtpQ-J>h7&UNXpHg+oQxL9gGKaH=#99b*>zm1S+#oEq{-uWmeMC=0@$dLL+8$! zL9l}o%9X?7MvS1sG-&MD(e&MI*_O0(53Qm5G4$x?qQwhm&6+`0ABLe(aVSfbzrbgt z8PDjwXTt|}#>2Oio7RmVZKRkqnQ}hHDb@PP(SQ&vpEg!7KonLPN;zfHUtg;EL`e1Z z9F=TBrXEUEaYt2UQ}~o%ij+PEJ>q}1kY)0kx`I~eSqj?PgGDqor9HSjefk($7r3|X zx((~t`R?q6ix;n5yYuMb{q$6pv9s$JjwRKi>-X;7dH5hJH!CCSF^fQ5Wlo+l6~{cpsp)qr*p4M?6|WlP$EP7%y^{OeZE{Yoz|CeWKjvk#`0+( z+C+pRLA3O^YEZHJsG=5F=9qOoz!G;tD`!+cr2zn#4HSjO*L+H?=Pf?^0mR?*?__XCeiW~5!@QI%}Bp&@#ONryY&MmrT&ziZ3=GVyz7^4);hpuz&X=R@6$?U369RHe&b?RFpk2*jaSv?%muV zj2`=W^X3ra6HiGfB$XqHX<4suj3%irST+R2z4W;(|tX6*Bw6C@mNZ$=c}mMEUHM;D`?7H~8%EnmYrQ z&VaSUZ*3=CA1G=GyKi4RF=xh@abt!qTsUXn{ykLQFW+E$9EwEuatd;@^VrBCkJ>tW z-liNrc>2uA>>T<)@@;mUPqWowyPx(Tc~3HH31>_mnR@GN2kx=ThOtznlWZ8yCsHYD zuVXp^wY$#UTVK}OP~H-riz@Tvb0PDb2- zXr}?ee!K38RUM?#;jG})DysQZR$fw_xSB?ipGlNYo)VZrl%btWIY#3kNhL)+&xmrq z!6$0Ww;;kNO(S@xAYJRpt*ywZ_vD5l(<4L4DgC9n__y$xvLi8m;8$hc`jB|Cl<~3& z?bqX!evTV6iq%HR$vgMrOYYxG!JVZ~Y{^VEo`tS6XHFh5T=gue(1xxLsWg$58iJ=f znALVH8o{iioO9>Q96f57w&sE;Sdvgu#nULU5nHbxa%2@Vy?(N~S?y#(4OI3QRlX=k zBOz}v89RYbAbj3j#n60eE3c0RpZyJrPti|NOEs|jcmn0x6VQATKr!G`5EU1clD$Z% zB$sOP+pF`t{e?Y2_{{I{o1nL+p`^R1w5BX?_qNr$k`s>~-eUClL@M*3ITm>u;*ie0JBmHJmzZ5}1_B#414~78zr$x3I$~gJ|5~_b~L+r%j$Sd&XpL&pBcUJ^*GN@FcK`-cU()a**Pw z4CFMrsZJzZ;xews*A>SjZXo3YV^0!T1yL+}ESAO`F#*e$tu+h~jm1*UC1N6FBv~V6 z-!+X|JfnQRj;CVzBv9%N6O~}f&1w{McniAxW{xuaroOPPubzZYaer&2!~AgTh9wK; zO-Lo!H!nYUc>CV{n>TJ=Idk?j|EjCkE?>4{(VV%{rcEC|b^7@6 z6Gn{~K4{&_1?)iE+gOEVSAMUgVKtu`PR*zO2K6_{-}^-I$=||Bx&?4hC7#krQ5EvT zr|u|_zU$~pB^?<>?XR}-(oL%Q1WN{dLKl3pKAJ5e36Wm&sl*dFHJ`vKe8#{DrSb() z9HpAd;)TjMA`O1HdPMnrOw za%6lQ>$}bW8GLYBLbYtd=aa^A4X5yl zSyVxj)=zERqb43jmF#H54Is-T@jiUw0opL}s3^xMsAh7ZIHKJlV)>%(kR3c0X(N_0*M17J@p0tZ9m5Bw!C$^EIO3Qeq-3JL1{bDE6J-7Kd5-3P_69I!Q?p zn`j}WlTb`N>?uBhL`Bo|JX@(Rv<2AHe{~cKUEw4x{=NBPPDa)Z{hOgBMZG%7}_WDQ8GD5y*?J zuPjKtb$ZOuuZ@iV+R(VK4|?yNci;W$J753zEa;GZ7*y~wa^w*9Sl+sA6DiFUL3Gci z%^NV>!-pjh?BJ#vbtRp>M^v%Pa^pEgESouiG3j^7!0Rr0c%xG^s-oJ^>!Wc}PxMLN zP2N!NOfn>H%*bJr$5Zqi$8pMpvHHkl{OFPI4Gn9&%8A%TtuOp{%>1 z{Aqi@ly!Icq8ZB;&7LuB;{18DBw(C7XYQ=obEdOT-kdpeXHK6vdD_eg3l~j0xPNO- z>eUZA{M|uUZ>{S|Lpf%>n+)xn#RVcDVMYb)U&Tzi1V6H zO}>F>K~yMJsZm9yXsI&pSpNz&;j@n9pitVU4Kfs;ku0cqq5c9rIs&4f3<9O@fVH=_ z_?d`vX%_-m@97Fx=BMAvPQAH(^Xm1hmMmR3d+v-W3uaGe%$YfP*^+tO`RDM#WS1ka zzoWLV+1n|Vg|goI((W+crn9?2G_$X+tdEoe;z_PajqWfPltSjymMV_G$-rX~y%m2| zb}!M1Ca@Glm3>z|qJpR_E6_^<{ba@*lK;osRDnFlb_!LPb@Bi*aq7Ai?yK?*RCb5} zN0CWcUw{-u1yo^GDYNF2BTOQmZ|NuE+2mv4;TwEHrtn!s5kJS^6BFO)b3D3rcFgbr zL*w3w8}yFupV#i9m}54ar!qbwfjuI&?I5wabB|gG&(7g9W=y37FM+w#=*2_O5~@*Q zdk3ki#~>QvlfYb+^)-qHgvR7c5sf_wjble~FR)PvW-OWd8jL%9zH0Y@C_<{EB#zNh z`K(za)ZkN*smn}c@*{jIX{Epi-rnrIr)hyv$xE(<%K z!WDiXrJs=dNt^FQSAAE5*TqKiIS+EuZs$F|XU=()_2@>4HM_O0lBb@uh1!CADJuHt z76_LS3U#qgxv>;JMHulz@GVInDL6R;oE)DtS3Ya4)_iLFj!y!gq;28ca6#M2KoSI@ zfe=VSl9+`)J+D}Nxb2XxEhoi&I-t~27m@?~@}-@A-oeE;0z^449eotjHH~m618=i1 zGOEp-(R^x=1WwJTw(ocmhJ?0w3mHu{cnEB5oyTC(P|0czIfg69&u2@z$&*LdQ*DL0 zv)*&{?7<-e|8&UUuO`I3OJA`hDsc!ULXrAA;88_dymTQq0Nt~9mms=l7wOHZ(T}1PbH!Fa#8@*&g!$qa)#!UDrZ-%rK`5EGZcd; zi{k=?*muI2Ct=s~CQA0MXXJd8X(k?_ZLg)q(N$aaL1*yW&)PofX=o3We$-X}QCIDY z7T@z0?~{h=7tP-OFs%ZmJ!09Ny@(-jl4rp8Q1Ym;)MPRTNchzH$s9_t@9?Q4G$uhR zjU^JHGcfmR%UnB6P#ttT)2c_0mgM~DedUbw?{)Z`-9dLxsI)s!+8re4UrN$|*Gh;a zy9BT?iw@7(r^Up=RXv)kO(>@pPxv$tg-L@?jVMpGvEl$kc}W#X{=CM@d|u8`527R$ zzvqNKxd5-XJL6_jClBrzHu&EUPk46(nR9klq%Bh{xHNJA9?i)tgkQCC1^4q(;r#C8 zT{~!W96xa!OFQv1<+4bYNrD|rKTK*kU-OBaiX$p6Wh`KCVHS&`ek%58Jo}~Z0E)o* zVTWIvctvJ4imHd9sM;OXaZ~gtmK`|NU=ndtmDrX&Yc6}x5s0#E zQ;TO;Q2b8MFPNRolBO ziu$V^JvH{14OQQ0_w!YH+Emdk?4g#fKFKgh7XkRi!pTDoB;u+0#1Uo4HL`n&a#@M0 zqB&ABJi;eoctH4MPSZ85*KNSV3YVZ=mJ%>@f| z8%R2-q7tBXu<$TMuQiUbgZoWvUAH)Xz`q+sIi6ai+DMs{yOeTN5}t0QF=I!vE#8h@ zJ5mm$5Ycn{0916sgt63yBuNz~TiV9>1`)ge1uGxnQv(KXv3wdQjq|Z^GUoz=%t#(U zE4GTcv~mBaGoU5ZAQN0Qu`gR{p5cf_OG{BWBV!&0km4r$_Bt0!fjJ6;E5FarVKU8Uz$Jkki=Sfma)%5F1 z$p-LR$>b|O+t@Ot+R|22C<9Ba_|&?}iY?8jR#BML{^uXz6Eb0yqaIKy>H5+vj*=}2 zhysHs#e(`8$A*>j;s^ZcNR^XRn;}a|lME7>A_5}1Lnlod&t2zs?%oNX1ob<2?PQa^ z@#97l%j-lFd>V*Swf`cVqMt+KaUBiPG>Tdn-#Tf)sh`zHPHHmsv&{8FUi(3N^+#R4 zZ*==#b=16UtI~s$3_X9*7Ii_R)%hr*S~)4%OI1`y`r1>)vOjN$=qG&Y@w}N3o5dqV z-NB+>>f-@h*qs$F$u`|QoqhReMb@2=DXq(0_^i_2=PLZfU-rDr@}$J{ZC}~9e6A15 zEKluuFI}cjs_fsWDDJl9)aR#$P3fKGmgf`*=)9uJK>q+4fJX61vFTbu4L-5v2A_S= z{1U-_ukITpF%>G2+;u@UWZWIYlBle#tnA7+mx~oEc$eaJ z2wM_MsMtUp)m^73%$`#8lBgDV=lD3HkQOJ9zJw$ly(JXGVd14Z;j>^lgipa45FKb1 zpLzGF_=rz;_R+J0@-rno(eHRnB!Z@QRi`Cu(v!96I7G>+^zO>b#b@@FoY{A7`}+MW z=QUh9(Ug9&{`|3uWBZ%V9j`udp!wXTM`c5J_eyAW4tM`Aotv8s9L6B_$PP zc2!;mQBZ_TOj+QxL8!7z3F<__Ev{B=c01VDo8bzRcK#a6FfmGr6<|~lE^aI|>w+;U zvimCG6C;64Bq4gfwfHPeKq5lbVGDlFC%K3|`_%^*;(YS14!*B`kMjwl2-Uj$6HP^D z2#DF(D@AsW?WJ7UGs6kO6X6{8*uxK?>n+w~bKIDJ#7O94Bpy$kG?5IY`OHm69T9I4wpq?^BMQNP30Gm4Tx9;r^ zIrw^e3Br>`2Udr)t1ZQtpwjPB@uJkX1z4U4Pu|52>?3s3kc%Zr!}6pJG+W1_Oynw8 zr6;Q|_cHX!UCTR@b96`M!L0?y_GItEw^KKkiGMJ#jTJy({xv$+5j{`4_t? zF)B*B>I;BJ2uXO#)I_bL+-ZH7tQHfqv4#%Zd^!81xmH=A3E2glAZntO&{J7jvMjx) zoM9;a)*2VlpNZG#IZ$iUQ}0)SVmo!jn%?&E|1(=HGt%O%gkSlazTxDASE8 zjmdP6*IysSSe zn(7d@H$K7733NT~T3!KC=hoh`WAOAFjZgfYYi&i>+6u3>=EG+%zBx%FRlu2*pxA8M z(KFlDrR~{Vb>_(N4J%Kq|GMVX(Z-8s8!w$JKXtVF{OPhY$LlViZ_UqW$hz2)pOLI6 zY|Ob-a`tG+*%PH_kC&Z3R&(}5;_`*$=dyXA9#)j_&7aHjU$se+>ii%%VE$i+G>y3$&IBiVL) zpzB6o$MwFB+t-q}uO{zYP2TG5y4v2{T3%Fg?&PK2+tPPzsXBhJ>C~~xANO^npG_8I z7N0qib?Q)6-i6+#B9yPxy5Z+BdElh=;U6^?Vm}8yQS{zteAeY>)ny`7i5dc!mw2E~ ze79`zCvR*R^ZpNG-rfGyJ3Gd{yLZ9|`)T9eKQ!^9LzAo>p7_!J@gM9S`vZ`cZNMfPI9ZWoEGFKwkUjR8}XS6+6zvEPl{4>y0!jB z*|b;N*k5q9xv;-3udVn}#-SfhY+jYIckAWd-=}T={>N1-ijN-5J-ol*rz5R}*;iX? zuXZ%w>PuYhP4xG5bal0|Xhly?M^|TSPcqTl)7hKsy4Krsy|?>%cl$td4MMf>$iCW> zKek^yQ?!3q*|9^lm(Da5WLD)~q=1a{yNpjjCG06adC&r!=y&92#3%C8$L^7w#Q8gF zvv9X>G#2h&JoSs=56&C)?5fvZSoiw#>tBCv{cF#z{_|6-UVL)H$Y<7%eCFFx&ux0^ zr7iD{-15%Lo8EqD?OQJ{eead}3x|6dV=6u0UX|8doz^H%zl_*EgSnU5@p^pVQhah4 zH31|)<9tSBQV7VxSrMNSp@>cR$KiEigar=&b4EX3v>U3Ez2( z7Lg23fpJEQ7A#mWe!@5=ypy~HmKJ`;$A9ACXGc8w*{2`Pm^p)4v~hiBgFkIum;2+cv|XDoY~NgPX#dHL z>(lq_sLs4J&{B6b(ReM{*4Nq6mu%}vwzF~y6_Q!OyrZ+N)0bN7=xk>(8TviliGgIt zt%2?volPx;S(o>2D?Yprt2O_?uD-IuD|Ho31?i{ue%I4fDBTU5G(2}x?iim91(KW= z!^cp>douZ%hgI2KZGn3D#QtorCS&>1wZ@{=lRo=o_^^3zja>Tr$W?ERTJ!d(Rd0@5 z@#>3fUVC}d2XC)`n?0|td-t^se|c-eU*25v_G?St9J%%Ga?6qhrqVGa8@u+I@ znvBU-_@wYD1tJ)@m)XgB5XL8N5lUZ_M)EW8X^{kG#H0?82o;|M{AG@^mHcGFa&3Y2 z4Nh|J3vuDI4(;9i`DgFVolgX55ux}g(^wqpy-}}?8u9Ei_&$&M5;MTb0$w(qjiIGy zpC0qo=;<@3vnVesC37$sGrt%!0E{Izfd_=*B9y-93S`oeucU){cZg;m@=_lkvc1t+ zex2YFV$-+k(7)1F9O=YVOgkV-M_!WlK(9uipbZt5*F70h9tu45e zclvPd@dIam*m!EwhU#;tG7j!eKe)TArhK5YwYRe^+0oL~(bC<~(rK;LT31_F=U{VI zSq-qYtGy-J(bm)5(bwB~qpz!@yf|(5_S!Q)wWOb`ICW&8EI;S)p8V5?dg}0_^T=8u zst`o_K$Bf|;FAt~b~WUY(PXR=r6V$8mj9WYD%kSn)-aK!{ra5yq&z}4J{Dqqr%wIoiO2wrUT~*nZ zy^l0@=85oYSRb-@1MxOX&gCZ})Q_=hIZPNlxNGB=pZ$edn6u~3oIZ2LS7XP#`Su$=NW{F1M_Hqj zVh&F-jr|D%N;Z7?+G``JOE6~axVPVZhw1Pie()|!$wQ##C}9amVA`NkO3(!eI|ruO~`@Gsb7hm9|o|*?xmh;FMh&_=M7d=0f(|Y#S#t^bx9^ zoh`{kYgY$CwWG5;kxV9TBoi$KS$T){*IhVWd-iz6Plt+*?JGKSq_HrqtIG1O;S-k) zT|l@D4=%3UU4G^ew=_y|l&Z2pw5#ee!7)Z@NdxGtFW$Uv`J!3V*MGfy<%0Q(XG~c* zb^Nj!6PHgLzhL|qv%dIv`HV@6ri@>=Xx_@Xvo|bTx^?}!Evr^;SU4;9)S<4re1h{P z`Y;=6Ed^RR&o;8pzE_O#gCRP8;@H<-A4$YX3C6(r#KRO> zdT97lHUP>3UZX~he4o`pKK$_Gk3L`$aiqbYBF8>*-o+Aw?{R^7Z$BU!*X9Av_}L=n zV{o$PW>-ZJq2g1BUTMFZPZF6tJ2bw9OnLW^$q1>BKv`7(W}lk~InM6edVcS= zoTCR$ZP}Q8aL>h^TQ48n(^XYG(B9D7DVb>kGjMp{4hJ}&@GL$P9SGR2w(C8K-ulXt zlgFw~A1^=gW7+WoS^Kx;oYe#4^qD`(AK zI%C@W=@XaCo3?V{?6nK#ES)muz;`Q?^#$!147Ef@Gc1ztQ%Eec)N z7G|oGCOFv&PPaN3oWe^oFCOaDl&R<@=xJP`x6dO@eWlMRKqNM@s&y-M0i>7|bP zf})(Wb7qX6I(7U!T=RLeSVn?Xz?is!ig!vMqDb9SELdqZLYeaR@G~QxdiB*;-hA^- zR`OyGlG7sgE;;Xl3Zd!)l=p5WGs#LKF3P*S1W~s1gUk#azQWAlAVO1}o1Nu1I?B`# zp)~c)o@^_%7v%#36q|+ViN(ZQ_ZKSB4w616xbJon9ym|4Q)icK(+qRY;ikVFo(VIhiPQnrU;% zOl4JxQu++QVHkt4RD3kw(D`gGvGRakB=B^DRerz_HUAxrZ)2(WTxI7wsxuo)sCk}Q zU!HSl_vW!*d^ig$W%=SoOBOP*oyF-!j2OXeccds(m~sbH7hfdRGv)-4G~QwhrRwui z4f_-Dgrz#+r6`T>4E^x&=p?BKfL;dv34B3g98q50>8|`g;SNmQ;l)GWe0^f$x+AMsoZPVH()O*Vwr#1(&FF4zz#GDt?Ct9e;xW$Wefvq1oW^G- z_Gbq^zG?jKWNUwSb6aJ>#oa#?9@v|=Z};UrKV%-=Qf_{(k z06n$Uvg}BtuVI*ydGQ&+_|by*cOIKObB*Z~OAuhc|!CZ$kEIb9q`%W1gLy z1<1%hNeM=uz6Ww-&M=G68nxFXFyWJqU6LpMA4X7JIz#vb)ZgM$5bCB&in2$1 z0;*U=zuzD-&uD3~@&?Yv@!eY%&EB_S$&LlHkF8&M>idm(r;fJOR&=yAb#=A(uqM6HsavX;<@={?^KzOBn}t75{kf(hpm*59~5NbIw}UAHvh& zWGhkGld?Zab5be`ppv!Xi3JE%@}c#iL(d zI{u|46JA_9Yt(lO-(LZubKY1r_pPs|zP4u48ym*I{q4B7){T2-{^+-kZ(Gyv@{`aO ziGM@`e8!fSSu-Frc@S`Uz=l5f1sd4{pYfgQlZDu6^#lT+7?!7jlevdjq3{_fmAh15 zaEjF^D2UO@fN9A&P6FqaHC>U}T#-{>nz`+}Rg)%;p~C8tWlKm1nL1+{lZPn?!t@Uc zFPI4RH8&}f2u@%FDr}*J4)ZtRQ;1rWJ~e_V@Yn`E>1Zb%EKkOX^cFiIlT*d{3}Tu6 z@uv8vL@01doJR5a)>4mu+V~L!3QiYFHl~SYcZF_2PVsrIr4&AIb(Y=ft;7&LyKl?( z6$^h{w~BG1b3bgnwD*V7^fN6D)opE!Xy=~Z&Rz?@|0_P@@Ah=MMQWyLBAG~ZyMFKN zYU=5%ugOc#KD_VZ4_opM?mM?*bKa>#h35}*g(Tf69?O=Cm-Einn$Jk>W2~03B!(j` zyPixWYrV~k;Ain?=APdD#hb%Nzdn5a*f$qVczOEAkIo-Ge8uEfR!$qWcJ>?JEO>MI z)aRB=d}i_Z=a)?wxq9-OTjzeTdcvrwA3T3->(`8$k$Iv4g5IVA!4D^DmTltG35B2d zbc)RE9J13t66X^W^geuoDDo3`$>b;gQNgLkf>R6u5MM>=5`ATEDbr7y$^5U(ZYa-T z))A|1E?P8?aU~|jPnbC27pvdNO2Qd2mcM)4yN$5x11d$f-gLLiPWg&oKTDf!uK+N{7atz$b`uBzzK+htFS<6*tK&KeBu8+7ePse~9mcvD6^i(j2o;U$BRs=Pb-Fu@WCEd>tlb^3+Sbw0 z)Y;M4*Fmw*^T#)D$UU$(>);-Y&(nK0F{WUbYQfJugy)rJ^W`HljZYJ;md)ZjjZX^d zy8ML9zUBh*oClioFCW`7=G~{q{bl&lDQ_;F_Tr)mPcI$!?6L`eo3;(jdR~yGXCvTJJ$|0k>;13^yzp=Q6pb)y0k>~gJ|F| zh)^NQOGG*?aLY79iBS1dlwLt>N`B%ZNq#z?sQA12ED3zF)(<(=hNuN&t8$C;(mwt0 z-RV;&ELzCu{2UfPqJrW}e}3^%#^RZh>{H|{(}^`cY+ffR_vRl#4pNjo;?qD)3CsYb zWM+6NM8iwHD*`!BfRoO@<})rxBl+1fI4LRc$$h8sGRRM~e5#rJ#CH9qyW)07T+Dg5!rS?%z>x?r2+Crk*1v_u&@Fe`k;Q1XOE2qspOG z#GYCxrr&AsjGn#TUi{PUuRj^}!1U2CET8el`uVS}oc6*7#O9oLR!)0;>7-X?eevAv zFP>dCZR9sg-rTVG^$iPN`+nI+3&;F<$%J=K?p%%hv*~ z49k=WYVl8>9~VTWkCB;z6N}Vx0x>mhBD{nZUI~)3kW_reYJb|ja^gqBmrTRXd~5xJH`dO1ed*+p zE2qA;dh#oaz8pULlgAd0eP-3nk>4(UbMw--w=8{c-JEykeDSi&&o$RL31KT{eLAP9B5%j0wd23|WX{Z~RIQ#nZ|=<5GpS?!{0q;r zT*9bVUuGrl>C>kmU_bisFNjSc6wtDl)M&z~k4?}JhXGNMIfPHDi*OP_9{3cl@uxt~ z!f*0P{L=-au#~`*ikE)pUFr82pO#pHeuvLHoh+et;`^0LHZ7U|!=jmo*DU(++ZE>y zY_G2Vz1qw`Kf%>}_(X(u;Uc-phDrN6^P2`|Y_TUa9bln`)bYN@UZUbH z%0ytF0_mj}RNx{6ofLY);Kb~NOi9rcp9ZJRnewfb^|PAogcmXswZ)s(F8}=F_h56u zLh3RtWTA@BMpKIQb5??8HKfI)IW3&~`DY)&XHfCRlKV}@SZw?#h|qZFUPN*Eg*qn; zF#+uv!Y41`Q$A6UpGZzHLoHrywxSI~_!OLMc~@jIcZrU0z1hLp%75-+ThmmYv*Vjp z8y3vov2@Pi)eBE-{QBhf?@Dvho15z4v$L}~nG7M#h|+*%ydy+AyKMZO{q5~dt?iAi zZ4HTr%Coz-|Fm&K_MV+bzFBkO;Lfwh_B2-Kb*C&4$mDM^IML=VIRR36)MTYH0|m|7 z(}1rO8GA%QnIv+pxs=%=HxuRiHZ2(Y;WLY-y*1{8M`w?FZpDmOR!)Cq+2j}IefjvJ zaZj$8GGg9W!xoHvY~}P9zMl5d((x~@oB77dDX-1@V&s`UtFJa=Hstg*6_KgJ78kzT z)+{qgXei>8*}TI0AR74OFd<6Ef)Aw8QjdkIl~g}t4AfgE(_|`0II-b-g`TMJPpsYt zm5fr8sEAOX6KdmD&ZnXTsEqasocf%)lI%6h7fu}aC56J6g0y7$vMC_?#pneT7+JVr z?3k}84Lb7W7m#MiIwLb?C>o!44<5zI4E?zLl(4lArlP4jt;l4{7b#FcC-4~%waBNZ zfD&mBe1fRtr&t9}Isgflz{yai_+)#l)551ecU8jYwT`;WCk}nLc<#0(bGI#=accA0 zL*J}Exo>w(S#e8qT_RDROg0ZDgv7?*F)!6bX~d_&*^+2!wAR*GT~?5O^2mV=tIuxP zcx27jKYjmg?%5xktMfXWOUVBOLf|ydsHG4-U6rTCzZ#=FfA+&ClZjBcMgnu|tmX38 z5!nAFS+i};tWRHibm`RhmQH%}>nWpFPJL*UpwRu}h=z=;49GSG-(T1QNN#?^X& zlTHw!f|G6FGfuk@{VhHr(~VCPhO#_^eow*4P?3A_gl;@z?OLLoB&8eeHn)!E%Dwz{ zNBPek6}Q@Ok*b@EFaNM|;mYX~*3X%`ZOMXz->yBh>6CV+l(vR(Ft;kL`m-P5(EOJs$5U|l_8{}u`u#-?csum_$htc6QP>HBBpNJn}&$h(pGJHs32aWFji}N$acXG*qEwd@5u3slf$6iisF$8e zN|hHAKz^ook%DWNATWl5l^q3|bfwi>TXSE+bv_Ie#uuld6a zX!vX>N^2-$aq^$WkN#-lxG%^QojKc7{2On*{@n8;9(m*;=u{5g|iybZ~wgG>>7FP}gTnW<(qz@xeLHYi|q$!I=6VB<$fJhtNVW=DnQ8V5wB z;(eRq6P?vuM0Qh(&vLxoIG?|CR{q>s%^8>vo^$HJmgVy{kc75y&YsmPk8Jw(*w*i| zPaSC}&24Y0=u9+XNp>Wfh!-W3KJre46Ht3Py1Nrn=meB(@Tj?=wYsvQq%iB`vF)o@ z>|VQK6GMu#CU0Ck>)4KO8jI8WTT0RFOzseqsb|Wvd_zPyFc(6^FP zOoC<|z2XcIJ-lnn*wG&mw}0>bw_bn!6_z%B>4gzgaI!kLRx`vFNi=L2pnvSa;cSJF zG`?49_$@wz2o3Tx#wP~DAfFMOVDI0>FoCJ#WL4l3d5FH2<0C{NGc<0rHJ?ZhpVyI- zCO_fRW`#(8V#nSg=cKc=zqN?@6eOyh+`Vc2ylIHg4Kt_gT(NlHx>bj_Y&?HxZ*k^@ z>hgm6hKjbfy0*4@Ov<+Q#zaRmrlxfgO^ku^vZbZ2p|+x`D6b&>{HX)GcWqd`f8#n5 zGrn6iYuDOk8A#5utShbMkclv~ROHln7V@8dbj`Z*SfVxi5tTHmKlRd#Pt(F&GnLA2 zC+iPwUOoEN=f=PG^2Z~d{OESqn`Zm)yF@6`LRz&KK1_d zkALy%^W$HCb?j@ePW;PT(?@@BV(;c_9aWme4DAF|O?bKuY{@4vKJSLpgA6V|%^)$G zlV*~WXIg{d38sFq6sU}`3P$NnntG7fZRQ#a)EKiS$`w@VxrM9_MN@*|4 zXeqkfP;j}iH0SW{??3v>>!VmGbJR;~S1tYWllLF_)9*c9gy5tt{|Y}8>o&3JFUe;P z@)Kbg@KZ-BJ}yGT0V&BT5sHy8C_gzT4+3AuYW0i}+{34(#zCgEd?3^1XEcyBKu$<2 z*-d5e$-(G%@reX>C#&psk|9oNC>3@$=GGRUE6+S}cJHRG%jRsDJ!R{XdAnDy*tcQL zksmgn*t6r@(F2+1ek#qmTvt}uR$JL#SJhHe(O6m9USFN4uc|FAD9_I-&P+=`eeA^k zo%_Duuy6Ca9qYc{wQj}Ht>0#y*q^MyyezxgR?*j7j3%`Qr^#$%t~MAaaXy0xW#<4% zS-4J2=<{kkeKvQV$;i34SA%>CrOWn({G_SHx8K6z*H zm+!3^|LN+nA20vngT7;(Tp>Pez=;bG)dmVRoNpT zCXtZT9YvyQkKhdQTpc8*;B-DQN?m@!r_D69C=*`1)uygYLyFS68M|j4N$1m}_g2Re z74L)cgM4}xiYK3tpi+q(W^M~Zx)hR1r}|lYNqSQu%j0HH@@m(XwbLh!-nec_N#3Qx z%V)=Y{0^&uJoUt|=Y~@=+|@1vppQOe!$xL&TK)jb;W2#V5)=$WQIDmoeo| z$sMImaVjIz+#(a#G;C4$4Ei0q(C?5*la>#BN+|hlEiw#@Pm`a{XE~Bn&G{_7)m3u6 ztFXT(0iBA23<>Y*@A9o0YpZuG;h6>XW;_zj$b8{+S~^ z^~{W}9cU>VXesq1>LNmyI7ER@E?#g3DJnkox3Gmzlb;MT)#W;+j4S2$wUo5i=Uq?M z?f!1n2d_Lg^UIG{PMfg&i;t$g`uwU--`&4t*5+wnEq(8`wV%AZd;XLivnMWl|BXc- zzV-e5X}BlFl+|j!mE`B`uCm*m#kaePueTGLMm^cs#}#a_n(PY)wp3-D-1E(fbqi;0 zST^t59eZ_jR7eR)_V&+cuouw(jze+b$g3a(eIPpZ09ZId_O32jPiwhyVT-vS-oPX7s|Rk0?2x zR=ZNM1>SPQ2|v!|C$WWC!)PH0z@_H&xCr&JBuZY`pga4Sv%aZc7u3UavN>wX){;XnFp4nW?0BB|tivZTT320R+Nnctzw+#e zCx$U`6BW;zv+zSn2kP5L^%z|v-^*u&qDm{Lym;-Dz;xTd;{d625j(Rt!Abwu-kN*) zRMd#=efhkCUFY(12%q4M9ZUNKL^~Lr&%4r$QJ2wMpE=N&J?v#KPWx`joQ;cS;rr}Zz4*lT?=ny9 zuPaFFVQzavNpC~(%|zAhWG!%B@2t3zDC=uA5`dFxPJ9LUaoKU~boNlor3MSL=ZMWV$^!deBb%bOmuV3ejDg1`~)bw^?sjbP?NZy+H z8c>YGshtEuagM~N1mKN?6$%rdbol*+Xt=rTL4Hbjicc}AkJC3w+skgX7hP*97~uD; zyWCTk*4L16y)FNCXK4>LLe0#}Z=zIYU4G{2gH<`_nYVuC;Pyj5tUtW%o8x;n(@#6G zFX!B0p50_U!NcM{YAQCH9>3a*ty_L8QF)`Y3hQE^y|k~js0Z27YyyWzfPrt(7?I<) zs9kf)Q4*YNkz*#xP{B4eAYU~hO@xwqXI0JfljeGJOcBZO_AViPNTY#tTTx#EBOE<&IaA6- zQiGS0VrtE3eEj;NOKjV#a+@o2S1+3J$RB_A*u%ena8JdJl)H=#N=yxcM>q@)(NGDV7 z%=$F_K7>zu*-?IjDQRe>hAjB(txq3l%7o8rt=#tvEd@e8Fn7PdwXCByzpWzYR!OXi`UB;)0DI(!53?q_J44+mG zLJyRf3^az0UB5T1Xhf*-2{ROtNXbtXf9^B;5U(VU(~<;5`P|-m>_D`$i@w%E5Vhx& zsuWf~Eq9hi}#;+_&*;V_J@Za_%F(blKkma>t%cn1GfKLK0zA1gV=;g31@@z5i0OtNIZn_ z6rXpxtb;QSqPIJ%2pj<_a0Whgi?0HSiHK-QvIdNT1ddqkEWegO%wp|ZeXzd9jBBmg zH`?={w5K+$r#8K(mV(LI{nW#$FCY$Zt*!hDy3(2nN!tRa1IgCDm#OOnju^3OE#<=Q zbX9YO*AwMe+tJ0yz!aZUFG0ooH3vQekV86lPE@eZPVH+#g0aYs;XP2F-A`-C8K4bT zk;*ZKTbRv?En(XH3g1FLOiL-_p;sD;25b$K!fW>A8k|r%;00CS(}ba01#C?eyIlcq z>~}a_@&jZHQG_SH-k_G};8Y?3C&VzY(I{U>lAtIWKk}HW)2teBY|SZoNkM)FJ}r^Y zYWw7M*5vwvBcv{q*OU?Dr**u3DHX=B2ihxhS}XHPv(Jxy|BZ+K>vuL&{^=;k7`jZw zd)(>&oKF+B9#cYo20rPylTit8jU!>DD zh)US3dD`tR3!eta#pkt-k}K`p2;7a#-ujGzrmP!nc{dZtw~Su+tWN8#PVcYH=&#G_ zCn(ftL~&XSKeKGD!=O0;r~s+g*lG3=0k`_@bXNV+U1NOubUc?uh&7XJ+(d&U^-K)` z^4^yO%A6J^0HS#v82D_!NWdIrpp)r!&;<{~MRSvkO5pYQgzJ9P2T_jZQflNv>6J!` zIfGL%F%2Wr3Oyn1Iiaeh-NmQ$yT5B>N|hUsDTXaX=7Y0Nsb3ZE-ZfTBC@kU=d&x{D z%f7?Bq?riCMUsCsgij7&E{W2jI;nDIX;irwoWM!3Q_ALhQGM3Vqp&%}OD{8PslJe~ zKQ?M^V+l)~@clPldScjr4S#|V6CgcCIEdlS5iIvfF~(FbXy^=8>z-)GPD}zD zFf#p){4^1Y92@2gEWKI=02>$8jjN;=>~ zpb9bG+K`7<>%(*|Q7mMpy=(W&{ zBuukhZ3j`Ly6q>SIjSmYTY3p?l|3CTIwR2Kfn}W)&#&*_Qt!128vHL%lvLu4`M0>f~G{@PwzF0G^4*E zYoIBcx2{?WaiQ0{L)RA(>c2{*T)Q1MX39-LBywL&rSgqkCOl1kGH#W+wnzq2=n01T zR7$^d<$=|3{daX3rS+4;q1lRl1{;wyhlM@%jP%fOqli9vUA`-AMZGQgT;BDBmGmJW z`#PaE#5>~G=B`@I4L+^*lu0~$UTFM)_H=-;+Xity?OkEqWK(AJYqK)|sWa%MU9CSm z?#(ggm5pKChhdDzRBdaM1wKJkQWO^nS;|X7pzz6x72=b6K9=EcNvceOb~ugCbVif{ zpN*xLL6pgT%qs-Zmh$YH!nEqb%iF$P`Sc@y81dvIBYa6j5=d-`B_BTuWU6y7pV}WO zcUuu`kWWo2dFH7{UU^~oQ;+;%@>idHzkUULqMe~jd=j6B&!4+%;E^}A8CUDGuD9gf=>QcX;bb_ZxkZZP zG(OpR!;3LO_4W25yegg%u!K^!*IZ~&ki@4gm)_{Cypycu*+GQfBDjF&DI?O(652mxNZHmtVE+Zfs z=QFjYq?w@dl7uKKKF(*5pYW-CX%hRXH{~MK`1E$UnGvNDM5wh6n|W=BwpC;`B2-H= zn=A8|%$|gn$3$ef(?aVk3xYkF((i%NA$(eCahr^6i^(cD&u1+g3Yq!jun|u`^5V0P zzy9(wD;LeoJa?=#=R#w7Zjhhm99ed8IT3qQyd)1>-B5ly?NR8-wjFll-}>wR)^B{` z``_v+y~8$P^7Cp7{w$70=9R|mo2~h`+9APWO_WfEQW|`+2Ysqvo~^Jn`5E*ty58?2 z_lR%jstnF#6tLwZyvtj5j0+!K;Z>`AVv!%V>X^1owXhRN#Dg9{woyy@+P2@lCr`t3 z8`AVLikmZ+n2EHvE1;s{=@7BMdl6C>p)C1? zAzEgjHkW9n@(dPLsV}(LY;rYY(aiB99{VG4dvZwr^uX``^rt_-CqOb?ju{_xE@+|6e-vYJOJB2xg{;8Bz>9hd}EZ$_zhsuN!F zk${Bpm#~=99bEg^!BI`bEkfO`RxAY%8c5Ab5*&_oJy-NV9}F{;*%|W(zx{|3_Z9c zMai!RsQL{No4iG{AY_*rpMM{y|Fsuj9L{cRSAyqoLXaHUm~*8mN5T^o51$?j&7fnJ zXG<=JNq7p&2u=iP>g$Lu3Qi8=t1&!&P1cb)%vU33#iwo`N80Qs&m0hnxLV@V3Q4&{ z6_(LXCsqSM4xtIpIGBpDk`w__HSV>~ZiK z&O$WssdtBL2ceB@QCaDRtGs^XCw$@@x%{*=667adk~vDYxOu#u!&&5KicfO*fthJg zQITHcC&NjI(7_^6EbWwXlp;QhZ7j)FB?H%@iv;akS&XbI4;B2;JFhQ z+QRd-h36Uy&$BcnLyfG#$Gr}4(!mymPhil!7Aoqqg-jrEKJlt5u{{5Ar9pf`=FdrN z*Rq@K)M6qOpGUjinuqYbohU+zf+doZtu(v68{-q+WOEv^DgDlzWqGooYf**nyZ98_ zkm-mT)zLM(hlOQm75Ti6!==W{!Da<~uqBXVJO$`_6fuY*JhAcYWgJl(H)og$&mH;Z zaikjiE#GNmui#?vk!LfWJbZ#(o6z+$BG$7%V=S zzsz76i1yd#^phRxk4rzfAJ@EwbKGd-E;eU$zIsN_2l z($uorBVT-O1bHCC{%g{gAFf|M=k$*|I_e524OLTcwkDq?_RiH6oT9i1RQ&G>9X))mlhCJJ$sZnqbj@!7}&%Rr}l z?rwQHy%t%bK_>c|x$}_-b%||pMQ2Y3$ZNy?ywhF9)p8{JQ^FJFUW`Ga>(oi)YWZm+ z^4YJaAZ)L-a}x8fsnKLtLdU%iw(K$2DP@#KiBFA1Haua&1ZL+%UzdWA{amwxZ$_ZI zOEJtmoJ(le;p70`8iFD}HBBEG?3-n7_55k%f$Og}liLP!t=TY#CosdOBZ?AFW5KD; z>L{rRM1%b~giqy2vxUznwLUf635{yTr;pAXpA3UC=vj2Jt0uplQY6I$MGL-TbpE0L zc=C}ykdnlRJQZ@H_Bdmd3|an(b-u`IX6bn1nQy)F-0Ed>5AFE2H0xBdskps5#}+8J z1<}%2u#9DMn^@49wIB=0Y+<_exxgnKNW-M0qhB6z#IMAbd%>2SNY2|GRW}l532-p- z)0`>G2`RhNjYKu!8RzpxTmG%K0{9$YAcGm!OUd?H05lT;G~Y0N%^ zA|egVhi9}_;>316QSP`L;qCv$=%VTG1 zL>?gNIG-6fN`#&a(c-k`!i(KjUX>bX`Hdx+4P`m&md+mW!~>RVXEnen&q!HO70DSk zV)(q{-l#QYPdROAqoTMNIwlS$zFbSMATXc^F_3q{q z*r4vdeDYgCDZ=wsiceVrG7kp%OcaACk`wt!e%-Z}JUk;Bj(iZFZhYokYsn{p1X}pP zK?RoR0}0!3-^C~I@?DTg!~QfrolG5XT$#ygpV9~I431ymlRau;G919m!P|W|pMpg4 zQ;ImwCltxZp~El30RbxbB>txTf-SHYI##$c#U~%s!FpGHc7}twLT?C9@oB#jh>A}` z)JCa(i%%q{@kvc)G`{;sk^QOZB$hJml%nE+Q_bgq7@}%{&lXA4l5|99FO{sSS)(Vv ztvs7jY8|zOZ~l4sV-HawoJHHMC=(+@&kuj>&(A;k@^g=keE!j8^CxCsJlsSAYC~>o z6*dF*Fw>kF2(S*8GYvb0W~F$1sWSLv30-L9KGI-7k#@VSsVT$y_4v_f@X1YK^b`5X zr#L=H&R=^Gyu6I$Cw$5THT|9_GUuo+Lwve(l*4a(y|qB{Q?#%(%=}*PMcf^~M-j0+ zI0k1=9U_#OzZ}ewB2(MzBujh1!7kPL)E^aq)K0x*s}{~zcZk)oDIYA>-5qYP)c zBtdBNLH>5WBAg*bQL!Eqw)zwJEA$)crG81Vr?dZpQw1o;uWQbQiYznYe{)BL<5;p@ivy^UJ{DFqY`(XX>^ekrEn9? zCeyJzE)vz|gIsDMnYq}OZ*BqCQWkE`zIc4!(~l5m`Zw09dzM_0#~yg;nTKC_;j!71 zKh3&usIxJ@t%mAk7wbrwwt^BD+MGiaI<3VQCZk|JOtNMZHOcFX&eQ6O&+B(K1)9>= z;#0q;ekJ{?{O0g!awEkjh@#dZ^RIo?wAse5F%gLi-JsR2$JHJSN#!N859qb)-@;Uf76`0*+GLsH9; z1SZ2V?!}?|KzViI@PdxDF;SYWO-m7z_ zj7~qbzpJr`Ayih=V~uI5K%>M}^-*%tmL13DWD!!z^0Q_rGPC$XL&=5u;tRkT_~d5A z`Q!&2@_X{LyMDJECk__Rd<35z;rX|IMmfz(GUYBlanIZHZzYOuunJOrCVa|yN4Yzn z`IxB|iAu4g8?Bdod&DOLh(2V1VE=1xy&0dK)nPhwicE(b`RNw2;gM4AT%`J8Pw4QL zziWO6=ZTsZ&xcQDuX80<3);udFg^pH>{rvt>5Ct>+8^gr9BRw&six};I)cA6yhTR` zYbSksD>Bgs`Ha4?1v}YIP*l7x)rSe%#|ScUX2wChMIw}?qFMH|%EmXlOn&|*pO&}G zN_kW(HC=BjG9ySmdxV*ums?6b3R;%IBI5|prUJ^Cm^kGaqEBt*nK(PG)j1oM&seo^ zVk3*_H59TUB&A20C5l+4#-mL$u^FRhDGb$Ie!i*fB0fjy_{y7O-D(qs&{Jz&BUK9QogI`Q<2Zs2Bmd>K#IL_J z`;(ha3lQ?zIG>@$V^YRlu;7%EC+kA=1*LNruZ~1FDVLO%E2=}4N>^aXd~i8P!#1}jN6q( zB#O{IqOY#-ET#FV>Vv#RQ-fThCW~dl0G@&>ED*^eJ1F-?rs|hpY%aTG0uzVGh-G0N z2C5+TqN(_j^Yys%CFi+KGL|LIV4OOirufpeCK7Dm(_->!S6gh%-NdYCgGlV?F6-_~dJsGZ#amrw=yGAM+^dLd z4c?$gu(&{LGcdTw-{Mn$rnX!^eq+6^-m3<-jwmMZL(PBHmFlm}r5 zpY~g&o-+D&ev3O<=t)h#B-?n?rRpvI*!WxYc~`{JvA5hZzhUVpge*WblGtXwbJ^}6 zNq(Z@dHiXpDn(1MsX8enZhT%aKC3U2>C{U@gbo(`v3bo{p_rhuLXn>?LgRc=H8hh& znNjzY2W%_PYAeZTExrt&HV4Gb0K!u&6hf^M7FDb(JVV72#`IxTEy4Jd&C8>x4p!?$ zmd|SjRF*O+y9kF!(dOdwmYq^!8T%{_-e4<_pzEXf5)9yju;mxF-`q6|KQ&QD1w3ze z@|MXQ-sR>J`6MzApAw#IQS>$h-w`?QP)fq*_14_0OyPv4A{6;a z8fp=m9X@Hk_+1pD$6J9<8sEjo+0r;6XI9gm=!^7>IG@hypa>1b($QORID}8$l4#|v z=y(SbK3#;`1EQ}JpQhq1*`7RAi-H0tzq5%@kLJN=7a7YWFX10C8M*p0X(Sr8kMl`r zBF?AH?ySb0zl%>xAW^1$8A>04nbBNib3n|;cRmT=Q~eJ)i7aIq2xf^|4c@xkL^V|_ zvsfpB>C6ZnO?E<>|hLw-_n2XRUqiI@_LLs7vDj~*`xamwO27^}K*+%Lh&-)_G^ zickFI6rY4WO+IiuN5Ur~n;?zZ37nDh-cb_x^bkFMQ1(Crd7j3nemWc0bkCiim!@A^ za)~qRV#2M#Rrd??EUCa}0&Y!)$Y}Jdf)fk)=u*XDIGbL^xA*6>4U;2+6Fxb=MYmEz zJP6Ogr+x=cLPzbsd_rL0Q=~<_xr7XS22rVZ+3$~`QLSUx&&62Qq9IDh1qjcG%g<|E z7>KrzzG8d=r)54^9-|rjwC);o>ji5StO;d^ng}Ja#9btfI|TWuKxmv#qqG{jtUNeX z@7y86X#zqtHJ%ZxhM7@nf{|68YplXQ<$n>sW;1yCAgj$=N6TPUh>qIJUGXf zwn7;tA0=$4>3n+IU2cDELv?{F^JxVUkxNCNNO26x6oXODFlPJ2xl{i;**`?lecbS zheyqRUNy824Hb_FrP)A|73HJYh&7AS*Q7D?i^++oc=%+?Iv|7@$s}=4--?oXV@^{6 zrJj*>O!}bq6m)^;po;f3?oEDL4oRjVidy$(A*K==mk-XA`AQaaVt}5+X2Sa<16am> z7pt1uqB)ixfTxCvR}+wqB4s5A)l!ZxondzjQBKlCe94A{D#1@=8iy%R@8y$UmERv` zTyo+QF(Ce%e43X;mPmucscd8up7`|NvSN>=B>Zqi@%@L`O26 z(5zE4O#^#F;N-0spOFzOlf$=Ww8s$*d}4@-PedquB0pWlrzLAIL8cmfrrmVzfE)_%Il3te-QeiBCyL`Y_2hycD_+%YOVLemZV}ZZTpFZ2s0)^S9n= z3VtFvL%Z2wL-2&;aqYbnD&86M0d;<^ zjgFc=r3=;{qOaCh=y+|hSmZtFQm?n6s!W_lSMKQYBlAU=-}bl2&vKJkd9Q1LHS8A-*fh~A_+k);Dpj{pK!`fNlpXO3+Ey{jn6WyWdc@) zsG9hMKsso1rKY3kwTEqZ%hYS9jTw=lA#BVF?XN|@2R?7h1cgBImzs00V6Hc0u)W&E zm{E#PEIqTM@HGeuna2^)l>#Q*Rtajpixxb!=$4rkI*5045Jo7k39aU;r2smKbIBTu3T#l#xT1O-&&XR;PP(^7n% z@2t8+aH$tQZMrna*z`RKQZIEvoqYI=gr`)zA?mA*AV0;Ysd7S{DLz|Ej8Ca}4t7K% zy3o{+27Zx>cljv+&K`+P1ZkX46g`8URCqRD-ZG}r5sZ?V;*&mb$`BR0+HX_REVGV# zZE5o+Imw(KsQ-uaDdY1t60Z{%$r6?b%fsXVsh$m&#pm_rJR~YgAG`FHVd;FPWS#D* zZZ~~8Z1ohd<-ViegYYy&?}gLdD~lmSod}Y9`2>4*LQXiz5YpHWpYn;AS|V1-bn+bR zC_bQz3}@hMoKwO%JPsTqK4YE8X)=Ln7hs_PPI9wTB-<0mNjRSU`mDWs7oRab@uvw+O&M7mhrlQhZ9q%K%1}vR_&rKE*2X6Dew?PN+a`xqQZ_ zWTq(9yhxm*AU}gpHAl%*yxju9`S-rMfArV?y{`uO37<&LpSw%%bd?bHgilxQOp(jF z(un-bMwUV>jX|ZG?KX0xrrW9;8t%1T!X#frgHLNVc%%SqB$RYKXW(wq_>@}c>;AG+ zcRU=xOZpt9GeAO|Pko9#Qto;eM6pqIwFZpS7%o^HPRNl&d`iUwrwdZ*HC@7g$9Xq1l_?>v+$BGaRVt_u zWwN#NT!hvT4oxBItBqI~(+22`PdL=Dk!f=a+FwSZc2-a}M_!esI*TqdL7h*VQ4*>6 z3=9nVK#Ovhe&-k(!V^TQc=1ziW35J;a6#^=3!nk@CA4L|o-+NUAfTPr^0 zydyb(PL_fwIU#TekjPJ$oSAq@;A(JsPBH^bfzS9o9dff^QeQ*|`x8D9;8KgaYobDT zHjYH-b8;Fl<9rICz<^E?w$P%Jr)1_`ev`h7gXwUFZ~@xG7jdb9PYz&@sG}3ElMx5? z+0w#EbhW(HwS_COgH5wrLqiSzbpDLX1O-uZ9iZjrF^Ikqd6p0!{t2!8>zgPylHH7Cs2kQShP=|B$ z3z?r3tb@;P=ab|m^td@k((f6tN{U@1KN(kY`FWdVIy)jk#2p!Or?{cqXodhll#U^a z{EXrZ@pudN7f_O|8OSZa5F&`Ztlb`%(Ac}Jo zDIi8XQ!bJawTw_p3XP(mz!^24fb1fa6bLHmS#U{tS#%Jj*fa8yKoKhxJ`K^TYzpmK z+zG$QW`Y=^4xt6`ag-3Ak*+WEvW<2Cf&DO*$%~HqN{$kt@aZO~t)2t7L4?W*MR=m( z>GPJX(7>luJPZg<1Sv>BX4ubL;`6UPHAv3?zS1D$^JklOWN`l4V;yh%4vQ;8f$BY6fkp zx~cdS>Jo8wOp3|CCp3bkvXp@nR`r(jyO@z=8CODtrUInq4cTzK`SOV}D&-2@NPbEHA|!!L&buRO_4kM#h))@xY&ncA z7Ac6bO)%c%i<1AnuK`4**^!WnECHvug-_G$9(SVuYm)udX!d^&G$1@NK3OV`IY+?x zOOh-&!pQ*^o&VC`PHNerwD~Vvw}#PO|(Cp-2q2?#LwLUP$b4bARot z@yShA|KV4xP_M}#X?X`X4Vg(xL}C*%Sv15NBGk2NIFC4iNxq1!byAvFoTuQ#D`bKO zRTM0Ro+(IyPYwutvihxIYM2s+s%^*=8g+XwO;PXan&TSHz5=505b}wXHo-t4gEs{9Ot^CCtbHb_&PQf;B0Qzh%}DWvxYw^^vFrHHaApi-9BRNXGA4@!d`U1;7;w=cllxV zamIiM)Of@OjfvUJusfO^EUl5D>Pf1WWJRMg1Ecd64TH&cX5wQ};E59gB1*~7s$JDD z$!acDeBSAXF*8d-)%(a42w^0HAb zKe0j?wnMo)qSR!=K12mGuEbcL7nm})$cyX_WEFQgPUyE27<@t#*os6v3xU z&VpNb_r#!lz7n?*Q}!45bYY0}qfbKy(svW0k+0)%U@Pa{?pNwwhrT3a*#1nJ5!dyR zM3r1ko!lg?AK=Q}#tKuqG75L2F3Aw^ppDIt;sAt|WYjGVHl%^X4c*9RF-guIGs}DH zNJ!w(yVG4JJ|(NgXTVSF>7@{5kB(6XK6%MoH0wls@+~y%ZNoB%Q2K*>DwiHgU4AN2 zWROoU)5K6Ci#5oLAF%ptQ7uPf#-~w7Uc8CWTw5h4#iz+k=MyO^J`tE5sQ8o#YEe)X zzl}{;hEE`rig!Lugvz22q8z~Vz0?doi!~uau}k$SL?s-mNx#!asIosUJOw8b6H z%2ti55}#bKSA3!-9{!HG@JbXBpLp(+uO~m*Jrpk>X~yaxpD-gn*#;4cAVnT3*{Pd) zj43`r6kF6NMTEKuYFitKr-TvG)fWGvulN$AAq;8gbj{Oc|x(Q|Qv?Fx~@cKGE+C9>FIfRKjzRPr^#AxwjJq@M%ty zN9O6@?ke@$XNmXXQ_Rq(1(5h68ZQx^Y!!WVum=suz$BeuTgLh1gL+8^w=RBu02?BL z#Al$?5cTLHaEjF!pY}}Mmrou#m!JOh^+tS#>(m2fBGCooy?lz^U>Jx`eV1Qw#H7gN zm)FQ$xMuhyQP8J*4D#vzdwP%aNqAhY3Lz$HNCrM}-sK<31da0vY$PLrC^e-!9Z}#^ zi});!Oi=iA74P-KQzBII6F8w1pC~oY)JCuPMnH8fZ`C9im#3xpWFU#r*ev14)(ku! zFG<)n!dF#keBA;@RFgOvI5}`_UHZO_9Cp>XmI>Zv0d=iQk zFyVxY_t5EZMlPd;<>k5Sz1}1JmdZg<4Tz4&e^QFdxbOzdm{g!iCaCz7wW(0jT{6=}XvDDRVgB=KBZxBc{@2L( z&CEuAUT@C1$rQN`+;$Hu`Gh!AOlA+*bSIkl{JE=)yZP51EDa-vyDTeL_Gf@CAV)`> z@RmNp9`*H7A7LAI0;l-o%eYs5O24x|7|)WQ*DWEC5Pg(OpW-vEm%w96 zej1;Ip{)E!o{BR=DR9R4w6RVa!V^TvI=1zjQZqr2r53%1Qj+m1vk{^2Nk9oiiCfAf zfKM{&1D~C#{1EYp$IiMhbnB zP$a|dgM5PMjn>>-M3XwL&mH<tj zdzIOrDLykWK_#pCI(;KT8xbn|lMW4BB|?Ey(IoMyuzcWCrYX>}ADfea`9H7L|HDU) zG_B)refTvMzY@=M{eGh@7c!BYG$KoPI*BF`lP@H!#4|2Fe~s!n*mO}z09k7ylPw6v z`3wNi5qR2@;=v#khjBi`yONx7K6#5X1U|Xl^u=dfgld*?oX>mZXLxEjp{^lxE*tiHa`PpJ2Xb}A^KCL!m#HWqC zi%&X8HeXoB>Pb@S96mvj2_R64|1J^A!V2OOxytC0^{aEy^6)7{iHo|(H9m<@-OZ;B zZVF4ji*FRGXnDk@vx5M zPT}|QB0_-^CJ~r44I8nI!-?e7eu(<3!xPrtd<5_Cg? zOF>M50e!d%WU9kw#pe)cg{a_UOVa`3Gr&oQG0IyE)3mx~hfHs#=zV-#a#i2UHs1KG z;M8}SwWH>I$^->Au@?vwpTbh|lRod#v{N66-tt@;IE>3rUdj;@pV*&>(BQ?}B-yq+ z5EY-wRTiHnIqPk?N#L|7C|X*^*eBaXcGV3i=evSWNRm3d>WkYDA{nQ zEiP}(M>!2$K8OU?3{dLZ0H@S9K-#ysAcaqf(;z~146+nHffM|s+{K}MC7GS{)r?XP zE}`Ewr^Ddvt8_jQp6(nupVII6O5)S}ch~Q5=z6@0DND#t-8lcv%VK<5*b|D_0&0K_ z03b?|N>K4u{9X7IgxaZhwUzt~TOAPRGlGr5Al9#OJ|#lsvzzd=p?Pb3HD9C)(5K>X za%SE&1rg(u$57OXOl_sag=kRX!cu&)4Sd>uTp`o!v?x}<_uk8A+=~~V$WM=gTJb=O zFU901dmn1^PC(Aa$C#I$bl7yM~M53biU9Q?#5<~O6OT$qTpFzJ1QSnJ`o=svh zK7B4HBLz;TD1}eH7hI**!BTtzGhPyWqQv1-${h|NQ*JwZc*)k3JA9h)+3??Pe7ajC zKFfaXE*77XoZ^#M{V$!x==Yl)6cVvvO@3tOu-fqMx7Z+eK|eA#RPqzXQ0u@+@Do0T zr6gpWPx?+mq~AG!mk_9(aX7VK=*9sSpMzo$PlriB&Zkh+7O|BO4%#Ip~a;?qQ^EhoVuBPKuNeD>5u z#Tg_&*`n^TLRm&xWXcLfnn`}f`J~hj_9wy+K54f0oR4*OOTW7a#q30Q21MNiwU1ze zB0@W=HGk8#h|1s@>Cvz+^`yHXW9i@}Nt^n2iw4$cu@ z#7ST>d@#;uDtY@Z`H7lMh;!86P}I|4%36G~!Y_Q< zC_LeMLQHOm1`!IPu!>N{*%zNaJtT5}L?B-pSMhu`FQwvj09$0Klsim9CbE>Rj8FKK z*pz%BQN230M^KQzW5WU%-{EYDV4}w%F zK24Y)RBdDv0zuT$O5t;eE)EciPi-Mh`^9QpHV&aw=yE`uPd*D;*4&t6dnd)GD7D~x zDmhtj!lWJqK&1)MyZN-|NW#-%!ao06W=x#A(3e8zW#E%N_M5n)^uOhkpHA{K&L_e% zlAjTu<{}MFM4~-gir|EagnKwQe&@|7ek4U=- zPf!F%bb8!)Ts7}j_qQzyMU<1N9pY)-Q zEseM6b8X_8J#jwcaMBNN@hQ<2Y3h{yi3#d_GBf=yKCwTg-#JN;oH~OZ5M7L44?Yx^ zRBe?jd?I1w38qwbz%q1($j`v1*yi07pLh93FzL|Vhfj?kxqdhKi3#fZogt)D@g-OB zmT%waOUyZ}%SBxiL2rVJtlVFWW<`N%r8Q4?XbcLw|f^*dJMbY}mtp z{LlaTk6{l#`1}9--OR~j`rD}2Q2USG#$T;-{=3}B=ij|5g4m2-J!2riKB&$8cr4VSOBWRn_5{|X5N z?JUle>!yhrDBjFQA&HQOjUfjbpEaw+XST2(a|hen%X%Q1X?156`6MSOd43 zo)fm=Wq5M*fblZEN6#nQ&=Af4bEE11x!uuGo40n(yPFrhboBpc?Yz69%C>%OAc(oO zgQ*SOZO%F8oFnEOP!tpuvm#=`oKO@*GDrp$K?T8_Gf37x@4bKAk8z*5X4R%dPrv8A z-<=)0?{-3L7HS<~0de^#6TUNK*xTNKlrN6J6-eBwW zU-nIHxo3K-UGqEnt{gjee9svJepxrIo!|OVTh{bhH1oH03)`%i`TK~TwHD9lnSUoH zn`rP>?5oV^Hxxmpo2-FUQieUb5&D#W>@D$i^g~)7Nq?gHQLS@q0PpiGi=-s8h7pf$ zGl?7h^k&q{8&NMZV{&gq=HE{E^yu!TtzO5MEyxKC3ft{_dd>2@E9dTooH??5!KH0J z?;huT{P?Q0q_Di?^Oxe!6{Uq=%08Es$|x@@avXZmI`ULmc~NQE=hCvm^3sAYjFQ5? zi$8PxQu67|gY5X@2ZFsl9$q;0UoRd!y_aF}$;p4qCpR>#Tc;r#(zI2c6D-f zbE#9uwMnD8$Njgz%1M2Y7MXt|?%?h%9sX)PcFf30lPAucJUB6=Mi*~$$ zi@f$#PQr)BHxk27u9-J*%d$S}-PrR_Z3TO9Q6wS7^m zRYP3Y_jOy_&vp0QKesF!IAv(d#ba9RTG@BM*YG`FgSUJ3+2PS+HibI0;SE}ZgMc6HCz*O@8hA77Rg ze-P>nqVUOZgits%xH=w{TliE&;j=;zg-qeIqU_VZir&7zlODcv^Ci#K52C|9Kf3G4 z=RYG_PyRz-YeOb{vKd~zdTuVYoZ+)UgSxext4$p{78arxGpFZ6u zOd2*YxmR-DUUs{{B8A_jvU4n9*wP zxMsc{qo3ctmX~q`4*5qtzr(%)5qWfPVbArOWFnwZD>7|-S|7|h;S*ZtqMzCEFYd*^ zfKTEUYKU*M1)wzDc*sVow}{D8uO0P2>Fsep^upyGJ|TO3??(kk?A{vWx#rc)wNaeo`+e< z*=f=DGOusnw!TBhHX}w39Y10G^yxEZ&6+)X_N-a6rj8ylAmsd!cX>C6?(pi+PxaU& zoJiL$kg^=*ZX!v*(-xmq?7Q+w^*=oGe;7V3`|hZp!e_SdY1wzxPs68e@_=tmhbkw~ zTHm9dD2k5XFXtAY*0WsX9p2nY{FtAb7I}L8!U=0;4IkLP-pW~>_j>o;wz8w&sy@dy zjXvf(agXPyIm4RI?p4ooOiRD{eGjb}x_x=?4U4+@uIjsQ-2mUEoqQIx-MOZ(--eOC z%X+Sw-e%#fcH6xtO4&q8h?(jOe{nY&K54nd?tKpopO5%RXrNs`tNatV?l^=`QYZJQ zXh(9dJtK6vExj@y--*jfkGK&XoR=JP^{{`^>A=Jj2QO~l^eE-*5^(}Tykva zA^&ZwSIr+ZxKE!xJ;#h0J9+X{(40Mc=G=L+h79TxuzSmkd+9H-6X26omulZlta}Bz zR6pSp6VK2S_`H|G8q^DrsrKFYqT+wDg8I=-@meka6H#R#p<>^&r5vFOpN{rj;pC`% za;);@S_E3x`K}%Lw7z;?`!*-xQ+|3z*lC}IQ+BPGF>Pp@b&GoWZR)>yS=+5kIv?3Q zYLDkIucd<*j_kkJAFUt%F`Q6^+i3q6a5&!JtN}TjYv*dn98au2A{r|X}JNfw6LVCprqmxj8#GLhYu^tix}nb`9%=rp0YATxS;H7@t40# zF!%rU>D}9ml!(pV;cHj?kNB($pY`1u)OBlESK!1Qbus_ds^uyV>$v{fsLn25uMHk6 z=ggQgW5%QzGp7s~*ndF(ffFW7nmTprjG0qsyU&G)NC z|3!b#r@06HjDDLN{~;EptRVXd~{I%%QmzR(8 zf=*vsyY&BoPb3sRiRIzb*@-R2-D=ma-LPSUUOl@_oig5i_VhWkW-pk(VBUh+W5jF(M%nd_tqS=OX%vND{v*pR87ea4s^w)lc}8rso!)#>7kHUWHG6XnmE| zx2)1+6Q6JOlY8_!@A)z(>cfNNjB97T<__98w}apM!G0S?tew$m;pnE@mUY{`yz|b* z?T+{i+qtT{$Bf243;tR=rSpV7zswrhYTcwiJ;waJec>OTvwrnj*lPWvHk(%TSnl3$ zWZyq}w5jX8csxv!;((>6+5B6gA2aipaScA72%oa@_9336@m2Wnd>-qQb5S4av$pdr z)&)@*hyNMz;MTSLTTu@)WAjoXldqnCnw@q({rbc7q^tgZ*_VP|#6<i2=X+IQ7YBoxn+K)!z6I(2K;tyhO_aep2+ZtSAP z^A|3fyI|q`8Pnazj+roMaQ|MtyZ`Y=o334Y3>iFZ{P^)xrcE9p zn@J&$ZpX^*Rw+^kfPUtg{L=UK9X_)Y&pO8M$|v!WqkUIyjU#Fc;XxPYQ%pR^J5f(0 zzhu&&wh;cu@sYx*U#o)Wc(lHEPOPt95WOk$9Ix-kyvs|<2tBuMbSLkTzxXZferVIE zoofbr&iZT5(srAsH})7;cbj{=HRGFm&S(~}Vc?N16E-gF=CiP)@7zB&O!|52g5Q^q ztFv%eZE{oF*A7^_pvTgQ9mln6>@#!Zt4!MCUV4yu?J;#ccZg!loR&-sNSjN>!%Uew z;UhWnX?;lk366Y{?0$)Tr?m=;CFs?~(l6-+LmoN)UP^3kO5B~ei0Gq-Uc|*D2KWbW z_j#6?{P*X#S{5 zfdhv2?Ag0Zw~np|Y5Zfkb_iwZ8k80k-n(bcHZ~vOnI~VufF{|~) z2@O5R)LuUFr{&|Dk{S)%Jmm1^VgBp;9osm_e`VLL^ILmN`Ne&J^MZl(J*IcqxOm{| z$(?5R{B7t@&OQ@|zsZce$L4>TSAp|Mc5FUVDT0}zRRNl`&@ z;hQh-9%n~ibi~p+|6V_Z&-!lmQJE>NQ@d8(I@ovDwr&5KI(^!habpJz8rZ%=$3OmR z^IOwKb?Ve0@6@u{&po=d=gq6vtXZ>`n_J!b_3Afpb9VZx`7bwPna5%W2+>dce9bQb znDVJ)T6{XjN2;I7=Oap8jeY-?&&vc&RrQm!wwXq<_*Ci~j~1VbsD7K)4LsDZUXNV2 z9`Y0uy05ZgKip5wj5zHvpzW&GbvJZxzJJ=l$4M3MzROiuH8^keg>^pjUPTGt7VBqz*z9?}l9^SBj} zeee9$=pc*0Dp&E#514 zZC=-^NyAz-tC5MS)8Hq=XT3&s>(+OwcIJRDB~vITA-pV#xIu46$!)@?EiKQ?kp}XK z#sxL;JM!pg-*x_oEAXj<8*oG)-NZG$Mv6~N{2ejz7&S-xu3x3ao+Fu#Pw^F7_c#K{ zcg;hdwGf5RH#te4^3u{nj`_?QeQ5EFQ%h$ZnlvN&N`j_HH9kLbOnUzg3phkA{hICsFH zZ7UW($-aIMM_p>r<6dOOKc)g(rZ)IpUwdNa;3ShN1!p$rQ$MyNpZYoVI($;s{uob{ zoIHiCHzU}9JnHhP=XWzg{P)}sxs-UoFF!ma?7-ejyL><0&o2G=`g76yQn;!p{kyE} zOGy#3SyEnDUe3%B;ZGqkxFd}!o|bazQ{V83AFF}9iRBT`ujM6pv)sefQc2mzFXbQK zJiD)O!l#9(b;45?qUK!q6hwtgo1E5mbFNp%wLyI%B-e%w>NRcJq|wif>*ANzahoz_ zY<%?P+{}b0cT@UyZsSs|TJ2gcwQ9T7F-a=7n$`Yl(ir`uH5h4U5EVY95xKu)oj$ZT>gayoLA^Hr+F)nf<|mg;IkaZ%eve^?eWtFR-E;kn z_B$7M4Olne;HD9KH;ve`w5P}PKRsr5-nMYqq4|?NTQ*(c=DNQBz};&+H!NMg%V$IW z?f9H@2#kA~dHrox($nGUsUUn@$@gc14A zPm*gQHRO<4szN5bS=>7CNmx{jeo7+LX4#AXE-#UrzZ8*cDlPp;-u~0OR~Da~==gVh zO3W^4C58i^b)4(eaj9F!6(Ox7hTaWd6pz%^x#rY~W3$rYZzqL6xD}Hcb+P^LO`K~~ ztLap&R_z)t&edA{TtD>mQBq9gKQZpc#3P|Jo)h*IKP)>#9Z?&f4*Ch70_(KWTi(B9n`}}si^lM{k z1K-AYj|iZoG0hwSpNh|;pIMJpKCtrn;!gZy>GOqzQu~haV)blX>gDs12lwR!pG`ct z`|iclC%xCDgkC5o?fLOlN%2QQnDQ?r#!-~%@-OgN_NlA_K1n)Na%cg`$@hH96$#r3 zjy{(Y`x`#LmK8I2pW?3{OTQFIqQ0bvAl_oK?tj3iNqbh|)1`JDmwI*4MUJ&mPX4P~ zyG|`vr$+VM_V3t)c-~D3dy|(Q6MSaww6W7Bj_la3g~y6H$B!!@=Z>H7T=gI$p{(EywHqZRNM#Wc`t|QJ#O*7y=Vg#tP%0Tw zaYr4o)bmQgbO_!_C{^K`jw1q< zCZXDzFkQ-!SaB*1bBO{>Y2e-KY{7W?E_;PuKSRYiOCu0W228Td%;+a3o(PLQnlxNsaPV%FR40!}L|r(mI(Ov=5wT zvzCdu8$#f4&Y#_-eknOMJaFTRIe~k2?eJXVvtse_UE9wb4mf`7@QGsqClBvFxOt7w zyy0u7v|c={uE)40`&ReavaH9-Io+4e9pCw?_b$s<)>gmAbqyABQw%$D*7`|)E^X!b`f_G&l zQVJq-Muae>44pPABt$Z^oJFF3wt}Uq6GoPJ7BA(ohmtClZ zPio>RG^gZ-2I3-6CM=RImZjG-`uPo?v`jr?{7<5I8=nX!@~CzkM2%E_qo0caq-cq-KVm@L4=((X=UB zJyx9Bw|&RT#p~zJ*t>Dvc5l!9dv;woe)8hcV<&cRJF{=w(e2)QHm~v9wf^Yd%?Gw_ z+_Q1x$~g;H&0graWcjLv^A}B;Fm1??(SrwVUBBkZ&A5kY)Om)&=QHw4@R`mcYH33( z^P4vPl%iIcR5;b3tA47MV*0s3dUfT*J~4cltngR+Zu38lS}KGpo2sHPX;VmJ$9ZV+ z>3H2T=;Eq5EJorwMql|f`pHV{#1FYCH?IbIEtdF?1*tg2m7pB@hmHWR?(Cqn?eF<@~(Wcfso;onRJ;`06%zB8kJJcb?^{A@w(yj z>0LT;kolxb7y23Xm~GzDB4a|?-0t#~9h;L5?7esX#KX&{PikcoRYuzp!d*YH2-ipY|T2W%(C zj-jS=Ju!t=ei9;yt!JYqmhGjnG&%$?6#tVDiFUdiaX|r9fy8!mVm%}Xf=_ij6;6&O zeixrX?K{)~q$0<;N+nku`Q!sIzUNaun1g1T#JIG$RXElEG&A7%pJWN*-ri40yL!ZH z=`{bH8$Fk}ubw+?*QRxA7th_fer3>sJ?q_Pterb)#f)(~*R5DPbIOXbL)XulvSIO@ z6|*L7TQ%Qf+OQ>~dafKlaQU>cE9cEvK6~=q2_qNGn0jR2_D7l5>BY}an5B$QXvmWO zjeb6q9J}yYB{+gfjC&P6W#NtVl8bptA3BYXZn8H?L@c|-hX;l4*>>ktVEQ4yyXOPX zZC?B2R%*$|Hzl7xmSEzI|4Aa4R3oA+r%8TEBDf#f~*?r{`73HLQ^=cg5+^FXiTd!7L(oLB7e_s0W&9iPUuGNvy zx^-&H)O>?_4Qo3kMPFum_nz#6Oa(H|5?D$@ zZ{N8MZq@&UPf0GBf_Y#AIkoRx(TH9NtO6&`D!M!?<*CN3E1!C9T`~6EhO4xvyc_W@ zFEJzR{D$R=_wDmPcH~sR{-cKv1|Hb8&uhst?*;Syy*BuJdoGzg-g~k8@+o8I4)3?# zeY(fo*{kQe`+BaOHFl`Sf|;up%-yukYwOm1`~8mv_#fWm>m!BVw_@*03n9(G+x7E* z&L`rjCLX0lJdJ&q^iqM_u{T39q~&$r6zaoS$LPD7{ zB61;$flnwZ`{=+YMWH{I&>!-N!$~$tOO%CAS;16H%wAUfv841v2_?{Pp60|wsD5$; zPHW(fa*mKHpDLuvX9G;OgIa3nWF5^@_Gx+vtm-yk)T#Az{aP9ESMR2V!DoJYD5Z?7 z3uW~wd=fmecSs&K9bLu$l=@_rxk+<;a!T~t&=choAeBAkQ*nh)$NUl(8EW4t4Yl}G z`wn%=Di8Gv4lq-MQkTxs37)kN9@@myERRxvbsp9Rt-aThJ|sOZ^?UcCKR!skdF8y% zvc-XZ{wD*D9`!#Gc+Sw0GbBef|ghj~+U7>fnKsfd`NKZQaO3F!_1-#9p%>$+KJV8U9E*sbP#v zfj9Yiv&j(DIXax2Tfw{fpV|UjR;|;3hOM)Ve#(AVx1zI?uf_)*NjU1C=)Wg+&$j%q zps)kmlY&l{y?RRivlyQcKl4k;m#?MpNr|$fets#l)h|_{6gc&YQmRLp*`&O=LI!Yh zk%0Zbm^m#kA?zwFd;huURmJm%Sr^X?fNy4)h zSN%^+JOel(P+8^NQ9pUeKs-&-N&HX5lPVF#62^YOCzvsXjkE%o9Z|*A;t&RuPwuqw zU{1KfcHNwaeb0@5mzR3$%GqtRr=D5A=JH1G(;jP2dV8JO;&aAlJnaS{J<0^Q`BZ^BdM(+PLBD`gLc$yiRWLI_AAGz{}fz{e~l68xO5rf5dbBu9XYV z?cXdCN8gH@5if|)={G8FJd-|lK24GVI4wS*QHLX+L{HkQ2tJvq*GOM{16-He!e{c8 zlQ)A%&hJk@e(?TPa`k>p#}_@xEBpAmsNk){N5tLV*>|$^@F_+0b_wO&0+Nf?xtbio$(syNL?+SA=6ZY?8fiS~TPU{3d8ICHdrAm%iLSq#^&9h`p z%{0B%J0Z5ziPmy%_H(_2h@iVPM5eHTus0J4jmPbQPxcOhPoj9K57D+|hEH-z=qCwJ z(+=je!l_=U8h0qQ{7;3JD~hNU9~nMv_FY+0{nVo&Q1sLAX_HXuCE$}QHWbYY6n(8D zoO&)99=>0RkxlEydvULF67O9*w{>{$9bMY^b^3E}yFUW@bPpKVi?Od;y8shumzD(7kE zCsP2%ZxlEg;(1C~XIj13X2HOc&+|Q>po?YyhELg|l21;ODw4kE*-4=>7Y^sfgv1{4 zOFelcF)-lv<#VTeyx!ctQTFb65fWPbQB#*jKZ)gKeiIdC7M?Vw%yl{p5=4$zDwAB% zDBk#-Ff8|wJ0(5!zsgHUfl5N4;(cks^PFp!!+gALe140lil+)GT58LVI@FLTr9b48 zry59BO09e?0(cW!)voPaedeSQ`M0m%NeShr%HnH`yGFs}XY#1bVIe2^&vcu9V&Z{Q z`J`_{JHMtrVadiwB6ggyP|2q{S2UtB+YRy)V-_hlJv{q*NPco;+{uG!XOG1M_&te@2;Q+JBltq;t0!dz zZ$Fc$NBX3YSmuu?UXrClWhPeAxp+IDrXrMz`pTq}vZ`8Y+)iFANlqb_3cZrglAp)F zE%{hc`tHMnthj^wBDZa>>0CpG3s$~TMaAgbhsCO;pQ@Q^%72Kc)Gf>Eq*`?w$bip= z_3PKHR;^>}7MV%mciDznrd`Ny#KR|E6S0PEl?R(JnWY&V;SLc0LK=CFcp8 zDx{o$&nFkH_(+1IZ{s6Frtw0vNE{*BbfF4m-jLY=x~A&joQo`O($52pYRZ*QJ=*mX znPmu{sP3EWr~>MfublQC(CO0Dq30(JzBFyrwHf2C&KMgqd)&2!6JwT6i(NSF+SE}O zNA!uBG&F3~fX%{BhcA z@k{DsO}D-mw_^>TVRqs&H8e3cq`{^F56H?LjFymIm8`Lp?9A*m;ih6n5`cyO=e!^^Ugk4038;60iJ z@8Be>S*XKK2=4bw3e&08$ zoP51jJh_vUlXi{GWl0q>i)80h8WmYQIPwXl1V`|hLl1p+a{87}q*CWMRW>;SXB9pz z{}aZf&%T*hmk9l~WL#yXCsk^OPwV(7gTA7&h{=&pUM^0Ekx))7J`o@E6AT1^?AZ7B zQ*VWy+A^@+sXo8Qj_rA&d#m7XZQ_UbzCOPHxt=X9_G%H_y~(A{znpGU|5~qB(S83s z^qb4Mj=v^O?t6Jq+rT!Bg1WU18`bMTmlj9+ci7jh^~S%N%xY0TcJHh##8cqm3BoIw6@hD_^c#9q31+%G37 zDlazjX>4@L@xW(M*UtNH%()&}{^?ag!7H*%m}as`I>l-9(-yW@8h0(HFQ*JirZ3CI zH5E`R`805v@CxlE;YptdW}}KrN(;Uge|nd7GxosV*nnM6VnZGItZBFM7M#|JP)gp3 zIVLw(3rHd+RZ;7Tp0Mb{=%;JF`Y!eAE1&h?la4xlyS2Y@Jv9F&=7~kz@hImveDc-X zB?~a{Jb{xVDhi+XXwhI0%}@U(K7voI`9JXKsGss(e8(qsr%I;T((Ed5K1ydEp^{IF zMott_>%>qht{_SU0ME0pzsgCz6?SUt;I;?a)j!pt@rhRTF0}b6=#NI{+crMYy7s9y zE@xWRJlop&)bB25Th~6@y6(wVZpYiyKHk3mkv6pho4X!u;dY>Py=~2F`TkyKYqL5l z8aYjCQ72^Ax)=14PDbiQrJqyOZTOTL8{_?gA6W7MI5I5fQ-zdQDW8bv+uS4x;l;mV zK{0%aoBhPdEPWnc<)_D7J)ImD^x{VH!?@_g<42!FgruA}d~xsgk5BHGetug>lvMnw zlp!I!jZBr(3I`Lf63Wp?sN81b6ISg$07afGOg8c>GyBEdQ&L#+@>#~&vzNTq-#UN% z*5#8Hf!2xTb~?LMw+*LibXek?#HPt@)}W@YF6NsSH+7|vh%F$hs}E627A2vvwd&WU zN(7!oa$&oUn@hubuKv3>y?>hZ^mgoMYek>@hh%LHG6CNlvYfia=m(xYcoY)w6dpF_szHUwZx;5Y5t;L=$ z&9--L;@i2YPlu-7fBmxjkH)jQw>-bk`!O3ur}6=ahNPrQKDKNbPEt-9)JTsSdugjg zcJ_9VnzKa>IFa&zd)MFGPkNo3D7iJWm$O7h5=_bb0$HXPIZ0W`S5qT`9^Xu23FpJ4 z_>|)Zo=05?-MJWHIR-ZeU&qf9Av9SF9 zggHv()5TdpRX90v?&4a*t+uluYVdPJRBDywKlpTW;IkI_e{%j#HL8so+Bf%R{Og<) zx{mUV%j$LPRC?%r!>7msA-rtzUMc2-nwu+@#KNy8&|4*2TnXpYbdMEPDD&LL~VSEqKbqv zTx;18{YFKRN-3fLA)n*{2%74Cy~cZ*NN6FxOQGCGNVntk%V;gj`8!l%j4v+PLvo}-Y=4?e#w5xVFnt=q)^6gw_a z(oK!dY|G~@PeVSB+z|SZ8#reXv|T?fIE53*k+RAu;pP3bq==v%?ON5YQLUbfvs=yT z4c%PpyE)Z&t=Yh(M*W&K>er~zkc&>$e{!ncux7RTHL5jqs?o@$W<%%d4V`P&cdAj} zxkf`*-1S-wYPq^quQt4AmlxUb>^4e&25dLs7!a!RSK(7#7pb(7b|SWR&x)vI@9)H+ zoNx1zzvojSH||!^P;RroK+H?$mHfX;Xm|IGG$2aDE{IDkVZMVNjAMg?Y$kNkKuu z>(YW3<*y$;OO6iN=@aF@BlY~z(8Ifec6*=Sx{jDg5QR@2_CgUGqQa*QQHk-*sOemt zAz6Kw8ntRsnCHs+xVm-8#>>(qJW@NK7<#eq_3QF%Ehnd%ZneeMySY>wHK=<|8vQF+ z+(9Q?`JYKh%Q8og&a`o;g8_#V6sx5xX`A z`EGc1E4AXotAckgKYe^#SoofZzPO_BbGaCOs*j*fSe5D~v+hG2PMezwz$K*yr zrwAb#)wTJn{TOjBKzo-PM^U=ySmLRCs`Dwct5(^7gLuNH5yndW)Z!3_7)tzb$g7Z#Wn#U!{6lf+hqA(g&mZ3we0ur${gZ;n zcQS*|oLRdfcK`M#F~M1(Cy%dRdd6#YV&LI~6UQt*Ekvz|UY@cgMVSCaJcUf2wN7d{ zRcEFiP%+%!PS%^%A{shLa3t$Ta2`O^O+qBM2DNJp>DxIgIfC`AGyHQ(6{H~nZclos6<&%Dwj_Kkxv-+>BVF%Gi(VELKPxDF zTvYI)q~OJ;*E#nyB4W<%IpVV*{LrT7iC3~iP961Jv16foeXu8kox6n#}4kG3ni^E zKvc}S_;pm)N%v0W1lh#0W5>xXA)XY{TTE(v#6zsPJ$^UAk)6*+H?hc-EkpnkP$iLW z__RT5_=HBKRR1VPM?P6^B7>4?aJ@l?=gSXhb)wwkZEciWpY6>S12$g z1Z{;4q(fHjJGv8$A zQzX<$R*2LTufWOG6<<^q*_mPzO*e^Cs)%xAE{TV9xUK8v)Tc)W0{JI*5*~|HViJV8 zo2xIfsCkf8oFHM(e@d?#_{9HIL^VEAIPH8&d?d!*p9-phQwpLaXJyx9K8!-Efg3t6h@sa659QQ0U{zaB-u_kl4(&d=NRuTEE zF~1RcKRt>?f~@4tqtSF)_=7Y8D@mMUZ6BmvW1V$gN(kNdRns8zKKnnB`7}G4a3|Nz zehiW~Q2kUs`O}hpwN+eK=9etvuKFpzi2F(KsnfLvOK6w*CeFEN8uZIPLyt3AR~7R( zE1pk#FNJ1JDc8>&&Ny>8_0++u`!;7?K6&fhv2&hl&aL&hdF;gVxY$p**fORWNjxmR3)EI)R9v7zs+Feqx4+1t4xUkN9nUbSLp?HiaS)k20=4y>*o?!RRM2&@1e}>T}XAyzy`2ia`Z_c^#Y_HNAgjcb0jUQA# zTjiw*qNY)5RX+KnEFA=$u`GiTrB8)qH^3d=p?`8%id3;F8g0hdxbNV|8 zpF~*E5BL~&67n-*@-k>p9~XZrAn8QF&9g`24((1q=Kn6{+RM<(vD(`LC(n=)tI;?*-Z`z|?nU_;cE!>=HH6N z0HH`Pu)-vTgiq3;p}8r+tSP?FQsTr5DxQ{pI!dU_zhik7PL>;|nH_L%pm^dcfF=H? zs3lX?RrK?=O+QsdZ84FVHe$0AvlEXpZ+6=Ij!%QMgh=XtDxZ(}ePvMyDz)@xnp_zS~=Fc28VZ`W3qbAIlG88l?PZ>9U;_zW3M~@ye ziQa}&CytpoZsMfLlc!9YJ!hK7nni~X?6C1k63L+oRQasw1a%B45QRy-lN6K5IPp*h zPw8!>j}TA&k5CBIHDaOh7>YQ#a6_MIE5=%+|1e?q3|g->OMZ~Uojlg5!oqp_HH482%@nG-+mD&OOR)Z6fCo-SNOr@e8)`Z!ciut6Mw5AC4$EvgZ$cq)J1ssrG5U%0iG7Fa zAM!~+gn0fVpSG4GQaYY3e42#66oo#@qBUsLi`(({ldfh)1;v~{bnDvr(0#sPJGaF9 z?F`-OePN?VtpCoGlL5CbA59B6ni6(8=E~`PyM5dj%$PKD!o)d~r_G-+mEIha$Bi62 zYV3q@BgTvzICSWcQ6mO%7(=top`%8QoIGjr@DZa&jv70LJI0S6Gj7ztL4C%J8EWT~ z#Cv6xvhrDZG?7u|oem_E!?)iie5!t0d45Vt$ctjjQPDcJof|eVlb-YrZ6FpOL|vS! z_3qX#_g4JCsvcN#ufP0Wt<_fsygmJ>cT=Lwt)HSU~9{k+M~t1ZSH z3!(@hR^G0khEK^)3ZFXHDSTpZ`CGy#_;C+Mqos(ayosgBTx4y&I39wo-M&jCW%!gC zC|mv$4k_MXebDnPHY-hF(dCm{(x!vv@O7V2&QD2SZkauo28m#h$qF@zgC6ix zCETVn5*{GSDw)+GL{|t z4xciEkEdau`jnz&_MLf@4W-!nNb2@;6GLOePTbCj$W4#A9dlU0!sy9k$4nkQY5MrdQ^yY(GHBGuvG6#21o#aY zI%E(ePM$Vt)TmJ-hK?9IVCaB;1BQ(pF=pJDk)wu<964;l__54g>hz@zKMj}c#dWY} z8U0E=bza_Hk7w~I4ybWKxdNg!Y9g&QU23_Qrc6R6gS8&@A-0aI)R~x#ljvH%dyB`* z+*I<<4^ywaVxKU|gy_hW&USIw_n-$dE5tQ!`Hl zd$NrJM%M5t87HZ`GkL8vTu@0UDW66^CCw$7F5`d7d&(?8h{-QW@U8=Bl~3kIt^5)n z9tC5&foH@=!sqozgh)(E0vkUEnQElQK%7Y|JB6=WV2pV3D&7R+j)aO%O4|g|8uUL8 zm-xu8pOjLoR{ytrDx8vUmo3;U^%E79i?s5SKZN^-Z`}ghHv&X=}gZd5V*SCMaAw!1_95}E~ z-@bkN^&K*7C`T@G{0BZQGL=AiXs;M?VwUq;h8^uY87G;Ia>hBRQ5{5GWVIc+CYs$z z2PerpF(jyWqdjP~YW;e4%uEdBi{J}-i6?f8IdQgw4bkmX3CU$GeBwEVrs!2al}|Qn zl=U4FA4%}eo#3P<{(C-Q07}_oPx&OB!i;Kus;zG`d;+K1aa(;|C7%*=Ip#klez)_< z$7UE)YnC8M|83$C_RvX$PuioA?uJjK45<{-jOWP-e_i=hH3K~U)#`uZK(b^QhPmw< zJ{@ZOxc{H@(}<_VC*nzDq?4PVYtQ6=!>8&n|CjiIJTe6*P|BS$jO#1SMlvFwXI>|$ zkGgas{KC;&@gb2xhtp!upFg?BW8IQD3ue!7pFVElco^&7zkgSHXLs+`wQJ`-y?YKE zIG|^*o?W_i@6@I9pMSI-GI(IGUOhW^?b@wJ_rKb=>)5qR`>q|k^pJ@c$2){pZa&dMTmDS7EjHlpgsi;FR1(~){Y5ir{GPR>xk7pfgxwwL8BZ``KdA)mj zi$v4w90J~Jk4yrG$|X7m=cESbrb!8?l>(Ir{f4a#W2Kw|r@d&zYb`r8$GNy`GlwJ z6n-z3EYoAmYwBg;GwI@8>g!V3fl~C7_#GD%Y#@DF^B>x`XeUwTl)^5T=||)oYH)gYvP7~$ESYRCKVY={LVh9z{yH#P2j+iq6K&i z3osMf(3VOrDvAn;KI95R@7H(G7jjfMIoj#6@kxM&a;kU&CrrN1Nv`D6{8$uGi%(AU z3U8%DSv82_Y>>zm{TBmtJN#|l^;dZbchaujObtJIJYeqpIWys3EtmA9v9ZG$>4jq24ruxrDM`zg=wGPS|V*~<^nt`wPnzsoFA%EddzzB{_07M~{7 zY1@?sN|jzbLp9dK?~;vI9Wbdvu!K)V^m{%HoMOApN+?^kiTVuinU^ZLP)9xqw=wR# zkJ@*>WkrQRL-I>mv_T-*i0nv!>pKkCi@PI8Z%pxnds5FF!6E7i78>CN<=0)#rae| zl|WEb{Zu}!a31S#GCz_j()=g?KaJ%jIATz%OvgpZq9CDF_~a$9$IH>r@A;HOJ;?;a zr(!GxEn2{ALzF)xd|LWR4IzB$AO&O*sFKJ-1>Ze7fp6JUt>L0c7 z>0sZ5PZ~1Zz5Y4}`<}v)-eM9u*(yKfuQ2puTNfQ4lB>r_##LsGk^z`l{@Sbjgsk*y z7tREDtXVe4efFd&6Gx67K5*~=L=T(ZxohW+ojbu8_8JxC96q_;s!i)gKR0U8vc<^J zBRY5O1foCx+89b1O`84o%WuCzHI$;I5Z-S<|B<6dRZ{n3J}YNGEARYc9{ zL6Z$&8^-qfJ={?cwd2Fizs^g8A`EnX8kNcC3EP>4H-$`+qG!OQ zA+XZE!zcH1T@h72ZDEe74?#c4>hQMs{EukT&Mfnt=%?A@g`zelx{=yB^*=d*QX{|W ze~Jr=eoETPflpjkgQ$cv8q|O~L%&(RM?!dViI9nXH~LA8EdHe9P(FEy+IJp;qV8(X zMX=$aZX(W!#iw=Rh^X?(KSyQU@;@~>k7g3K6CaTU#Un)wEfoKmPiY-SSVyj2wwaSn zY2`|G%=O6Q7thfOW!0iZi)PGpA2@hm=dOU*4(V(2=N~W$jVR&hv11?>PqKT@9ypl{ z{7*W-_aE4g(WP7G0fYK??A+m(-+qSAmaSVgZQdN1yY-Npjga;*A-VA<9kZJMo>igr zTRv?T{6~C#Z{N9DRh0Ju(dsqH15|gb<7rk$pJvul__U92oKG;b*s%Oh;B@4ZbNJ+F@u_$#nbuJSkRJvA z9Qlo?eJ8_j79ldTY4NGX-Qp8Ut%8GuF8MR~UvK`w! z*LbX&e8`SpPrceUz_{_r5rnU?%1|{TW%m+;wi(Yt}}8-b^gbE+Tb+)r){qD-^53f z5jFPR;U(3nbmThALhIFWYuvDIJ=bb!aUm~q5>2I$q~&v{SxO3`ir!j+gif09=LdX( zC<8u0%V?k^i`4(LRzOI?#FXg3C)SW}^8vGml%AT&Q2YV)EsZ=9q}%*Y2R_CB{Dx1v zekz}Q-9|i38$(L7zR}Nb?7Jm{yyV~UsrspWDvipg^;!#2(Bkjff_It5GL^=LPni6k zPi|%aAum)!IdZO7^hD?tKEobT+HJP!e1S)l8Ga-2(urfcR(mX&zi82z@l*cl(4~8? z9^{P>LpX#?wi%kgaKV&mQzuPf=hZ3m%sEf-u>ZgT*n03IK5E(ecbHV?lk-MD|MXka zU-3xcmLsE2zh1UgMJ}?c=)W12&{oFpmeE)HUNt_l$M4pR5+@d){AspJp!k_ID6u?U z9q4DfaQ4J!_mcRZke5cmRj|pBUnUz#iyO9srwUZ4zNYSB zPluT)kZD$~>lIlvXR1c!VOAuaZ1IlppP}?&R#e!fL*8C17B5{iY5J@ILr1pn+>JnH z(xeGv#*Z5_Vf@IkW9Bbjv~=Z)S##Z|%$PoU{J6!-m(7_!Z^4qqlc!JP;*u51CrzFN zt3*$TYS-@F$U6~6;TYhc;s7*j(agmA&2Sh@ZM>P>{LlIP*1jvEghHBMGDnHqB}773 z9hNo8d6lfbL9|BA>dZ0y)Bs3b>({PvGm&N8k?bgUFX;mFiA;C0>=)0|Dp6MbgirN3 ztMK{n^iw4iL(k;0+IRF*jJr(PROJ)=;8TseFmLjo9Ep$YWb&-hQe)gL7ZhNC6KT@; zomwXKKjD+>mfR^|N|(Nbqa&aCR24ogEO{U69rRB9qvi)<%BW1u5-2+QpYX|^NBEx> zpE|+%J)ah<-0+wmGC%6uI9=h-@5LrXojQMN-_oV?XUv{6c;wh_efu+WxnS|4S#xHO zoj3s$*Lbd5wbla~S5o1zY}v9^D;F(WI-RZ6=ekdrGP!sEejG_`;+GB?K5W9IiF4+< z;|bueV)tQ{kgD-7KNBcn=&}B$FzLI@(hpg+^z%o2QY*spc|}ww;>~PlC7+eDX%Mvu zuPh6#UIUqRc6N1fZbU;{H>W9M2IXhPXC*SZN%_!a>Ozfvo?!_mh>`+@K=c!xgin=I zOFzH2??gQszl*kHGAToA2pgXyMGc?K{zy!OcxrHDSIEjPqu6(yj#oY*(>^XkrW9?O zc^@O8wosbn5-EE0vkISZ1zmurBB{ipetJ>WQt#oYd~(#GOzKPY95|Ivwbyn&$x+B0 zmLs18M{n*Y;@dpNnG_>JXEvE{wT8u{o~V}cti4tw`f*nDt;CC0FCE;nX_foj*%PKr zA2Vs{WcE#;=e}~yYLB&R7cO10VbjLle*3oX-nH+5|H-rG{0|*ovSQVOB}=z#-^Q~l zs-wn@MM#H^8qtS2&Vd7Fy3c`aVki?IHPbM^Ma$m_vLM#Rr$e2*AxDdpzP0#2;!~+} zmiWE0z}{ZPgnpuKA{CZUtmdX$$1Iqxd=J`Tj-a@nr zpWo`IjZYv1QT0ESPrMrJ2*`0A4lOu2v9MIjUdd->B<@fibFY~Z=?NFFT|Kd8)uL(B zm?v_dJ%7Qx#S53MS-yDr^7)IF%v-c@|G|JG$BzN$p8bAXckJBjf8hA(Ge=LH*t%o; zjy<~((rvqTqN__+u3YKiF>ThYeuD;f?%r+Skin!+aZNGq*m^*PPtF-Oqx^0EB}?4C z@klK`iS+FPTd9in_+5an{7>@p`X{XDi4)r&oyT$6+RVCHT_li6c>d+k~Rf*qi&{#9Q5C| zZ0VwzGiT3RuxRP3mGc%a@bFseyL0Dmzx~K2z9q&Tn~sWZ-nJb$89Vpx-Mnop=mM?x zW}jXA_Ii46nmup+iZvd~SFav6YGltoy+@83Nu&>!n0O?V$tEZz7g~vm7I{`eSXm8X zqtSZG!MJWPVeX7C8T>d}7>n z@K(@n@kz+Tu&RTUPq=b4)|P%+e5zn_k8%i^3Mbd~qV<%%mGY?y45I&zPc8m@c`si1 zqOhqx~QBQp--1DeuQOuV`|fBYdiTM&KY2VevYw;3$ZmNAZq2f2I<3Utq z6F3!DJ)!T}vsozn+wqekd^T*r}6eE}X~2Z`-wd=iYr@KAYil z+@#64pfl#oMmb5GjvPITi6>_28Dih7#MkzRDvUNmZ8~1z8(IYuvxj_8>6nnEc&M3^5R+J;zN(;q{>D!OnlQ~krXI=LZHPb=U}N( zwrZ()R^T)NxKtfm`U#B;6;iA7yaS(dvyD$(o+&%o1>aAmY)|5Jo>I4ybIZO{p{jhc z#Vs|}rvHgl9rInA+zthtvQr7GBjr}87aUYr|`{a^V zlE{nUD4t+x;mx_lCvYfl=YaeSQdIefg@M=yYR`+!Ex|f2sa+W zH`lp`hUeIK`a*|a4%oe8)vDEtP|mgMJ!j9Gi@n~tcQ+zQ90YPF&z?suVRGLAOu7HD zQ>T4*?%IFwP;5d1WMchKojZFe})y_RIe^J-qw?VjprKHt2u07*GG4g(qqnN#-C?x5wWvr@hvGP1=5O-{zE=(Uz|Zy z>qF2_6xFit`Ld)=JWn{}sLS4g)8dnhR#;^5N%#goxyj;%vL*%6DA!c|F?>?_$?ky^ z*9o7LpldNF^_-@plT{l6r_ODe4phRYASzALD)}@fUa)l3PkpF-as)7=#x_reWtf&J zC%R!(K768|lEL^9pSk~wqd&m$DVle`)cdFDb8Od^JX zCr_O^ck%e?bC*IwgF~-!L^h-1;*r+dckf-k8j7x-K7a07ROHo&aKC^9JNNFvMv5H%1Izkpu_}y`;P6KHgD<(q2swDw|X6%j1r0r%5cz6QBjF*BpTwV z4RFj+7TMYTPjvu5+*Un;=?6{zkP0WMrIsyz>(svG&kdZqwrv_7bm&fUNLE6SEYnE8 zY+BsF=V>Gq!GurCzN>y(_Fc737&EIjpib>O;;AOykxwf=V*bdHPu?VtIQ=_5Ei&~) zR!AiapQ3W)QQ^uI#$z1eQv@@ES^~3kR#qGrl}ufa36KU(M?SC6oZs*%`bkz+;kV)= zHTPBYlZO_cBA%v!q%3nY`U$0AY52SjsPL(IU`r}_Eh703`pFxo#A+(8t#|=<*eOOAtL9bPWvJ}p)?{UdawU}^mE*#i5yAV6CzO@g7ZnjpQNb7M|Ok${fTnOb^D1Z zp`Oc8=IiYVC^@%Jk4h@Q)-Bx1VI(}jsH4U%6{KGocVsM>lcwfIB{K~DKpv#xxi zoZs4a$$b6(*PXUSiKVSG+Xwb zV7i>J=yfi}ovtSEiG7z9qk2Ct)Yrl%!_nebp5Kgke)l?C&s;jc4?ev&dTjRfC1pfZ zj!nnCJbL0JK{_W1Dd{(}?p_MHdL=A8=6d3_sOa$M7&I18bMo^;!^8I-IB+32ION*3 zknr%+7cQWv{)Y~d&v#!if8x|BQ>IUwJ=YyXn>1@ml*GxOZU3^fS83cWP%BXsRWz}T z*yYLzO7TuLhO$C?y@K6m$lRq271C;0aGTMu!R|;DnSf?3ml~~GH0j!*O`GP8fB&uC z^zlQ_AKROo5%oGh?S4j>IG<)_2U^fi!zWer@EQC)pQ@k#j!*16I=}!_Y}T{9)WR5=Wvf@ta$BZ;ys7e1v&M^+5t$za(#_4CqZ-+@nK z>t*f618LkL^rEY|vYOaff0JB-OokQ0!>1*loP5uxO$=G-M(V(1;CW{a-U*S&De=Mh zcrxsXeLeXHKKWo&M8sS^Q#Vfx|}_s3js96?Yw*o_ZrQDlXyj z)o_m2lhX*1c*+Dv8CmyoIDuiHJ$3GUR6@eOfPnKCgD~{uMaf4^ojHTKQLOyD1@oy~ zBE^KcC*9--p+%;iI6ftO+MG$r7g?dYV*q74r#mS7bE#{J(U-K97<6YYfw#GWFUo!i z&6_sr)UIXw*1!Ey-+B7D!NDi@=iiKb_weT9%vjcX)9zB7L&%XPUF4T&QA-zk<&*g% zEFjQIMLppX`pLOIMCBASOFs#bEdTR+K1~23&VVDI;%cIevb0$mIO}^_`boW?olnuv zoTT8~B=|I;jqt_<4UJ-iPmPaQhAE3NjeY{5+IPzr<$fXgMmTT^pJu^3|9r5t__PqM z!lxn%W9Xs6$qBHDei}YuQiFGn(l*4tQ8+KL#FLlkurBIT`i2i~M80{Lam3Gilh^#9 zivc@!Zy_K$cIM30$jB>U*T@=0#>PH;`XnJGB|1J43C+mNic3sMPQMwSl#-B=dObNs z%{`9jgC|e3b8@+odGFrQQ>P;1;!d7Bw`9eNwHwy2UbmJI9z^kMM~ogtphUU}L$6R- zC|YD%C;Ak(kwrI%R;Dol)sau@P3#Lo#p0X!DXG9`tsD<|+uF69TQqI_SLng@3ZJRuRHgKk!9||>Cq7Y8qn}0(g-@!2 zVMF=Ez7xV@Re4tVzfSoy`pF7J=3?%nIR{q*?@!C_%RSFVsj!erx8 zM#je6xSa`{k+IjqqvO!e_@va>grw9PY)Z$&c-&D=;4|;h<44b)=RSHA79D*(EiEKG z!pC>p#w{e8mXX^h3p#Dq%t=$HAffQd6sHB4MOT#*1yzof>CMViJlCn3)0dF%Gb&PD3zersH>Ws^p2nl^0OsKJ^gb3)D?%)1@`yG0-;|4uyR z4-xM1`LnLCS~SJeW6p((0lW8ZBPyiyBGNwZ&mYSy^k zU%&t6K6y+|M(nfP^e5Q~^zqAO^NZWkM+<|FY_fEhf{^^QE6OKTB6kv{K3PK~K!p>) zGx{kTc)=$o-inW;rGssmhHZ*dRMeiI$G%&95`*O05{%#2cS|baQ<>y%eT${|pZY9t z<|Ip&iK$rS69xdo7*l~0M9D9i&7Q(-?4?~*4lO)6JW=_Xu?uF7`oEmLhl5pBwy%5M3Eg%JNG!4#5VIl% zKt(bLR8dr*$PxuXat@L+6a|%Y&N=PMfg)!Kl7ooovClbu@89sgF~{1ob`^Br``&gs znQQH}L#_R--v~3T1#>3GCm!0qa}#dlnoXNSqoXnOiK(fs>ZInbg{5Wbxw+eR?@maA*fgA2vH@hP;FByU1D}{DPvlOB#7I#Q zMvst3`Lv?y*^o@8vDYf3dZdy`ZH9T!K}kSFTZ)p|rE~j!ufP1ptKB=a`O`bE^_=WI zwlFo~#}j3ZrD@#Jm___p(v4LXPtuV75%D=$pacD6VORiwiUw}PDeEr@ibyDh{i>g8 z>p6zOqTG5$LnO%IS+&S2p~|O?k8FO4bP#-+&_+hnDb;G?<#P>z3Zd?_OnNjN{}cTr zl%udljpdVIS@TQ4$*o)1udxqC-0CEVYQW;jCy3&OTFd@i{Zxw&u1f5)d_p6aiH`il z#G{|=!CpJX=&(}CYe7`s!IN8~pGaux>Dt_R($M>!!AXhkiHOE>^IEh;O=3uRAnMMX$N#M+G;=n~zrXBYOJp-n+E zrZY^Gfg*GvG5QWT3FJKmQ(0|=D~NK8)Xw_eP-?;)C#V;jZZIas$IwgU&I79snQy$_ zyHC%KeS3HD9rNWkfxc-mhgdVYrXZ=lIQclL!Yxj!vQFUGyV3}VB)k>Bi)=h{hoP7I za~ahVP!&bB@_-63Kw|l9D8v@aGK>^n(uz-^)FXaZ{lwP8XMG`^K}cV7{0`U@3nw4# z{u+)u%2OVEMyY)l$erFFCq9A=`mbemni9G%W#%|w2zVw)mpI7SarjhCgHLrnon!Dt z-NE~F`Lw~iRaEv+z>G}0>L(7cXYkI2pm55EnzX0ZyhKYYICZnWF#gtss`blejUDw- zYI?}t{o9k%()R@i9|;d9eWW@Gnc!DmRZWBFo{mmTImDuth$nEuBxC|A2avk2s;{eS zIu4%B<$$uB9Bo|Mvl?oP9K9bRY)J~cE_x-S{rYMz(Zw)0W6*)27M6Zd}{J76lUEV3k zGykHr$H<&OVsvY~Te-ROibQqN|L`wWmbAxp! zI6^!vpI`|yMq0(_*OBcv=r*|dwEic8Nl`2nMHNrw6P7HWk}M(!GDARBKdI$}pJrH8 zG&I5|kr507q#gjJVNo##+QOt%K#fh$&LY3Ga_t(*j9D`oQBUU-h%(QP?oQAAyeFc{ zr%-1^(y9LhoSMvR-p?@%+N2a&^~Z)QqZdJ0uB<@*N)!FqBVf?|}Cw!XzBRV=kQMC~ct$k;wzJ#ZAe=9z%ihA;?zNoU=icj2l@h8c_y5l1? z?tC8=Pina+jD1%uxwGLv(a*+`#ET7uF-Nw(((8q7+m{3%*}G}$wqsFI;jyt$S5Z@o z)yA5`9wdTP%mx2BJ z9oVt1G&laFgg==i(9lWKb3rheB#X0gqm$!>ml_LbbMs=D#X2`nEaJ8Lp%#9i6=EIDbiR`L z+^G7zXywz>z5|2uaJ*RTyCj$7?%4cMl6$ut(iT*VPY9MOo;1iW)fW&RO5|>fNHO7If*-P3T<(HE$i%qQ*9nU3$MoZ!QgZSokgsWIBsEHa55p)bV@38H z*nc4SFp)dG@T5JzU9>P@av+{3E+~tJ;Tnz@HQb~$O^3Kg-zC%57~`g(66k0KN!&^v z%(*1=aYl3e3+BCjyBB+P@6xMVr|zBGeDXoxecM)7<|Wh>CDj!sR^`RjEVA07!)B;W+lszPF|w^r-fFZbv6y3v~{Pv%JUseTdAX#hW5pi zqBrY@;C~VpQH(RJcOsLn3W{^Q4VR3M;p{YTCGTdQ<)qQWb)q8OXY8k+4}3E!`pEv^ zeTR-6#{UeDj;AY=WPD{+Wo300d8gn|z>SMdN+~ESOH5CTNlIYA=b?}zh2>@SCr*XO zBpeJ04-Snw8W|ZJe(XR<@ScMQSFT+@cHDTMiIdi>UC;WTZ@v8vd}@4zi8p-WQZn?z z_N|je#|1S7U(~pxB#RVJOC}34d9fBmx6WNVw)N`JX25%Wx2#!IU6fv1lv16?bc~pq zTn268Lm~y1&-nU$0t6xgrxTD1lRHSlDnBS^gY(btB`GE|dPm{3e5!n@etPmrOazuX zfSCq|DQ48h^AtWQBs`7(DU*2(pR9L8m@J`?{iGpF)s-?mlRG(PfR+NR>^;k;bv`+)6`zVI`xPM?mQ~)Com#7e zihVZ@hVrQkhRE0|v&f0^sqFouT0cc=E@VFLVq^Z%1M6Px-8SUt&bH1vBW!jvL#5K>znXeE*}5`&;|24HN!g z(IwiU$A^YN7kW`jL6pQM6NS2U?fg!^*Lr$!tEm?Ue)!hnQ`<&Ag-@N` z39BA_iYkg$n6#?ZPlXfHCVY}#a(7>KpPw$|rYN5Kq#;iDqKf zO&V6x@&cQTmUVlzCQW7&c<)$k8KS>-*|QpM5lR)K`BJXHxx1P!t-;B~i$V zk}%RfP5Jjve}YbOo87u~db4kzUfnwM@@hBst54S~o>P=TiZZ=8C9EWk$U3&hkg0r% zei}X-i<02e-InWJnRN8igHKMo%ANArtW+ir!oC9}L{U8X1k0BC>4Z7TCvOO)@M)xj z@gcG_7N^~)VzYiKnJS?qT^k(x9;JNpP3UB2Jw5oh_Pv=;cjAaYP5A^lm^qHID5EzV z{Y;k?cMZlWhW|uAIn44YUa0X0sO1ff2y*mzt9LC`ciHKy(QhZ7(9_j7_ zhe*+%zjUp#rt$2BE7i4)5z#U9c4p>e?%KC&@4-FCB0?5@znDJFo%{B0_+jg^)vFmC z5;$e@tl8g;_a6W8rvpC!axlw=yz=@h{od`zg35pXHyLOoL*9&3>dd!){f|FO6oh(` z*))cp$-iAYcYL!iqm4Ryb$D^?(1F|6FD=VXXedrEOOGf`39ZUP++u2T`0X6=1W`{u zjsL0kou&Srq@UGKJ%P??H*<9@pNgnZVZsw5n9`5D z{7;fF<|7D&obGkVv@e8D7S%D%C$%#O+4!7%bOctuHrI*V1=0K%(hgQ0l~3~*VNzL+ z+u_rqg&hY!%cp|Dp62+dC7<>w-od_!Dral`G;>CjPmH@2Pfz3|EFzvVheyJQ@}epFq%bEFvPOpdc4Eic($_dE%p)E}?_8ngoA(h2FluuE@ z0)naLFJ9g~ZxoiL4kf`YBl(YjO zhck1tLL=!I+7ucY&F~S1+9f2X91CaiTwGjY{E=gLppjA0v3vIJU%PGt%Q8;z@fkaQ z9IJ+6(|!CW4jB0HyC1$Y>g&)z?b&j-A-fBU-P%=r4!toq`#x`M=p{KWdaxZ3RKx}50RT!e^ZV=5UH%cn}{ zGw~58iaQaQl>wf|C+?y|)`cWk#LZJgRX-I=YAwpAdYKpP7N;^T2Gd#XRX1K$rCx>9kWM>_+V)s7_WUD>8 z#4bY!xN$>cY5b|`^aV4=eeiad^wiL8+c&J+xcX>hXjD>SctU(kQhZ8g>i)xr_Z{%8vHbPoRZTEyPFqBg&<1 z3Ja%jx;V-FK%92qG@haKjN9UA-xW^uvm!Hu0-s>1`l)=X52^a8-mLncnqNXZ%`H~N z|7<7_(um?2KhJ~?E4(y#N=1@d!YCc*SswXI)4UVzJ!b!u-Eple(@g4Xk%^aGZ0^n3WErh&E+Z<_k zwrCU+NT%=`iBX_1gQ+TY?)0(VV+ZEt#%U z0t1JzBA)leDKlmVP5;J(dII<;6CCA=BQaLmP&3ywa`Xm?4AGqYhUWq_}co zt8(M3a^hWC(dD3+83mKBG=rsVM!GViUD?qUxv`Zdxukqr|5N#t_=x;ck;A8)Jr6!P zC;CGupUQyME^FVdc)IO7_cfMV8$+o(gm2-0%Gyfq_=sF6cd?QCyP5U4=y~V;h0j9K zPt7&C_0#Z~>-d=}=736vCVcV|+DfnT2{T&wx%r`0KWVFdKUVM%Z@e7a%7V98`X3Oy#eQ@-Ide0uKWuoIFZ zqfd&0QJ^HUVpx57>cZJR^JY!R%8c2!fBW78yW*4MGqST2=%Ejf+I8R%qemD4vVHIF zgGUeVKXPFAfj#>U9ipsRw{gbI*{j#C*|K$uk1y+ik72?l143AfssAUR3>iLroX^B5 z(`U|{J)b4USO5YKlLs?>+qC^F>x1^`$-P_u{f(Esyhja-KeoHEEEBaXP7STfWdysF z%B(0CKxRagr-qlMgq5X*yD}onGh@m!V#=~&DstjM6gVxPyaX(X-_`##?PvIOpGJa2 zPQIsohXFtZ5BL-^&G;p&pE#cs$G+z{4UU8{RYI}a##?afr^6@lJ9!v-ju=_0;kn$V z&8?q0s|W~HA@w^rOpdXIsE z@X75&lTNr&>dR9W&-eTOyQ#6!hmHjA+J9ijfkOw591A1%*#n>PiJ?(ZdxH<{3qG)R z^ZJ$RSMNHoXZQX+Gw04_DDn^6e(>@2{-FPR!$u4nGe^=XkWXd>`a}h9KVM&1mJywwaHJ%S)J-f3 zsrDL!4xi=e;bp1er77X1so~s68u64mJEkfpuG;Vk#(F%jMMZ6X$&*k1YSm9^7~?{e z(TY!~prCKE#5d#XOjsm!UU*ebsMzfEDMEYQR~;`1ba7ZYr=n$O`=mcF!n zLYGX9;7-RRl z-h&5Od}-~b%`4Wf-L&n8A9ikAynM-`rQgk*J!|sRAl3sPJ$BTH(Ib(~p(BQmoiJhi zMAmbfJbT`}Zx=0^H-G-J<;zyAT(NBV_pkT%dbwMBuMU41^4WWvRxK<@jl9@cOz&hF z1g4V=kAX&DgE11D6yY;H0@;Mj($vVZG;obZLPb=w7{!6cN@iZ+lvla+Q}ds79`;?Z z#J(e*<~QY3@GJ9+pq5%xvJ}x;Zsfs$5XOfXIN_7FEAyyJ_&idTdlX73G@jv8*lVeu z!Y4l*`YDAe9uA*oEU0lcJv~fwfjFLkpzF5ps;<-;V#AOA+=wSKex)9|Shs-o(7BsAKW>wZr@9rMkkyb_i>mMQJ3pVTbP4Y7PH zgIo(Tbr1FP_{7*@5pY@=E1Zes1gYrFp#_~9EnH!UyDj5%GMOPZ^WK3|z!kU&W8 zN@u)KOiey%_5@A5i>oR68Bq+M>Dc<{3Ovtr_JmbsMbza+5!Es;m{*ajhfhjH^qw-1rCTB+4rhk)@uW(xb6Fg`Gr_oO)9fzrY(x}WnR*@xS{u4fd^I1Ms zSRDJV^M;xyn>%qCpDXN{PIkVM^L<+K>4fm)i%fh3D*S2uAE8+^DM*8?}mQV8% zCphxl$$nc5oGc>2RDC+aQn;^B z`XyBr4WE(lS&|k3pVc|>Rk)q-nHE}|9l@{3Pga!^S(Poxd8?nE@jLJ{d`?dY`U#($ zGw3$zgPg^uOzu=kEeeJV5>;h9QBa5ziE^ zN~M=SP}#Gax(7a0Je5y&@`f55X``^Hw^QbcDQ-0(-nd z$QQOWKDQyJTnkv1;1|i5UCJwdiC)yd;J4oVW&yjLNZ}1~g*aHQoU>8i=Q{=2&)#jB0Wj z3KA^%7F?{)|bMH!P68c+D?4e7dDZH}&K<^C_~)E#&z1O^V)x z%%acP;dN$3dV(YELLy?)HbH%}eJ3h6&L=hYo&F=No{8UOWgGUmk(1SFC@7>9KtJ5OYlPxHXOkelYx{mxQ?9-Ma*?VLKC4&se=;*O#f;% zGBR}M(IYz!AKJBd-_8x2 z)-PVVbjIwtK9eSo9P1r8b;i7fi&^1e)0R!^HmqK?dfCQ}%a<+t=7-ITB9Cn@%?o!G zgw+&BoTx@2qpLa37?-^0%It{BZ1ExMiW0%8GB1`DI5pCNO!h0EDxvUcm?YatP%kd1 zV4N4r2zhC^^PfU#J_?*97eXO(W+ml z6<2D0?#3b_CUU%zoabQGF{RXEX&ZVf^hgfYhVEojKTFBf8t{}HZE#|BQ@N$BxGJBV z>Q*$GO|hQ*I_}~I*zJAsCZ2_W%Vd%>h?r1R?yB6xtVUOSBkN^VCKsh2*|T+FPU_)m zS8{P-eDIMi2M=!Av1{FiO)I{cJ(;ZOwjJyCAKVfewu{js(UH3&j%~?EK2Vr>q$E4I zz9h1#Ear@hg^7?&_+-$L6!@%i^9iC=`EjJ&tFpwIgg2|7%4cb+B$u$`6!;_~Du?C8 zR^~p-XMA0;053vn_~a}ppZ2_2KKUV#JCRUC$=F}?Q;Iv^2%k0CE%;PKfm5~gS^Xsc zNoL;YCzeho*PB}rC8&N<%g4DrOw~_GBAHHhbwT0Q@Tr+AoKlBO4?e}z%CB$uR3I@H zG|(Iar~LZ(HvEAql^m>S0ULW%HhESXJf$2`vy-As+MV_Q8bGM0lv+=z<;P-9v23GVh=E(^7tDox;~v|uR~ z*@I6w1kPqY5kteL(N78IJ^55J%hL&(j9K^KQ&WVDq1hOtP=x|<21$|s06F`7v7^SEcU=`3*4 zn5x)7D5l&!ZBuUjsq$z7G`;}DY7R=jj9e*napqv_o$+<@hnNmcC>0uweMe5U%_hUM zL6M@c>EElwDzp|<`xFg5!1Bo^JM{qFsa!!6Ux4$e-mg8Y5JlVarVW(5)gJeLJJ3Ze zn(p*uUfb$p(}hZ3fPMCKS$b1(GGqMc31zCIBFx+-m!B?)KU0!$!Of@I_o^JR@4~0C#bT^+ zK8^ort-Si5Saz$QkZF8T{LdKD_!T+Pda%PMlorHOs`KN-(4(Kir+78?y0d3e&N(OF zB;*;)R*PmD=Tovkh9!UzKC@)5F&r{@2}HH1e$rMxO?<@Kw$fW__>`F)CRmVxcC=wq z-;&axZdAWo~FvmU@Bv zKQfSvksER&X>^J4MRD8_<4jq?*|LNSmCx5t^%brEiE;Pj6F319J|R;*Ps1nVL|5g; zxafsW|weMK@imY(ePdA@2E_eLi zl23tC`2L@sTQ!{I-$o-woQ*uo~0_j#=K38Y;vV^JSlZNn_#kfbuU4%5{ zlbsqL8P8K_RO60p5*E4nB-+Kki+*yLf8u1?uBAfl^+@ch8be#Y2VcgC4O(kr%EUnBR1Vw9Q0GJO_;QN!m9EqAMNCq82@bC zOXGranea_`mRIpL_zwCtxPs`HRCx4HfI|dR^;7i36o*gK_W^Z!yz;3WT0Zp*u#;kU z^3$*%2$fH5>zn9KYE3CCpXP^l@ZBA2u(xpY&MPtX#JY_!gm6GeUr|5N!y42|=NaToCfPES5L1{Co>{||hc`A3>x z;*YX?>dE2E^Ye*_8JAC_68EnNsd4y(D97tFe0qAGu8bof3b7z+^pjrr7WN$pRX){r zc=8Fup89DdluN+jlMjW@8u&DUn?rqj>@kQx)&;K*JM5_=XUo-C4Drs>h6N`I4#Dq&Ec^xbPW7Hxn(PKPgJ z{6LqEJncX=%3so6x@jU#sQG0oGcJMm=I{Z zFGoL>Rd!Mgs7VGnw|%D*S$c<^v0j{BIrAoI%%TC>t~Cu#i?!pNA1#(op5pAOr=(Ga zCyk1z=N?Zo6;3_IgXrq=d+_MpCX}VB0NEp<&)FiP?*5qu4jqA%jze4 z^qdQvrr`25$49OCl!X>s>1RtmG4Zmlg5{HuA@skSbLA{fqDt6GKT%FCo_und@J*mp z1k)Y7vlBiU?gb%csFDQfilSVGCOL0AkMt?)e+qK&sbe1WoO|%eSyDbFRwW(n=F>6> zqPFeHCu(Lq(xmNE=Pl}{{kGoQ>bGJM)`Ab6f?+-;NupK8!6%>WXPw@3rFjJ-Gd$=hP-^8yMlji!;hKj$SYrHSsS&cij!>1XgFL0Wr zpE#_6o=qZg?#jo)lDKVnn2h&hWmh>5EQQKyy$qa|gd2s-8*BQJ&o@Dm;6tg^D08mde9DwJ&K|Ite)hPE%$V}@D264Mrbd*dFvTO%l@{en zi>yeGtj>z5$&RHqJ)UCUozss0$?Ri(FLPG8@M)Mh0?m;*w)BND@72AWH92$P1Wlt> z6qOtX)R`fm=$MSGqh_ofgFjuFA>~9ynIUEA!Q~l;Wh^KmJiVWUNc68cYjVn+hCB;v z&50PAV!j59u_>?8fqXQoa~T3{u*A(UX)8rnQ(M&LS;;SniO)kYQKV!>!qEH1^~hjF z5l{Ch5!(DHN}$q9EvMSpcipd>!m0`9VUN>An`f0$^(8#vq^;LG`Eb(Ujio6%wnv9b z%96j&IG>hAxs*s;6hnAyeFpt!SZ3rTtMbY;I>y~)>iPzR|19^aYZ3Eqgo;K#Wo2pU4;4gZ93XC(tVhKCQY(^J zO-|O|tI0t>QBL@*$fRG!Su_bg3B$!woADvaZB;h@XEX!mfm4(-C7dZp<*70e$?z%N z^t=NH8#sl}YMg(Di!l2b&y(Dy@ttHn!;&}(r$w`N+`F0Jm+PIi+JsN(=5FQ_M-h+! zQm1%G&eQP8p2}?a1WqZIPeoVw2?(?D%xv#y%9T7kBcMz&O8AssZ#tv#Maw0*WRjoonQi!FD#tPO(TSCM2sOH)ZMg_clKj?o|;Dl6q_z>V}Q&>A)7P|p9I=+qb1RcU_%~zsG%!p%2mPuA4 z=Pb#6RrID6;u|2I^k{GlmeiTQ&$S@)_lb{GKZVZ>$G&q3Gcu%EKOOr{7d*+&W2Gra zOAVI5S)43wrYV&(sguJ<;7fi<+Lg>yLQaWKbK`JIaY5?}h~ML7hzPD4S9b~5xbe@K zn4@!Q7=ljsG}-BVVkjb695cq8z$7^t@`N-be8wpSS{b;W%NNEiIqJ$F`597{hJ+re z$U27TNorjZP^5#HVZoST#<@w~99Op~5*qM9(tk>KmCQfl>j`*9A%T#q07rrgR=xma zVgjm|v_c|1UnU9}eM8cr)CS{xd6~Mx)Gaa^_u3Q4fUnK#>;?D#}++!uis2J7Wms%Ah9HNfC%A0mC`>eP#%mi2X}I}xcLRz4S%lUxXf}--8FODAusK(mNmi4;jPoLI zPAN=}WX0^tg5>)0%=)sd`tq!b{KN{@*km>$XTF@8BPFdQT}5)IX498Mcaou$c#5uh zR@C7mNX}oZ&aixn|7osY&Kti4X1W!h1U^pByBQo(k`ivRpy5m|uFPU29?8$xitIS} z6#p~J8S=z?8uwMwDRROk|0%$5!J0Wc>X7S_8nV_9L)Pm|7rcbbGSrg{IXO=mR%aYP zA~lv>Vy+VsD(i=6TAsQz^>A6*VUuE_i<8v4v#uAete76LKBVR}n<2gJk|xS!Z90N? z8P+B4XfF0$)&#@Q*J9Sqn=lrI&vKa0ixKf8mrh2=OlDCFTgST<#nhQ7!^|OyuVJQu z@c^y!38G*C*qjhfg5}d<$$phk{S`RUoG!_T@{34Yt{o{bX&cI?UV!2=6eSSmQ0#m? zl~CnV5#{`|X%0(7KjnK0oD$rr+RONPaVbqL%L&GvaV624W7h3wbX?Avs_e$n^ph1? z4P{v;steNMj+Epi6y+op<)xizs&y6TUpja4>cumc&z-(<{`AE&O*bx|yM67#nZ}By zs)Cx*%(}A7<7KSohby8E9+YzbQ-qYOIPFY1aAv_L*TRLGbdpPseu{l}OiT;=E@R_$ z99=Y#5m;&|P7W(d!Vx7K&xs{ZLSdo@Q$mEyN;;&FIFeV=GbKST3ZwJ8WvX}yOIXv( z&#>NDGG=@?2_a2Ck)&}XCr^L3#Oc%$xe%(DrRMPu9kD>u^@%{tqv^=wc;Icp-br6z{i$+ulO)#RSL&qnpjEsRQ&`wHkD7^ zZ&|fWs>bRjpJn_C`A`vmH>;dj1EH)?&D9}r;$1s&lNt*F601QmM;bnfPq{sWH~?bV z4WHDa8-lTW{5;3wXP$M*O`{<5azo*nn!Mu`IaNjJySJ`czh=qN;JrDS$(JvlxpMhj z~EwZanimVj#JcD@;4xzcoHU5`C7D6XAdOR6nH`FiC($Gm zjTvhvVLpB6c%Cc)%JfN=ER*52a=!p6=EhYd?%a2l;nH9}f^Op6%Cd^a&`Z~qyJ*k^ zt1MSr&n&szNbWkvuz5k$*mtfwkv7v~&$fZRe_8t3WZ?87pyR`kd|2x6K`%j)c?y9T^jR^M(^cyl_$e^zV zP4M&apE7v@w=o(s=BtsTMvNLQbBaIvggM3EFI^lJ6Mp5&`Ky=CovN>>D#>VMW&Y~) z^R?;cs!~rg2d9+9^UdmltijIRz^hXo{mft@x#XAhbPDEMAD|6zQQ(Tln zv@V9eDDhZv($VtNFuX*_tU@HoBQd{&F45c=#x|oNRRrv2a%c^)4^ue7wJ-rcq@L4N zN*aWn&hRZ`Zp4}A_k~PsWleq*H-E2&2MLR?YciJJB$^~A37NDpyEvTUf2N!KQo1BU z*~wEJ0KjzXsBZ8B(a;~u$Or@Z~CAPw^(61v#z5`saqjXFc(IHVq@8 zCO)d=VkELNd?KDwp5YS}RX$BWyVz@G^;te0wKO(e5#?ahF$$c`%Bg%BzgJwQXZa*P z;_ub|ape<4DH_ou^$n#+s1q6Sa>KJ^iJN4Qo0$H_veY`F16NLMS$16b!JtWFdv|Z& zrE^? z)M)0m&zwD%di2+<+C5<4;DIw{PEScnIn{K$v8L=~ZE<4-w^~lW$i2Tw2yl04SK4_= zoTjpt zCRGwaCZA&*FkzAcLQ0X#hDhlWD6t9Bl9FQeV}=bozOD&EMNG`#1i3jKgHeTc$dsT) zj%QOksvuVJ=KXmqy%unT)n8=+UcL_!N|BjIZa!8_PsfxCiuJ%*>I4E} z-zAvhT8=;ALO^09=~tFg#ll&x+_c1~pg`ZAJ-T>#b?niz>npFmynW}6`;Q*6B;Cq& z>q0`q%E~M93JM})qE@e4_qF$gcR&2FLl>_XU+U1dL+6(|dG&dv?|_d7PVx&}xbVBV zbLWp7G5UiK2Yk@~lW!L-uBooLck{~0+9K|#PG(gSRx&#*^+FAuN34>Ue!4o7KF!iJ z^auMW>)22@Kk%vX5#v9B)8UgN(?d(s!pMBWXK~80f~1h5*ysC|Ev)hy(E{K#;?6P-}7CnA$8uqsiw`)2kj{nHgH1E-52d z1{2@tJ2>V_XL_bI*vV5IpzV^B!$nC4wG<~GDod4Tc}ij^iLfMtMEFlQFL!J)GvKh} zpal zu@q#V44gnhNUO$OgLn9>%WuJ_T)$@aD-Bbc2Z~rk{|TRfYWajN#994M@iYvdh$l8$ z`82ZWFsxFE*VI@dj;QfJWhSZI1gZ>ZV;jq`1qqczvSpb`k-;B-^nTlRZMmCU&zF1k z=-n$OKK|^*%iI1V3t7WQ-=seT$hOf-e3l8ky+PHMu@#Ha(wk$Y1j_oPJ~NR8Srb>w~;sZo0hQja!OWSp+aK3$V_ zqAL9aX|(FBGqt&=YjRGqQjqBsqt->4OniqUDl=cm+#0`KGIhp0NpvYGI~)cQ{$lkI zYP<`{7a_}%N@Dzsi}$3nnESAE>t4B+qUoVy)mBD`akng!abl#6tdl8cMqkAX_2cyY zc{iplG?r)86sI?oXEc_j(S2E#NeTwh(4pAAgcCVzJnQCt8h~DwVf3^-6KYjM@|=CLy9bM z$vKpSt$PoA#+i}r1Pk%yxlxT3*$Lr?2E5m=efu_$32YrYbz*J0^H;8NpZDte0uUmpMRChpc4gK=dVM7KD`)bgLp@WB;dgQRdqepzRg*`)aSjAG|x-?5ayiK7E2j+s1Z>|_p>ony>S>M=on zU$cK|z<5(8O!4=g7U(lI&}WMOgsB1E(fZN54LN{Pe-& zC)aM?O3TXhp5)!7dlyRQ?wz~!=!%wh@7d$iL4zhw51KY>+Gm5g6~cgF!-o|XWh8^>I7=dxrt{sCPY`9bPZFL)Mx;6mQ&@i~ zI(X;C0PoS0e7>GAZq&H3qsNUK>+L<>$7cfTj&h4nA78(TzJ7lGffWCM$$kL=te4BJ zmjWgS_)qo=oD#^A_^h29I9VQ3r%s(V-8^PcPn|h)`ivRVD6?kGm_2*eoH;BVC5`#> z=Pp<C-WE?uH!`SPXTFI%#F<+2s4makm3V)@FID_5_g ztXi|0vTnor4VyNs-?(n$rVT%A+p_tGO`EoC+_q!umLE2=xqI)f@W`;(_?Wl^xfyR# zN@8MiLPAn}Vq$!3Ohjr@Y*Ku<@3^76H!nNxN-xV~fQq~G2(>IAwJQX+;M1h#QBhKh zvcQ_S;!;=?h)uVCa?Y{rPNGTqR3^c=8BY9Cr{79@ujHd!6nd;oeo0+W=CbMdOcM-B z;0cP&YH7MiB>jcUS|Es;l~-d_q7f${;6B1_8r>(^)D~{`?RS+ zH*Q?7t*yCm@zVeN{eS=Z``;cv{p0@6PagjA+k?lyK6?E0;iD&ye);|3H zc0qMb#kI?4>MQb2aW5Y34p5#7oR=#z&152+J?(tw?1>iypTzINXBJW^4Ivhv6K^wv zxGB0QC88)fEG6>r^r-=p{Coog{ijY1qD-GLZTie(V~S57gFovTNWZ`g(&WTToC?Qc{wgotc-LU67wOYl`oo z?Q6I*IDWeSUTM0 z-8{^_?lHA|A`A)8GE~>(f7e|Ihz^^6P)z zzViSqe}4Rw{Wovjxp)8PyZ0Vl`SHe$pYB}0ar@?-`*$Bcs;I2uX2e~)i*k19>V+1! zYu~nSzgI_$8nI;gD*vFV0|tKf@u#2e+rRta-CH%~g{P|WPst7b($80>U8>2{`5G8; zJw>9QW(<4@U72n^p_C#W?)0{2LTqHABUM43e zB_+|IhSIdOG=POwHhC%~H7Uq%!oh8;aFR*7!jkf72CUK{#bQBX!DYV;!)8p8EcR~n zQ+hTjiXXM|sn*K!sjp>IVb=HIRV_fxeNG^6;iTcor{>d~;7B4M4TuC(*3U#XffHZF z@JYHjnpHh7G!&%9AASGb*SmD-)WxeyyLRo`v}?Cv`6 zgNMJ|zH|Tn{hxn%^7P*Qhj;GYzkcK9ZR(%!Mju|jdgI3JdryD=BRJ&PTkpKxy=Tvk zojbMf*uG7>7rXc9^7ec0OrADt$%<9O#*Ti!|A!dO(vpJfm(JFeW}S3Ne*SW87X6#b zCpFiyo-Q%vnGu#xrBwEqVIg#1!Y9tB@;P;~zh8h~V1QpxP@oj<#xM;booe|+N-dwL zsFF#;@~L~Qq5>(AVDw~Nskq{f``}M(Xs|l|xpCG}cB>G9g zNqXnqe4?K?pXz_Ys)du2tgI@QoOW%npLfuQA??-Mn-2 z?t`0m?^V~=`}j=i*1ZS*XZwyXp`YD)bm`Tn_uwHzzg@C4Xy%Mh2MzpaK>r=vx8Ava zg;>6cF8IofOEozcTH1GeqAZ^@Y_WWjO)5;5DV(aG+!Go5K4r>eN)W~9C*sLN$y9Ss zt%e>T)z-W96I59PQuI@6_@se-M?BdHnficI5T&hrvPnZJg-@(JpaLg|V&(M#lNfhu zj6R6MD&nb>9yoA-HV+yY`pC#g_*3yzK2uUs)Z)W!Obj>hiH(Vg^z$AKpA2gx>m~fi zuuR6@$FlT|oITY#vuX-VDxVsLSil^{OwXoj0%U68Dg6obV0r(RhF)Ha``R)~Z~3(5 z8=tea^C{vfLqycp3!;2ylBH&_D14gN#+|Rt$xS}?cHf@eyLax=t;L1?LzYcmvSK-l&vVZ>AMf!O zPBmRUcbu-zb2V8PYjZACWw)~LdVXM1`j3p~seICgPcol$dXibfzPtG(`Uway`U#(t zr-+0?rUW@gMM0D{H4m^vIeDn9R|!@9R5GCv!Gurbly)mVAyY*a9fhpL#qs2bi5Q zN<{rng;Tx*mx%%g$an4V^o5YTu=6r|!MF_v!n(_ay&i zt5!}53}8{^&j%08%TB+2^?XfHDw8`d)Z|{NFCglnm>$g5_FWOxx;RZn)5*b^k%cLt z`H3<)PV_TC`4sXE9VDxVfnloL?dgi;ih8cJajN@f1X19mC|$6p9BM=LljC($k5P+H zTZ=+#+cea&x8f5HAy9;|h#}~*%8g2;h1OF%&0*53i*o9aJjsZX+AU6Y%pSd@ zBNIN~`QUvr@EG?${`TLGfBE&vufJ(z1fN&0-9$FYEI}p*psqjOxCNPaA3g@o%h!IQ zAf(`W>)wM?=g#~3`*-Wn9sPW%Lp!f-U3&KJ(W6(-L0^3N-O{D=7cCt4#h?%Rf55MI z`}(DmH3i&w{XF))I@j`P&lvuvs3k+?jf85uFy$zGGSn0QvoJX{FX3o$N@#M#fj~cB z{{Vk~KVOy5fXM`882TxynZQY#8o^X1!IB5Ok zsd=EE)C#H^deqWdd4NPh)g9GXi9H(H11As5r&6lXJ#9=pjwm4#a6%&}qNNHdl(HEf z9*%^Dg&&(ZZp7j3Ye?n`pK$25@2&WRBIT3ds1={en<}R#pX^r%6<}?v8tY56#=bwx zCkp8)kLr9Xu1c)7EwQBHDe%dNh&t&Nm2oU5$}_mpmB07M&YjzL>)E}%SLeU9Ycpi{ zS3h38j((z$zdrq)g3pP5QX`iT3z-@mp`y3$J-qYaXWULKJ8dYva{VR^h`svLO-x;n zo;}dd_8mKP_UhENyI1d*d%gYMd$Z?%yJW@E;bTU;_rd$*4$m~zooOhkFH5>qpO1vL zvhSQRjgPE}2ULoqpXnHV86r}Ugq$ABOAZg1M4FR47THa155f)W0>s1z1;Hn>sc@3s zghrTDvrd}^53qz)(NhDeYAHafBdQHUrCaBi)tK}08aX!&ciZ$-+ zRPnTY>Zll*L}NO?fK&K%WfY`EfBNBDojP~u*0X1qo;~1m-h#O|e!4~MgOiDn(nCZx ziQLh}OIL6FM1b@NIEmQ-6$gMs=)Fh3prS@IA7b2r6XSm8!J|u8udUy-sn4sgbm-KP z(y2?wE?qnKg3ou~{dVDZ+|l@JpK;>tv#5<9C%! z3ujTPnax28G(D^UG82v!B^}F64x8-nOMc1Ue^M(xfm2MoG44>Rd}8835hfK;#8X9; zoxsV1r>vF7zQd%}G{6!*wN@d;3svRBrmKpAC2eXRxTD$zOAu8)X#gh=C{;ccRB9Yi z0(nh}V(V#;Bc*|I!Ya}l6%|RM)bdHEyr3vce3-RK>F)I4li*$XRQqnTNftuY0i-|? zZH2BQl`T!Y;`%I~>U=7MSaao5)1Gjv=<2i76pmph;wkzm2gtG~q(kWo=1v)(^ncr{ zYv*pgdw1#H;Q2oiO?cETDoVHvQ(^ z`)b?~&vTc4L`%s((VavVn@)UO(01)#Qa)dPrFY+c{bqeLcirXG<(m z*Ur?HC0%JOyi}8?d|LmLGe%o2yTVBgoD`V~mmXfsIFO`c`H8ZA2>Kc5CtXGG85rnK zehK{)6CV_$$_bwuD4~|1NE-sdlA2Ahq=r=*Jd{t2I}dC;a^)vdZe5zAwA%ryA22L~9OJ^KNSmZdOo_uQCy~p?f z9vz(;A1Qt+kIbpGX?f#(GF4XeQ^~aMD8Ui;12yAVOlV)6aJoFbC?jUThi`W6)}?EY zo?bnA0usi=L`IKq-nvb|4w)Eif_Fuf1k{x*{g3+l2L>HKe*F5Cli2r*^?8@- z^6@_rPtKK|0WN3DCzPs{$GGEv3ZLj_T3CM4QROo~B_b$5x-8*y(xi!k=#)VC#Gvzl zQq@mtlv6FfC!eaKkjYNFA2I=0gCl5UlQuO#(x8T0Km|p}L{X^~RBE;LJcXQsC`O+e z7ZfaklLlH!8z5{ZI0rVtGyfkxX2oFCIB{ zkLpzYBzULbf1;nFu5SCzsirMMYMr7pvL$}!{IioDPti$pZxqw}DaY#%L@g6cJP-vr zHQz=aWg?PHvoa%U%%D5qWL%9*3DixgWuS=60WsrPWPUMYxtW-jpB6FT!+zbmdG+Ys z+pA}<_8mI#*I&B)BO)p3qg#vsd5Urp89n~x>0KPld!(6uB}&53)1V`s{L+m(cYnNb z6DHvkF974t&U+6YO`kQZYj@D^(xzRT&Wvg5;nky8*8v|7pto?t51Z(b|LD_CCi(@q z$}4`lcDlYifytYfjsK~9>e&NSHdQ$_v!wc&pTqw}4(GBt?)cHj&|N`pj2)eEK82TXz@HN>i@8ts%o38|z;MWGQZuRyXBYfc*!sX=|h zgb5QTPWf|z^lSpU1nMp>@XeH@z=PACtWIknruuOp| zOD@Ze8#(N=PMuzY&o14&ckkBi*wK*2#wPd#LinWP^6@Xfkyg5W=iV<*euYU)zdkEH zM|bXBymaNtjT-=c`od*n>mOXZbqmEkd-lTUv15tfaXaZh>e#i5R}Zf~ul5@7@kh&+ zuUfxpgYfy;r*ju9x^w6Lxl`4RuEa}qITx$4P)mSR`>x_?`BW>f;#rW)Xv{F>Gd~gi zBtF8vhfNC#81Fp+@${n?J^=k>90<9lWg3A}xtA&(K4Frx$EB_M z=@}fMpBVRMK2wij-wTt-e}-{W`0u`*i)r`u^>z>qk`NwcLq83bQnAUy@`;eDtye_V z%2PuljDaWsj~FrH>#x85^2;v=4jf25YSbuXbKJOb?DX;RQ3<66UG@wbGzd_63>Yv# z8RjWI7Y+dxAXQEgQVE>a3#Hg}Zy^Bn!4m#<#FaveUQ6V=5{UAAIb-+pgk z-${P9Y1g)Mx9+|Ayxh0nYkvNI2M-4G>^PsXpMClH)*ZX9T)tA{Vn}e-h3c%!Rav5n zGEGCqhbW)4btknpWMqg*&eMNXkQ`c&c$DD1BpE)VmM;F**LNb;9sBO*=MywFQ0=<{ z36q9Rfm2AGG6lh;ZG=-QHSvll#u_-;33AG(zrX+Z@#E>8qCw_?nl_(=eW&KS(4A}| zuyD)4kcsI>W_btg4@F4fQ$14PRAS)`{Zxw&mJqAlDxxZ(unL@j%464_-Fx@#J$f{F zdXTS-4>9`5A`I2s*HwDnnPDn?RtksG1!-;rkDA zDW6tLtFZOv4XHgWyR`WraV^|~lW{Cnr~wZb6jmuTOpq=^L0Sbz?58N(#|v3MErI#h zOjhE^Q`On&2}j<2yHAG>FLvtG;f25a)pt_hrAt>%oj!f%-ksa`?jeg$e*fFUU!LMv zlHR<1|IrN-)L*-H{pRgkNF^yI0wSIzvvlFgk7qAldi3Pi5|?Z8v?;w`eWj}z!o(;h z@Z#a9u&6>Gs^XA!ezZo`q_=qv1a|`mXU%Omeo_Erfb&k6#lqRE`Kxp}-!5(on z-9_PqPxhGLUHJ@Q3TIw&Dj-CHPZFV)PiVA!8a72pt#Yb(!l(M8Sb5w| zjz=m1Pk~hY6h+A7!G0sx?DGXk$mAH>)GDgFbL`l$sP6mkzmKBgxMH;tPsl_hsd?ak z!X(gALn-@h18z}N0ESP6lX~awT@;4qG06GA4=b_ns-H{%tYn=NSSS-oxgCIQs$~Wb?#!*>2tM>CoAh38cv+Pa{VVfPsWE3h<-fqGx+S@TgKtTXRqFU z-stxxj@jCEYq#&*Hf`3-&xd|Bb>@uYO$}Ero~kZkrbFJj^0f11i3+6hY5mXV!pX?_ z;#A4Z7bZz?B=#LXznzD@o#;E!&z~+N?vvbtPq4%ns$!}7sp6?^j>IKE097QZ zp^iPgR#^p0s6$#+EuoP;+NS1Ul~7etYWCm{;GK>hJ(@Sin*~dNB#c5T0aCS;ouCVo zYyzr2)WlOGq<~7jjj=V`w_)Fd{KoHZ&8I9-DRZ4<`DmHADp`641GDZ>xh(L4eyV1g zC05KTKyE&*gksSDzw=1~6K#|^8oIQCIK7T|I^FI{<8%3x*;ooNd=`?eWkCrRZAdy% zl~t4#`&s`tUwrXDkWl&{v$As=PM$8WsxGgtrp_rS%r7V^FRyHDI$2g(S$F(IRbAt$ z^B3thI(`1q$+H(?6H{Umk~k8vMLcs0i@hiL_38T>2A%kb!a&g1UVrWV58t1+VE&HX zyEbm#guo3OHFD41J-XHilJ5^4-RnR&@9!KaKB zNePF~?-#+R_e5VG-${t)B>0>}t{$-5@w?Dz9zde{iF*l?8YLl~C?^C$rph2~z*Duv zy@W29R5+DS$mC#XwCc(W0g_`lUQImDau_>7^pj6M>C>kVa*8SUd zC3{ptslgTT)JzmF0o2V~f7rBn6FNG1(m3=}?K`vKstQ=~MwZ->Wtt41tTcsbW{DdX zy*yE#-cX#(s#BHOF|7GRp+Rx6Xn&^IcM40VQvVJOhZvv@Q$9{+Iw|s$7tfM+jcgdI-8lB ze+562exvIQN4j(I(xr=+ubjDX@zl8s7k<2c_2!+rrc)=*oJT(?m9_Qv9{v1Z|LcE4 zDRUv1(fMlM*L(MQ88WF~ex=Xb@4fZ)gs(Sm+q!Mn&N&Od9X@7kV9?~^;)4JA;|bz< zqB5J6KTnq@oULf)lgOwQpK96R(=qOJ8zqM1#uFS#u3q(X@wcaHNQ`sWuvx02s!gj>FiA4~ea3Kcw+)?BT6wP%G-ZEyuG*!RCd) z$zeQr2|E>04uDeLjR#ICnE_A)QH7IQ$@JtCL~%r+6f&V&oPo`o)^FTErxSc0-0I&sI~7bJcWJN6|{Sq_UY$ z^b^dqc)Fm!AY79uc{;~1!HTu^z{sfsif&|p~)5M8GSy| zUhCM7xfkEcJM zZLDZ0O=tZa?tgT;k~J7(RX>qOPd<%2ib!I|8Ha56WI}PM$$T<0gjK>AUnll`+2VQQ z#*Yy`C!t^y7%Kvupuz)wjE)9E2oJ>-D-V;@@JS6h;G{fMvpLzhSik!BLkDlcd##-s8Bn=Fo)V6<oRaY`V?5^em8~Hz7Rk zt69P>f%{>dVwJh_)LK@PD9`wAc7Rv=7rJ-%Iv5;$;?(KFlCpD`E}uSk{=}Ix)yAfu zID6idbEnT=$SW>sJb8+2{7-l8Ui|TDOhV$Ux%1xZ|Iyp;z59B(7_4TFNn&QuqW* z-Un~top;`0Q~6XCg+`oGVtgo7L}>u0f{KtrspT`!cbxd2hR=%J=!!g;Z1z7T;+ECb z=+|aFVHO8Y)p>CA1X=e|l~X`<-gPY=Q{klcwC~U56Ac8~=klqPdh&^cN*2@% zUgA|8!2Ph!R%bB3TUL`RPj7N%E}c)T^zWqZYHO?O8|w3liXf249l<<*<@)v8_v)HX zx~gmHjyIjZe1%Ymy2MqnW6$2Ny?s9X_>-+jU@4X9|AAZz-= zNXYK}2bQf~GXWi&I<>T{`1fD$HB{u{5ixTU&y(@vV2J723@u8Q!Ar&T!ly)S-Lz)h__4_5BqnQ4B4%-tO>l*{QLCUq8V_()#X||T z&8F_>NFXGKiJY<%hIs%O*?7Q%I^IVg`kvIl%zlpNFwg~8TulImOrGU!IUW@C0FD7s z>Y+o2lG`V!2T`y@LP3#Q`Gi%lWE0JVOg2?T;dA|l_2?(_J52nZLj2BDO&9B>$PJ*$ z&PjYkC`6Hrknum!PiAx?hVTh{il`cKl}%;U@~I7uePJ$ysc=oVg`BWgOwWx8oo1&k(7$Yxap-Tg{j%y<3;vHnYZf=WNk+o#*l)kTY zU9oiGxwEH|Qc`jYOX`|VS2v!xdguPtn|B!=&oHHH*RM4kZ%oO^+_`sO;M8e@hkW(k zM+4q{|HF6QeV-n7%10lwvdy5eW%(!l0X7GKIwiQKkn4f=_F zpXlq2e){=IcPC&de#!}`(10btQ?-Oo@MAv@_V6Ip0683>PXQ8E9@2P_l22oRmJO^Uk^-M0qzJyim&%%Ti(0thjZsn$0sEnJ{G=y%k~xPH?3N~dFP(}d-v_<6q4#5K62!zgFhee+5g+# zeShm!-1(lz_HCOWloJ9)kdTBD5(tz~Ktcp#yKO@|Vv+!X0?A;EvD+9hNjZlj9cAh0 zC?`ZP$!@3VJG18AS*vfaiL2NA1GDa0^W)6xTi@qwJscgxv4r~?Ul&EGcJ12X?EQW| z^{ue0t{Zvv*b$@0jJbN;sL^A_U3<;-6DLfYG5yYO{ey{d)~;W-blD06DDM1+@80{PhvjgYSIr=$?vC<$H=JTwCnVWyxo> zENx!e++f(!GJLW>6J6s&=;y7s%~rlgE7MmBpI-?+6Ay%NSmlDn78-FV1&S$^I(-nO zms5&7K%#YYuN?7p_A0KNGs(+LsIGtvN^+u5hLuS znX2<3M5YKtxh&Sx{Wf%q(`*O$LA!m?m(9B z8A-LQn5-g7`L~Q;(x7zElFCJRWAe}Fi_f9Ug-=FooKsDokeYIK-UHuVzjpa={^`Sp z#?33&u7B#;=L|-_VcHDBcih8mo8bpa>Z+lUVr?FC-1!D+jo8c?rmGQ{@?%hzyHtw{F@`D#yv(YH=B7rX0fB6 zN4CFja?vAA$tO7ZG(IanF^Ny&X-)=V^TudgNbgDwpAeOmPP?J|Pr97l2uegcMyclP&|s94@mnB%dPAEN*Y+ zw5gz1jyv29R7Jcq$Y2w_i~)mBAWB8SnGr$?$i!0NE&|;!P@i~w{=x;my)*u^;#2v1 ztZ9we^^ccnqoU+((NE>OmZgeM_@!(f-?8S{_SM1X_SSq>0$bDDG8r`~)@oYuS&1h; z3wvQS#?%T$KlLJIwFsOQpO{3WY-wTUrj!^~9iMGRFTC$XO7g6-M6cj>ZaPd@#_ISf1-TecmW|B!Z*Q{HskK>_-=iYnYy*?y6(A@NI|Hr@n$KSvE=Aj)7-FIVNBYeif zNk&qYRg<>8e|SfwpDCVMInt&CKc3BrOvVyc_il(`OU;`@Kk;c-Q{#%C{p5*TZkbi8 zMS63>{=D6IyB2G);xlPXL4y|dP?|vM<&my7l$+qOSBj7lDu_XqKxUW+3Nr#rne#$I zhXYucsE-~!8jwzvU}xP*NRXBC&ab%%oO(i&QXwEwnh|`8OIh&f!g=%ODd)d-TTMUJ z-_2QJ3EfIR%jZZ9^fT;Fv&}>g_^HZ!O+U3D)$y5DXp&h2I0x75Ua$Thmgh(Hvz||+ z6+~m|k{10mtS(CYwM_-uG)#+((Tmnyr-Y(Ucm#tSIr} zQQsh&`eut6s9t>T@rKn)|M_d3V+1B0j9Hy^}NdD$x%N_@vP%>F#?-6E=v6jwZx~!`OPbnPYlz~mtXwJ&9}_@ z^6iuj8Mobzjws$wxe8H`pfsUER7TXh$s;(ktgkX)EW3U(MdmE}mHVJlZC4?Ju3EfZ z@^J8WRgqLtp{vM1C#8#nHuV#n5Y19v27+m1DIg6i%`SVyic3iYKsax~{DY|;;F~t27zzVLtCVgJK0`dWMYDXxXN}$#>on^7rKzf2 ze8##Gs_j;PFvVH$nd6gFFe~vamTIk`zQD=OjE+%8@~`#o#uKv>A|_w=||l>FbLZ z(a-sQ-3#XP%J=tw{o#N4*MIq^U%&JA@#f?EO;)@$mWVBjA-aFEX&!r1&|6Z$5)A)z zbaQQT?``G*i{qL3Azh1D2VXa3}aTUH!YUDB7dW!2I}FVIgfJcO&1o`%z7 zqjK0tMDYnB8kjODgc%6IS<)iNJm6I5XWStoa8h0al9`&-aH_}>kf@{vxkLoYiBAz1 zO%z33LZvCwGtseGQF0X>^e%x0-^+di33`DL~)XX>X@)z{x!>1RzsW9H9T6efn%74fX+ zGxK28E*7TE^ikp)S6?O}r=R*^`C5yUwkFlCVkE@W)5m-zy-|Z?%tvh4nXxAm4oR?AyM43pg<;1T8{1 zo8X~=LL3r3k$(O9iD;-#ZXqPmAw!1_O~I5d5qnu5cM_+PCt#ugO-vFiwX7nRsoE5qmR(O{3{@GocM)mH2EayzH#!+=6wf_zwyTJ{_y)h{_%JJY8Ki* z{^qR{2by+nIC@~~TSs=hV>OvrP(~Q0a>_orcCDOQLL?UOI6%MuC8_cs~^x)`5JU;XK=v!{_xtvEWo4bs%mn0zK11<&9UmPw$H%W9Fj zIhVM~17IwXM9{(sLdcXku}m_>5m?R<>A6=3HiE3YmQ4l@hn(ARl2M~ZxvNZcTKWl8 zjwVCYHzTl2%}h$``IMq;V%XEq+i#wpe5QU{@9B6l)qIFpR4%eUh5-GiD1%*`8({MNs)+zkKL!L8c!-`c;$1R(Dn z-SzGf4VXLMJG@;}TZ~1q)YRTJ@9$gpYs*cxRO0GEO(2uc)KYO|#uT92OaU5vD&I@n zk($x#_!P&dQhtw3t8v9@@2i#X`!t7cUedg6W#Mx=65(_6Ekp^UH>GlhJQY5{BL!Cw z;@}4%swJMrXR0gG^pB^C0)~@;v^rK1@4Sf0bVo^do~$$_pAL5knUN1X@kG}>1f#d# z?uuy4#H8IhH9#>?4W?30r=k#5qZdgmvwRo&MEijUA9(Ds1-?JZ{`VT67L~IGL`-gD z{AX0R!HF?srq3e2m40fkD@9d=S*8m!QqQNOTKq?vn5rTkfyrlS{0u%tQa^=YgJOkc zt4P-Fg)#q0REVPfUhxSVWTw?Bj+QczPvKtEPs^j2+dNL9b?d;UlcvJc47)E@@?zyW zCy_KwqN%F2pvK@m3zjvld2ip^5BIJ6VE?*Uw0p@LAcP`rl&|eOY79A`!kXPCjADLsXb4DVQKk zG854=@YzCUe*N(gK2t)46>_jtpqtAv`2=UZc%n2-R{4`>DXsX-z9bE{^1!lQKU3m_ zvCv)%pQfC`7!ECLr|ncDY6x#xXDU7wIk5hH-IT{ohO9S#vunj-cFw$S<9p2;KV*64 zDodrwt1@>h%_N%{N38lg{jBG+G=4_;&MWI{eA+yX&wblhHLQ8<=9_NFiu>%FZ<&4b zO*h{%)2rvGlQ8J4(@*T7?()koCy6P8=#q#Z`i5T6N=H$qE0^(>7=u;@2PXLh!p_vS$NCK2;cge64ti)QhZQ$ot!XR780<}6q^ z@8%mP(@%ZVm3}JUt*xVc5B=2gSw8VdGBeDw`*c1N(Ui>!RGB|&c}9O|RlaBEdwmgK z>1QiGVV321hAFufpEfa*75UW98lS~lNIr{Zo{~^<7koCaWqhI(iq-PwtNeNsWosiuJ35Ti=eps@fnxD$VClpkh)a<3LS54{f!I zy01Kx**7S4Vwv`q{FsuA6w6liWpiDo zECSO&l_DMItW)XbKH1Bj>Yxx6T4S2^U{q#78`63_cT4lu}XEd0&X68k>+=hmtqs|Lckp1dV(fN;(3DLw-n7C$fU+PE2{*3y2jAcsulkthqNb&|<&7;ZxIO@Og9aSvdftisFjH9x60QVkW^8!KacJKT@)A z&z#7SImJ`9Hl3UbZbFJWH0EysgpP%7@@W^r1g%7xDNzQ_O$z-AsI*JdOoRD>3P`2A z5UA;XaL?U7Mliq5P17fu|5N!M`f0ftsrnJd((+8kgn&dEpJXbP_`+To znIg%moQY^A=HZooiquPJD?W2c>m;y}&B_>%B0rxI?e-9bqR%?u6d%IJ3bSF*GBs1a zgb7}%T&z*m49HWpczl{iG%7yjJEmL@H`a{SLUm-*hbOjtaJ@`PDChuC-<$kif$Q{5{m}LIm=KM^J{X2b(Ehlt=)Ym&w#d>-7qVt>ODd`j8BeHA`u-5BM&jvai?x@qRDS<{nG z40!ci5|U6s5hR?#9Xt}zf*&Z#mWqllyPQOgNgS)mw(xXBrN}HN^B^OYd$Sy zUX2Sa>&<1)k+bQ|x2r;>WQ!Ij?MsHi3enI{Oa8^Ge`VcpDr$LjNb7il zsG(5i_savz;61gc+Tbgv%?qVI*Dk7{FJ*n|XN^y4u%p(wS@Ef$*ZHn-Dfn!N#?J%V zLN_r-YkbnrP3soTo;ihV&YE@8>|1V8zR$X8`pqGywOR}SI255GSt6AOe>>7$ z$Son$U2zF2@f69F?|lDzcinv#K4(t7-uQX58!6wTe>29<`|6E;T5mDdM^LgSV^#SM zJRB;`g&E1h-KcMyxi{goS@HQ%L{I4_($d$Pl=xIXqM!H_hh@cQil;-W6!eA-(9+S# z{><`SOJ?$E7ho#=L|0DGRPhqoa)8Cf6GN%i-#LQ@%Ks#V_=V-0$vP~K(b9&D(O+mz_z4j)4WpIh*072udHq!pj^ zQ{{bc@>%9*FfGGJ_}sGU+jo9-R?vA<@Huk^{q*WNym{$GK4V;HVkU$QWy4bP2{v&x zYw#dJ2Ed>fIk2(GOf6uZ3JRftDCZ=c6S|zt;?Eb#(%a9^~7h!aR|tI zJ`+*tsh0Sxgf#h-o<4Q8GCnKiJe|*!(BB->9^RO6rjE&(M^(f^T4n@dFX)Ospy&n) zL`(T@UEQ<~3%yzm9)MUt2(ZK z&*x5>^y~P9C^$6#~Ds+Ke%Nvuaw89=a!A1#Isb6;^U(z+&9=v`!iNXFpysT z-Hdkhb8Ex0?|$oRH{NJu5#w|A^cho0DDg}qQbgRCi2MAO*`UAso%`;;XU3H4vVYSrTKSGoO-!*EZp-|2t@s39Rhn1&S@9_| z4in&{w4Tp|Q>R|VXRNZ_qMUdWPuMV^kz5gv&q_bfz-Q=f%LgD(DSVc1ISZdigDyCe z&us0ap9*@GXIbU>)Y5e|J0UtoqsNnv)pp9^z9BNFXjJNxPjjB{UH870PZg*lo~e`g ztRiHyURUv{1TURLF?gxe)??UGjtrrtF>Go5YrKf3$^_lJ$yc20PfH{$ZEjq-bJL1% z-QjCWU)bL~W9H0h(}Pb?rK5q~I}9KSu+%s37@>rpibep?Ta|jKqzNhyiZnw(G%W#} z&L9H_g%z&YvnzE{I!CKHaDvb-DSitaAjzymCkJXOr^BVNbzLO|(_N*6VwJNQxAZNm ztbF_Kd+wSsWxV>)`-iu(h@RJk7`{kPayjqka^6RR%cWlZSEE`bCmhk-~G-dQw8eCM;iZ zmLZ@UYU|r(;VVx|gS<$rEna-7$eTEt%^6|V-$O#fD6N?gZias5J=A_mx?la2TAE?~ zlvwg^!;ghmZOqUaj~%xOIi`^LxO#8J|n= zxvOEt_muCnY9`Zj=JYV|red!sXh2G!z=={+R5GYGL@=on2O-X0<1~<*&}Is#KgO_%hpJDwQO%1*tLRu?6Q!ulX7q5KM3*CgX zfE`_#6{69yx@)B$Lab39%YwwpG8LcVt@x~!r$u8)E00D$9EetYO2nrFi5MQ@^N^^v zP+)`P8(5F{RB2LqA!~29@JU7M`9x_nG&SKfmM^c>-%I0FUYD5TGrL^D=^o;DflR-e zmTp$xx5jGGEkb>Y&yvCbw%&B=XO8^L#?Ne-FMP%jrixFK*Qp=vHSK)ElBNh3?c1_^ zXTyr`-tm>-QyI_t469H5=#&EqW;i6I0~@{465ShU?`@BOe zngCjR!z?WJXST@~AZz;h=AKna=9_zh&-C>dWd`FDrOqkc4CU!Uc8@|mhAr7F-X<&Zw@ zFfDCCUTe&#fX^~S1PQ5E$tN~wa`G7^d-1@_T8s#_d>5Rx32b8%u>>|Ww$$G#C;y3j z=0)kG8^9-H7@yT|M50|b!5MrSD-vxEwQo{V%#+V7>#c9Ad}n0xS-w|2M=j-hjZami zs2}-eb90c%H*JVU(fy4p_BO6~@a}KVnmPT(Su>~3h~jTuTHZBn@cm|&d6@=*9 z@ZrNnQm=rTDoWoNoDxBT(n>`k2VKogP6(}3HspXAnPgLGF7Etzb2e9mt_VSPUFTtK zIyqeBaeE^2OB8CxcUtFkMMt@`kV`Uca#k4q#y7t~TXgtMMc;+guH_GhJ^_)G|eE|mI-$s{&+o#Im) zsq)=eCR6Yc^WrQ0BAPjus%H z$X+-p@v+K65W#1fk(nn?F79yG1+xQPRNR41<(!~(W9UVxNE)Tm^C+Qg96rURb6+#T z)ZZH!c<0oK<6e4pfyy%b(;S{hVy47ZrG8}myh#<-rBot~Pgc>WK+iHgBN?x-Oz}Lt zHLZdu-?L^^Do46aSJm@b*`Gj7IZaFQor9a|`NU+>f{NtRCN^M0(_oCu(k*T(p6C{j zxtLPcN=>{%G=%h{;+b%s&ZmOjwJHyF7jn5OMI`mpZTO|g;Fp5WBs2etRlbwZ zEZ^~2n&P8Egh@X6+`abY-D_XLr(UF>chjQX+CmlXI+c{~hBj?kwQ%m;)2B|>XJl#} z_4ny@`k4}%MZ7TKgb)WRZSr)Fa1p57g$LQBBL})1%yJiCk{rA_mW$%K6wvr_LfK?U zs-a~UILoDKEv2c_Zi(^fBrf3)KPBASRWOXeg&ZYxSGg(l&9pgACEPG!>`Tus{0N`) z)8LTkLRxcl=jt#fx0)PbrU{rjjavB*DtH2w%ajeB7bLH(hOw&q=dp&k8iB>Ia`f=(vPaNDonm7Agv%Fg2_ zx&$XKy!~swuDaFlVDfci^l!d`(fc<3sS7s+i?FQ#^f4`EDTp-N2Jzw zQ}y7~OQ-=zT8HZSOfrQDr${S49a8#k7^03kxSFO~P2@tO6beVgd# z;-=_4D*D;DLis-Lf$vYBHfhGp8I#q6Z0=%<1gG(@a^FlWLQW8ih z`1DJmrOxI;h3*(eWmnKyx}1{L2*2@l;(5z$H-F=sUkyH=UHHy{hUgLAx$4kP3+Aq* zrN$>4Ul-M-GRC7aKCxGkDN=c4p01zxOye`cozZPX9wFMory(e-N)HZD`6%CcECo?z zGCF*QHr6edd^(>&MVXC(-m3CzR#Ld9$42_Mno~lp>{+Y1*YlYw8msDPRiN;@BSJ`% z%%m|PbSJP3%RD~LGObFiXy#hddAAW?CzaNu{9R=Eemb8JC8Wyqs&KCi(X1SWgl>*8 zby0Z_KD93`*1oiV+nT2zpL@g9NjJ{Aak7~`CQrO!$|NKtoG48`MX&)*yYL18z>*l8XQe-QKM%9UvB9uZreV_WxJMcN`r5^{M#xEV(vEra5o{jt3Zg$`0$9Jth zQW`vq46aR8{5U?PQ`*dv5n-`fqjHq?rzw{$6<7FFe-Ai|geIJ!rAhC}vXDovlKG9AN37iy~kkhl425 zQ<{rtghUsv64ZniB-x&?BBLRcJaW)tHxfWKdWVCP0}@RZIy}#8(+P!y)DOf7@g@X( z8UTQlGgvG%+H^Tigp(+n$j!H!`0D06?o__xb3yWXaQpHD2G#h=V@KhWIjTYwGyTRm z`Hw5vmG4xvp3kIIn0)$*0isDM0@dG-H?2Ie*Z7kV&y-N)RQf4mNJbeo6!qeY&s5Gz zKSPc8tV@A~O>p9iHqDPmwkD+zt>aV49-ZldX!H}EYLU}z&;8Z=Y_#~vIW$Be{oNvXa&Q1MCJ~!TPJl>9V&p-F% zgo)Q_St@)^oIE+cVTWasnQ^#+6Pdz9H}Sv{T7*Vus4Fozu|ysf(8(Bhq=*-Sz6xw7 zfhz+Qmg1Qdo&|Rzm!)}vnr0_6o|j&qEi+g8 ziO)oo?xjK=+q2?C3!JU^3G(K5oHR%^2|C!VXovXt*}hY2cmg=krJr1pKB z>sD9Cr^Y4ipL~|=t@x}8_d+Z~lsOuoa~iW$>`xLJ`niAWO8WV;XC9q2>AGnu@66Ci z=G=~NCpoZXn4l(>1>5LgN<7fQ!2?t>LzF6l6IXcSfS0^wCvL=LW&AJ5d?}Shf+j1Dd`;RM}g1TH_tMRbi%dxoNr9p!7Z;_`EmcY zWryOs!ez&zh+laqCI~v^>-iLiX!0o{8$=V-1h}40&3*K<@EMbn6h2p$@;yuW6w)aA z!$;SAjZg3Wy)3@tGt#KDr+&uV5iw8cDuXw`$umBY6Q-aN_DHB`@|ikT@kz0L&5`=q zCZEwU(PDyDd@?xmCzIj{&iHt+{DP8C^cFsKW+k8Tp;H>tBvf@i;Z#e?wX>~Yh`lBa zN)4}PWY_vvd7s7cSz&MC^L6%T;d8P6cl!DBXCJ@rIx`Z-T!=uOJn1?^n3V9zCqj}> zT37EFmR>zL6E<9dN+KauCvq$mRj&u64EA!#WR+hE-g2*9T%zkdjv|8^2h`kM!Y(;Q z?4r5~cbllI=de&2QYpQjO!~~ZF`o$B{*~J%TvPaLUVn7^5`0<+(!5ppJh4ORyy{@f zVrTh#7xLM*T;iuH@5oF7pKPXM>fi8ss)d1~>y z=w{OC8Fs0TFk?rLZFiIPeD2x!TJkBfYyB&`n4pDEp_-A(d(qEDd+=E+-$Ort`lE#t zCyvMG4L3{?DSXa=L1?JKOjD=AGR0H<+PfB}D2ZbRW*7quitvL;85owRwu?ooLS#&P zWtxHzes+m)juMyRK&mU7lQ;vr)Ky4m{sgM?yQp*=ir^59Hbs(85u2Hk5!vMNC$LV; z!F&C<5ib<|G(I_yQ@)#5ZR^tGLFvl)M60ydF_xI?LSEaQ#1cO>UwSHM2}5u33DgQs z5o(Odv_D(%DY1gH;#MMG8|t>7IT}T3@)?!o{TtrcW5N2BngZTCvh$zb+W&`l5B&aD z`+xg(^Y7l-C(euf?(Kd5!~2K+_};b)OZm_vY~6#!2CfzK(KV2w z8uj{vn?Eqi;>Yue(%l?+ECqpA6> z8(a8nUXRZMjY|#$S4$6XS#s1IR%P*V9HJ4N>1|?9sATuuK~=sZGyS10H9j3)u#B2f z@j6UHGwb05K&6%$&rR|4Edmu7C`n_6{4in-Cyth)!#kF}b9l@8CBInw!n13ZytroB zi>sHsuxjxyRxf>F&C-{ID_(#8=RaQf!gEi&@Y5%r|H)(X=6q+)5AJyI?mO@M-rp~r z`~8=G`q)4I=;5U={pkG@yM4^A_&Bm-xf@76&4Bu*)pqu-j}`P-p=Cr0={{BHmj>9d zlgrW!F)35a$IkH!DnnJX&}V-9#@NvoK680uOs0M&oC^2dAJtC{N=7d2+PGwQjIUEg zT5LGepAlO`464CUmY#i`CHwR`vO-MxGFo;`bZ>t^ZZ zt{LglrAwzyojMnnk&7?BSVSI~J9g~Yp+g5dd-UjGR5SKSF@91uap)2XakL{D@FLJc zG9cxt^ckWQ5`Ic*VG@{O0$C;6If0{%+<{xN*Qxj9IOh28^WKx*oDbnK6 z+w$?nKE3m?$bv2{ul{5cIF}x4TD@!Yk}*U3^|<)_L4A7-8gR+L{=EnE>ou_dC4>5B zy!ZKEJg>tAU%2Rk&-0x+e6dT%^E-Cdz^Kck6pANel8Uu46@THgWs*rL43(Jw2cPU-`l%34EQPR)`c&y%vWd?a zTXNK#V~yaf_)J7mN<5YCt@t!1MC0eq4NG@7XkUu)b())w?%lk4>C5BCkDWSw>h)8m zOfWEW+Jd{v~97+f<4x(k<``*2J^%94uNbV9(XR}YAKFP2=Cc{y0GFd2G^?Ra` z%o1Lw6{ol0r*8m5qat3RpKuaPpfW@WAQU-yQWZt8mvJw5Bi z>a|x7f1&WHeq=x_4T(=u1S*|U4Eo+PDvFX?@HU3O`& zUOhWq_@(p4j<|gJYyWt(dD8(^xt%NCEPY#QVOf7~shSziwDF|5%1zH}Wm5D8pSn|{ zQWkuc5mO9K9-MA6i+GJ-$){`yr?NY%Mc_<6A&O5mqb%P|ZnI}Y3|`uWQhfSo&Rf9e zo(-#(zI4sCS5KLC!+1?gQzlQC6zxkij)v?LIk2Q%~7Q}2T@^`=0dUwMc^f$DU8XXF!Zq~q%dhL z2@%kIWO9CAu4Pku#O&EKuNgh;`KKRwqiOAd#+cmC(>vhntu2d>_>3#&oD9v}x9PS0 zn<2_CN3K;K@M^z72L zbLXy|x_0l zpD+|525GB&h5cE(x6n_+3GyS>?6hT+#^SlDp9Xk?Fe*f)uM?jsp>9=THq!cth9Qc~ z%BOx7GHd!-&!=Ha{OeL%_e_byy&6}f!DiFuYs2j7%-@Rk6zuOh{+ZpG1&>F z!cJW~cI@2g;*LbN>xHZ?)OPFEsY~asT|4*e-l<2|3%>o0+0ENlnMfIaA)DFGps4|$ zrGZPLzbK|qi@DHE_vbU@7{8RZ(*$ z>(gvpVg7lta5(A(%ih|*dEN4tCtR<5*Y$qG#3_^1dsvB@&Oy#>2CMXU=D8E@7NditTQvgXng`|~WCW^^n1u7<`aPSic zUXq!iL@D63DdcQoJIs}6h?qWZ)Rpv8{Z#o5Kdm3Wem%5h(c#9|4>T_FFe#^q#E?*+ zHZBq-nK^I~tDYXClnv0LqR|GGMSNv{;#0Yq;H4;e_25(!PYFFGR&DWlJUP)cx=W*z zulC|Q(;b_ZTr=j%!skWc>_4dA;481_bD1@-dtKVM4;Ag+vl~xByY=V_S|Q)Pdz5iq zVcEG0P&<2XJ-XZ5yLY!fy}R}3a^b9L*KS?Eo)RM;}4?-U$C7&5esWNz^$V^|^pN!8e)`fhG1}Qbry)g@DOr9K8)`r*o z@J}{xTDACv>&K6oHsgkglP8S7?%D|xqGGO0hav|yAr&oKQMe<}-z$`2GtE&EE)!*z zXp>qpGU+G#lYTlUTU2cYqT~^Zt!P9S4mtQ$<#ldK5s{c2W<(r#2u!K8$zd`RGr~%- zS-Pu?yPh_7Tc9F7JIwFU2}UFS#^8?bW+yw;tWFia>F?!v0JXw7j@L7N0$P z_v+ajqTMg+)1!Cyi)Tz8x9G)Z-)L^o&acJQjLL785hi+q-1eKzivAcj@W!5%ni2Gz zR`|KM_-M}5xDF{Fh6JvqxVMXzR%E7fR+Xc|XH2VD@#)#dLoO51@7=t3cT}KWJ-ln( zir0TJal*LCHz?dSA6S&TE>&ZWL3O!vWYtcfmNaiK2`Y@@!FlpCjFFW z(4awBB8&=a#V+J*Rt{a6l5pD0ND`RY#DhZiWO0dNwOJ)fugGMmsEAFTJjvJnSC1I{ z^QRuBpNFov|qYmk<#&lVqo`kgP z#b2Cz_c!-8Ez6!b{XQOAczlDPV8P0LkN6Zr}w{%TQ;dksAXAt zDLR{);%?R7C1!?7JVjDJqh)^mE8=CeQmyldekzrW7}*`fF50#JwVmr;-MRj?L%TO_ zZCGXEkjXbpBA$Z=_I0EFOn+L!DdJD|w|fqdv#WAa;q6|pO+O9->b&bZKKIzVz&oM09vnJOC0qWx;ZgzKkGn=jkR%n**&Gd7_D@_|hLfhn1`%>`JvQTfbdODvW zM5RYYC|>!>{-n3@v0iweEUi?;M_s;*Nv59%wl96-V8e?)eY9)GFLk)^OWx0Am-Y38 zk$K^T7k2E_(F6B{nOj~#^&-l<2p&^NdqzdRXRqG9x^(TLhNW!eFYVJy;nwf6UR^qT z;kH>5cWqt$#=#xO_HB9d;MOC%*EQ<}_Vs#n{w|9yq~`TTG)0xI40CdObW%@!_9$6^ zkL6i?B`sU>nWcR4nf0SIJL~l`e)XG|?Aows*T$vW*PC2%^}*d6Hm-W@(fJRI95rIZ zsFD2!^w&~xNw4@N`n%|%zZaYc^c*y~Q$kLpO@~7?LNNjMxG*WFIy*zOfB*ihQ2Zop z@Ka=0fQeK<%43tuNTw%v^`Y~~lS~Rr2_9_@oH&P1%EqN<_nUjZbDmKgp&V zL@Pek-|<#o#8-T>LMtXEW;`<&IIY^S{K&55M`Krw&(dFBV$1Oux@mX{-40~QY;Z({OZZ#-H7pG=D`q}K`Bev+)l}%e# zZ(Q}tJ>S0l>$grDGh%Sx%X%>-*&>=Cx~txGiC?0>RLG4OF~Z-g!g7Cc_uY4&{z^`y zN)eeeXO2V>N|ire$70gqoPr*ri4&GY@Ix5IlPICYicIm`Odw@TCtH2Q-aOpd=wv`y zhDE`W!zbr{IPb~JnlNGf$e{y&{=>Pdq9DYWhjt+NeD%Plm-cRanSLgpLIwT)&AgVS z;InDN%SxY*N@z@9nZ{@GnN_4xmp5jw42vp=VlUNFok(>9pEhe%r;qY!B)s%P`i&K3K&uRgGIJ?$)(quSe%V)tZIk!Sc(K0`c}<_$}BuD5K*(nH%<@87y==f)*l*1i7VJ$KBS zI)3z3gSvJ6Qnya$ckZMmt&3J6ZDcBT9XodD+NBe~1`ZtPoqE3*pkS)lW^qsV$RnrL z9hg_#{Q2`Gn(zIw#~!0pe$mb#q`fRpj*w#2t`^V&QD+7fz>+I^Xs>t*Y=%-YLq;kZ z^N{M;Qiw-8e^Q#7Nj5pMIsUrwqpuz_dd%qY}*!1Z&npje}8>{ zLs7^E_U9`FOX3+3^zPlLe1Cb*`Wl}pp+!aU8NHl2a*2M{@fpocM~#d!va}$2I-e=6 zsh<_DH5J{t;&6N(6%&sTC?zJVl?7-ti}vcDJ~jWcUR}F(>)5k*w@WVV)vkseVd|=1M##s@YDl}C}H0haiI zW!9s`HA}*eGo8B-sAX#|Jf$R@4?OU|f&~i{@yVw)DCPm#WW%c*u@;=%nSqDQ3=x&c zNvWL6aW<pa03i z6ZZcDT_og8q``5m6$h_Ic!Zqfnm%dvsK8DG8xiqI`o|*W}Cq@VP z#%KQ~ec2Yyu=mi8wQnBYGivzY&wuW`PMt39(nYUv_s*R=U3Ad}iuemJykOL*kqZ{g zfAB%8J$fgx@YB3U<~{t#!{J+;9h^3|;lqbB$yi?;(<_TPZ%Wab7Mk{$UOiPY{M)T- zmy5e~?R1I8tFE2D`S-Uybnl%_=KC_p|3Krfj_r8s$W}v8{0TZVe{;etrdvMneX%JP z4{Y!Y^(i-7nyDXM>RVSH*tz!D-p1XHs~5fa+;dMYoIPW54je}@ZbEltM8Az`-H@vd1@zsMiw`;Wb@wHZZt)Q$tJ5+6DWEujz%Q)q^}hoF%)0VkuQ&SNr^d^q7aNunUyD+R>lCi=GE4}R>Ui> zwb-a`obqLa=Np=pqFDDyvboEm%iFZEu^85YhC{~Z#~SW;;qk%?zhq!TG`Dxv1dq>) zSf2pZPnU{O2?!KZNvbekA3nWte0n2}N(WSl8UslRJ?Y(BBTTe=mpT`4E)hpgH1K!) z;^2Y3zkd6T3+LXod&}zGjmzJEqv_Wto8CXZLkpE3pg;4ygBz`;)VyV>8rH!bYbd{t zfL)uGzxw=-o_T!kH@-ZpZ||-h&i{PRZXNsd?AizUS`?y*vb#UZ72SZ_@FPl1BUZ3U z)e`4$B9ti9VwpxJJCj3Wy-Pl&K$QHJtMqUcKSNBq5dOLRA>`#il{6+=HWSFCt|Cxo zyxGf1w8_aMkrPQEbJH$Bf)kT|CLA$$KMa+tt{OgidzlPuw(h*ol8~K z4{Un*z$W1<%^P28TL0qS4KMDux%riS4KMH8_)GETjlYz>XWa{X*1g!YPM(+cZmHvNxOJ+j9*y z$7&feTjW3Lz-^0x zPq5)IUxb$;4pfoU7Gb_M5lwtDC!d&1GEpinJwt|>ii0GozG2b$tA~8)i|2Lhbb-R1Bc2H-N~tKKlP4d>)B7p{GY6lf z36{yEFtJQBD?~*&Z8G(shW>Pqei?16ml=rCt6T5xosEyWyl;;glgB=>;NF*?e{x5| zlD93Dcy!yncis8zJ8r*r+^}I+_P^--&!7LL&tbanrP{Sfr6F(zv;>*X#vb;*yvH;w z)3$^q{D9iZAyYO^y10&>>NhCjZ%+rvEJ5-Kus>bGuONznGKz_2D&q0!4@`5^PXJ)a zC%{q|6Uf{Y&&}K=F`dMU)dV#u1th@~Ca5W%m2%ejR2LXNbl4RGZog^D<~6S!ZQf{R z+QVT(u2gQqV4uNtg^B|{>fB*kfTjDl@r=*KO)+lPplSSEIO3!&3@Zy*1@KCh^H(Qk<(I&V%0EOkzDo5k)_7^Bym~w699AE=2A9 z{<-YZOZr`QN!QK{;V)eH#q$RA?LKbwkXvt>^4v2Ed-XK1%8%&$E?qkG>ea1#kIu=Y zE=GM=4BSilTsmmb6+?$z<;8)s=uHIT`DQbP=41wm;<=gcT?&jfG3oJ1C!Wrx{z@)4 zZ@>L^3aR;&^EZH#*(n8|nT}HA_+vRFQ#K)I(}T)QaS@xjOFUhsXFMKj`Rxlmx5|0Q z`D{I2&hsQG+`vJXUvb6d{rg>d%gxh&{M7uX9)IAur{+Gr@cyS4-2c@4`<|S4&lCKD zdw;m_-XA`8-&2p?_rnGE@J}w3{)5Nn-Tm0ZKX~-v9|#|RJgF1@Px2< zyB?qW%wrGy_|b=+;UDAYKJ)m)KYncPvyVUY^QY(k^vQ>xdwSkO_kVZLfIb~MoNtte zvWbp`l$BvQ1c5>2Z#R0uOYtH&FQs}l;`Q&&+oy{8a6~fhakYx0F`r>(uqh%`<)@z6 zPl~Z#iu)K9tU9C8-Ax_5Q^$)gyx@!d`}Vow3MJZ_NdMtO1`ZuO;7Wc-|0}QTcjb_NL$B;Vq=XvAY>E#VJRtp*L!=CB@dJhq87SSc zL$8!rcEx!jy*it74jJ5EhDemt*{lSTzcloy>akquKpxvj-PhupD#f_v_CdF zpvPRT$5w0osFu)0OL{G(_Dnw2TXC&+`M5<^`PHLZ{K(N)kI4AQv7<(e9VHxj^=RJh zee6hxkKTHk@vXv-W~ptp%5$d8a`oujuX!DG>I);sjCI4p5u?YB5b~#;Bo9#UINhI? zw=SXnhU=0)AzFV_75{`|>ULL$)JL>XwbRv|=@X-442mdzJ)SlX zJ}uXo6RI*txmNk=v$WcEj+<>hr9VgF zxUr+h1tEs<#OR&yL09FB9%;D%n{m^sDI?h zPido$w@-+chlj(lEl*KoT%EsW?6{i$7`)Dg&rj*EJg3w6bI$4bGag*tazO}?wI>VC z@r6(631ORzoim?*0?eL3J)$i?grFe+{Zsk;1k}!IcX@cDTj2C8>);%B?bw>XPUK_w zY$N%MukNpBXB&U{$1q=VvYjeBs?9_Hw8S=_<0qb?&7;nocve__x(C&l?`@trmcaPL zzg>ni|D~Uay>0%PZQgmCIonJ;$HUuy&NeS})`@MR@E4nS)_>+_>%en_q0Ph3G4WG; zfPb-co%@Zo`H)*ZI(b^9{B>-$`L};uqCdHM4prpO@f^HUr_DE8McVnI+q|Dv$Ns73 zY<2j*^JbfWQGH_dzUmV{bMfl0sE_LX`y9*5MQ;q>@t@qzzY@A>pcB&Gg}ZDyzv z>vNttULEp@kv7lM{-fH&rp?3K%-R0LHqUvEiEaMv|2-$R`8-=akT&z2A+gorxp{{C zt#-9}c>5FE57;^8Y<1i1+-(0J^0%Dre*zwUo0;4BQreTP&GWZEF+af&wz?JwTcw=E z=EvQJQ&qd_<5l7rB5mHr8S?-4xvR~m^d}{HZq=))B4_t7(Uk{Quw_d9<|oG6fAj6X z$u`f~{==)B_2;P){}wsJ6R3Y;t>R}m>MVABB3I{H z@wY=hl_zj6q@CTdpLpB#U{vu>JnB==P+j`tBA@yd|GH0jcC?(uQT31U91pKQ)n_i= zCb#YXE80B#9248Tdi&4W=HYE7p2d6rEFRM4-~Jg&Z0D+&JL`Ks>!hJw5MiRnScixK3HuJPPq<-@pUu*rTs`xpcv(@2w+E$5wt((;|{OgXi z|Mj)`9NK^QIp+Ltd-Zd?`9H_uZ9b4T6Wi3KHWSb8@HUUC68{{3``^xa)=!{%FZGf7 zv(=}cOZ-y=tUA@X+|S1yTmQHJE&kv93AFi|THRHfc`~t8&fIJ>e+AXKN7_7p`xD!| z=f5R>rk{Yf-{$?dpNdvFQQ9iyPug@dnZN$7q||SIV!X{Oe&SJo%kb~{1loK@?N4m~ zt^PUA+5W3nIqPquO8i^oFZK!4zpqyDv%Y7uZJ+L1ojfd{(K!NEe|VemPwPJacAkHSC-8BwRq1WsL$$X)vg*9gombq-hc3G9|McG% ze!23YPEW4<`pVb;?ujSwd2-ImhyKIR)zdQ1d0*}S;JE+yfQyeSDJ(W*8h(y!ENU~x$@rI&VRAxCOSWP z&l8XQ;Ql#J-1qq0N1vR#aKXygzy9dl`yLta!$$`^`OusJ_s+Y=#=Q%lo;Tp$IZr-4 tXU>9>{OCNlKH#1O_m{25@0ma6k+}=XwmtXF88G+B0SoJLuKeMs{~wXBY#{&u literal 0 HcmV?d00001 diff --git a/experiments/robot/openvla_utils.py b/experiments/robot/openvla_utils.py index e12e9a2f2..30d63b884 100644 --- a/experiments/robot/openvla_utils.py +++ b/experiments/robot/openvla_utils.py @@ -1,48 +1,287 @@ -"""Utils for evaluating the OpenVLA policy.""" +"""Utils for evaluating OpenVLA or fine-tuned OpenVLA policies.""" +import filecmp import json import os +import shutil import time +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union +import json_numpy import numpy as np +import requests import tensorflow as tf import torch +from huggingface_hub import HfApi, hf_hub_download from PIL import Image from transformers import AutoConfig, AutoImageProcessor, AutoModelForVision2Seq, AutoProcessor +# Apply JSON numpy patch for serialization +json_numpy.patch() + from prismatic.extern.hf.configuration_prismatic import OpenVLAConfig from prismatic.extern.hf.modeling_prismatic import OpenVLAForActionPrediction from prismatic.extern.hf.processing_prismatic import PrismaticImageProcessor, PrismaticProcessor +from prismatic.models.action_heads import DiffusionActionHead, L1RegressionActionHead +from prismatic.models.film_vit_wrapper import FiLMedPrismaticVisionBackbone +from prismatic.models.projectors import NoisyActionProjector, ProprioProjector +from prismatic.vla.constants import ( + ACTION_DIM, + ACTION_PROPRIO_NORMALIZATION_TYPE, +) +from prismatic.vla.datasets.rlds.utils.data_utils import NormalizationType -# Initialize important constants and pretty-printing mode in NumPy. -ACTION_DIM = 7 +# Initialize important constants DATE = time.strftime("%Y_%m_%d") DATE_TIME = time.strftime("%Y_%m_%d-%H_%M_%S") DEVICE = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") +OPENVLA_IMAGE_SIZE = 224 # Standard image size expected by OpenVLA + +# Configure NumPy print settings np.set_printoptions(formatter={"float": lambda x: "{0:0.3f}".format(x)}) -# Initialize system prompt for OpenVLA v0.1. -OPENVLA_V01_SYSTEM_PROMPT = ( - "A chat between a curious user and an artificial intelligence assistant. " - "The assistant gives helpful, detailed, and polite answers to the user's questions." -) +def model_is_on_hf_hub(model_path: str) -> bool: + """Checks whether a model path points to a model on Hugging Face Hub.""" + # If the API call below runs without error, the model is on the hub + try: + HfApi().model_info(model_path) + return True + except Exception: + return False + + +def update_auto_map(pretrained_checkpoint: str) -> None: + """ + Update the AutoMap configuration in the checkpoint config.json file. + + This loads the config.json file inside the checkpoint directory and overwrites + the AutoConfig and AutoModelForVision2Seq fields to use OpenVLA-specific classes. + + Args: + pretrained_checkpoint: Path to the checkpoint directory + """ + if not os.path.isdir(pretrained_checkpoint): + return + + config_path = os.path.join(pretrained_checkpoint, "config.json") + if not os.path.exists(config_path): + print(f"Warning: No config.json found at {config_path}") + return + + # Create timestamped backup + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = os.path.join(pretrained_checkpoint, f"config.json.back.{timestamp}") + shutil.copy2(config_path, backup_path) + print(f"Created backup of original config at: {os.path.abspath(backup_path)}") + + # Read and update the config + with open(config_path, "r") as f: + config = json.load(f) + + config["auto_map"] = { + "AutoConfig": "configuration_prismatic.OpenVLAConfig", + "AutoModelForVision2Seq": "modeling_prismatic.OpenVLAForActionPrediction", + } + + # Write back the updated config + with open(config_path, "w") as f: + json.dump(config, f, indent=2) + + print(f"Updated config.json at: {os.path.abspath(config_path)}") + print("Changes made:") + print(' - Set AutoConfig to "configuration_prismatic.OpenVLAConfig"') + print(' - Set AutoModelForVision2Seq to "modeling_prismatic.OpenVLAForActionPrediction"') + + +def check_identical_files(path1: Union[str, Path], path2: Union[str, Path]) -> bool: + """ + Check if two files are identical in content. + + Args: + path1: Path to the first file + path2: Path to the second file + + Returns: + bool: True if files are identical, False otherwise + """ + path1, path2 = Path(path1), Path(path2) + + # First check if file sizes match + if path1.stat().st_size != path2.stat().st_size: + return False + + # Check if contents match + return filecmp.cmp(path1, path2, shallow=False) -def get_vla(cfg): - """Loads and returns a VLA model from checkpoint.""" - # Load VLA checkpoint. - print("[*] Instantiating Pretrained VLA model") - print("[*] Loading in BF16 with Flash-Attention Enabled") - # Register OpenVLA model to HF Auto Classes (not needed if the model is on HF Hub) - AutoConfig.register("openvla", OpenVLAConfig) - AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) - AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) - AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) +def _handle_file_sync(curr_filepath: str, checkpoint_filepath: str, file_type: str) -> None: + """ + Handle syncing of files between current directory and checkpoint. + + Creates backups if files exist but differ, and copies current versions to checkpoint. + + Args: + curr_filepath: Path to the current file version + checkpoint_filepath: Path where the file should be in the checkpoint + file_type: Description of the file type for logging + """ + if os.path.exists(checkpoint_filepath): + # Check if existing files are identical + match = check_identical_files(curr_filepath, checkpoint_filepath) + + if not match: + print( + "\n------------------------------------------------------------------------------------------------\n" + f"Found mismatch between:\n" + f"Current: {curr_filepath}\n" + f"Checkpoint: {checkpoint_filepath}\n" + ) + + # Create timestamped backup + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = f"{checkpoint_filepath}.back.{timestamp}" + shutil.copy2(checkpoint_filepath, backup_path) + print(f"Created backup of original checkpoint file at: {os.path.abspath(backup_path)}") + + # Copy current version to checkpoint directory + shutil.copy2(curr_filepath, checkpoint_filepath) + print(f"Copied current version to checkpoint at: {os.path.abspath(checkpoint_filepath)}") + print( + f"Changes complete. The checkpoint will now use the current version of {file_type}" + "\n------------------------------------------------------------------------------------------------\n" + ) + else: + # If file doesn't exist in checkpoint directory, copy it + shutil.copy2(curr_filepath, checkpoint_filepath) + print( + "\n------------------------------------------------------------------------------------------------\n" + f"No {file_type} found in checkpoint directory.\n" + f"Copied current version from: {curr_filepath}\n" + f"To checkpoint location: {os.path.abspath(checkpoint_filepath)}" + "\n------------------------------------------------------------------------------------------------\n" + ) + + +def check_model_logic_mismatch(pretrained_checkpoint: str) -> None: + """ + Check and sync model logic files between current code and checkpoint. + + Handles the relationship between current and checkpoint versions of both + modeling_prismatic.py and configuration_prismatic.py: + - If checkpoint file exists and differs: creates backup and copies current version + - If checkpoint file doesn't exist: copies current version + + Args: + pretrained_checkpoint: Path to the checkpoint directory + """ + if not os.path.isdir(pretrained_checkpoint): + return + + # Find current files + curr_files = {"modeling_prismatic.py": None, "configuration_prismatic.py": None} + + for root, _, files in os.walk("./prismatic/"): + for filename in curr_files.keys(): + if filename in files and curr_files[filename] is None: + curr_files[filename] = os.path.join(root, filename) + + # Check and handle each file + for filename, curr_filepath in curr_files.items(): + if curr_filepath is None: + print(f"WARNING: `{filename}` is not found anywhere in the current directory.") + continue + + checkpoint_filepath = os.path.join(pretrained_checkpoint, filename) + _handle_file_sync(curr_filepath, checkpoint_filepath, filename) + + +def find_checkpoint_file(pretrained_checkpoint: str, file_pattern: str) -> str: + """ + Find a specific checkpoint file matching a pattern. + + Args: + pretrained_checkpoint: Path to the checkpoint directory + file_pattern: String pattern to match in filenames + + Returns: + str: Path to the matching checkpoint file + Raises: + AssertionError: If no files or multiple files match the pattern + """ + assert os.path.isdir(pretrained_checkpoint), f"Checkpoint path must be a directory: {pretrained_checkpoint}" + + checkpoint_files = [] + for filename in os.listdir(pretrained_checkpoint): + if file_pattern in filename and "checkpoint" in filename: + full_path = os.path.join(pretrained_checkpoint, filename) + checkpoint_files.append(full_path) + + assert len(checkpoint_files) == 1, ( + f"Expected exactly 1 {file_pattern} checkpoint but found {len(checkpoint_files)} in directory: {pretrained_checkpoint}" + ) + + return checkpoint_files[0] + + +def load_component_state_dict(checkpoint_path: str) -> Dict[str, torch.Tensor]: + """ + Load a component's state dict from checkpoint and handle DDP prefix if present. + + Args: + checkpoint_path: Path to the checkpoint file + + Returns: + Dict: The processed state dictionary for loading + """ + state_dict = torch.load(checkpoint_path, weights_only=True) + + # If the component was trained with DDP, elements in the state dict have prefix "module." which we must remove + new_state_dict = {} + for k, v in state_dict.items(): + if k.startswith("module."): + new_state_dict[k[7:]] = v + else: + new_state_dict[k] = v + + return new_state_dict + + +def get_vla(cfg: Any) -> torch.nn.Module: + """ + Load and initialize the VLA model from checkpoint. + + Args: + cfg: Configuration object + + Returns: + torch.nn.Module: The initialized VLA model + """ + print("Instantiating pretrained VLA policy...") + + # If loading a locally stored pretrained checkpoint, check whether config or model files + # need to be synced so that any changes the user makes to the VLA modeling code will + # actually go into effect + # If loading a pretrained checkpoint from Hugging Face Hub, we just assume that the policy + # will be used as is, with its original modeling logic + if not model_is_on_hf_hub(cfg.pretrained_checkpoint): + # Register OpenVLA model to HF Auto Classes (not needed if the model is on HF Hub) + AutoConfig.register("openvla", OpenVLAConfig) + AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) + AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) + AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) + + # Update config.json and sync model files + update_auto_map(cfg.pretrained_checkpoint) + check_model_logic_mismatch(cfg.pretrained_checkpoint) + + # Load the model vla = AutoModelForVision2Seq.from_pretrained( cfg.pretrained_checkpoint, - attn_implementation="flash_attention_2", + # attn_implementation="flash_attention_2", torch_dtype=torch.bfloat16, load_in_8bit=cfg.load_in_8bit, load_in_4bit=cfg.load_in_4bit, @@ -50,14 +289,82 @@ def get_vla(cfg): trust_remote_code=True, ) - # Move model to device. - # Note: `.to()` is not supported for 8-bit or 4-bit bitsandbytes models, but the model will - # already be set to the right devices and casted to the correct dtype upon loading. + # If using FiLM, wrap the vision backbone to allow for infusion of language inputs + if cfg.use_film: + vla = _apply_film_to_vla(vla, cfg) + + # Set number of images in model input + vla.vision_backbone.set_num_images_in_input(cfg.num_images_in_input) + + vla.eval() + + # Move model to device if not using quantization if not cfg.load_in_8bit and not cfg.load_in_4bit: vla = vla.to(DEVICE) - # Load dataset stats used during finetuning (for action un-normalization). - dataset_statistics_path = os.path.join(cfg.pretrained_checkpoint, "dataset_statistics.json") + # Load dataset stats for action normalization + _load_dataset_stats(vla, cfg.pretrained_checkpoint) + + return vla + + +def _apply_film_to_vla(vla: torch.nn.Module, cfg: Any) -> torch.nn.Module: + """ + Apply FiLM (Feature-wise Linear Modulation) to the VLA vision backbone. + + Args: + vla: The VLA model + cfg: Configuration object with model parameters + + Returns: + torch.nn.Module: VLA model with FiLM applied + """ + from peft import LoraConfig, get_peft_model + + # Apply LoRA configuration + lora_config = LoraConfig( + r=32, + lora_alpha=16, + lora_dropout=0.0, + target_modules="all-linear", + init_lora_weights="gaussian", + ) + vla = get_peft_model(vla, lora_config) + + # Create and apply FiLMed vision backbone + new_vision_backbone = FiLMedPrismaticVisionBackbone( + vision_backbone=vla.vision_backbone, llm_dim=vla.llm_dim, + ) + vla.model.vision_backbone = new_vision_backbone + + # Load vision backbone checkpoint + checkpoint_path = find_checkpoint_file(cfg.pretrained_checkpoint, "vision_backbone") + state_dict = torch.load(checkpoint_path, weights_only=True) + vla.model.vision_backbone.load_state_dict(state_dict) + + # Use the model component instead of wrapper and convert to bfloat16 + vla = vla.model + vla.vision_backbone = vla.vision_backbone.to(torch.bfloat16) + + return vla + + +def _load_dataset_stats(vla: torch.nn.Module, checkpoint_path: str) -> None: + """ + Load dataset statistics used during training for action normalization. + + Args: + vla: The VLA model + checkpoint_path: Path to the checkpoint directory + """ + if model_is_on_hf_hub(checkpoint_path): + # Download dataset stats directly from HF Hub + dataset_statistics_path = hf_hub_download( + repo_id=checkpoint_path, + filename="dataset_statistics.json", + ) + else: + dataset_statistics_path = os.path.join(checkpoint_path, "dataset_statistics.json") if os.path.isfile(dataset_statistics_path): with open(dataset_statistics_path, "r") as f: norm_stats = json.load(f) @@ -69,39 +376,195 @@ def get_vla(cfg): "Otherwise, you may run into errors when trying to call `predict_action()` due to an absent `unnorm_key`." ) - return vla + +def get_processor(cfg: Any) -> AutoProcessor: + """ + Get the VLA model's Hugging Face processor. + + Args: + cfg: Configuration object with model parameters + + Returns: + AutoProcessor: The model's processor + """ + return AutoProcessor.from_pretrained(cfg.pretrained_checkpoint, trust_remote_code=True) + + +def get_proprio_projector(cfg: Any, llm_dim: int, proprio_dim: int) -> ProprioProjector: + """ + Get proprioception projector for the VLA model. + + Args: + cfg: Configuration object with model parameters + llm_dim: Dimension of the language model + proprio_dim: Dimension of proprioception data + + Returns: + ProprioProjector: The initialized proprio projector + """ + # Initialize projector and move to device + proprio_projector = ProprioProjector( + llm_dim=llm_dim, + proprio_dim=proprio_dim, + ).to(DEVICE) + proprio_projector = proprio_projector.to(torch.bfloat16).to(DEVICE) + proprio_projector.eval() + + # Find and load checkpoint (may be on Hugging Face Hub or stored locally) + if model_is_on_hf_hub(cfg.pretrained_checkpoint): + model_path_to_proprio_projector_name = { + "moojink/openvla-7b-oft-finetuned-libero-spatial": "proprio_projector--150000_checkpoint.pt", + "moojink/openvla-7b-oft-finetuned-libero-object": "proprio_projector--150000_checkpoint.pt", + "moojink/openvla-7b-oft-finetuned-libero-goal": "proprio_projector--50000_checkpoint.pt", + "moojink/openvla-7b-oft-finetuned-libero-10": "proprio_projector--150000_checkpoint.pt", + } + if cfg.pretrained_checkpoint not in model_path_to_proprio_projector_name.keys(): + raise ValueError("Unsupported HF Hub pretrained checkpoint found!") + # Download proprio projector directly from HF Hub + proprio_projector_path = hf_hub_download( + repo_id=cfg.pretrained_checkpoint, filename=model_path_to_proprio_projector_name[cfg.pretrained_checkpoint] + ) + state_dict = load_component_state_dict(proprio_projector_path) + proprio_projector.load_state_dict(state_dict) + else: + checkpoint_path = find_checkpoint_file(cfg.pretrained_checkpoint, "proprio_projector") + state_dict = load_component_state_dict(checkpoint_path) + proprio_projector.load_state_dict(state_dict) + + return proprio_projector -def get_processor(cfg): - """Get VLA model's Hugging Face processor.""" - processor = AutoProcessor.from_pretrained(cfg.pretrained_checkpoint, trust_remote_code=True) - return processor +def get_noisy_action_projector(cfg: Any, llm_dim: int) -> NoisyActionProjector: + """ + Get noisy action projector for diffusion-based action prediction. + Args: + cfg: Configuration object with model parameters + llm_dim: Dimension of the language model -def crop_and_resize(image, crop_scale, batch_size): + Returns: + NoisyActionProjector: The initialized noisy action projector """ - Center-crops an image to have area `crop_scale` * (original image area), and then resizes back - to original size. We use the same logic seen in the `dlimp` RLDS datasets wrapper to avoid - distribution shift at test time. + # Initialize projector and move to device + noisy_action_projector = NoisyActionProjector( + llm_dim=llm_dim, + ).to(DEVICE) + noisy_action_projector = noisy_action_projector.to(torch.bfloat16).to(DEVICE) + noisy_action_projector.eval() + + # Find and load checkpoint + checkpoint_path = find_checkpoint_file(cfg.pretrained_checkpoint, "noisy_action_projector") + state_dict = load_component_state_dict(checkpoint_path) + noisy_action_projector.load_state_dict(state_dict) + + return noisy_action_projector + + +def get_action_head(cfg: Any, llm_dim: int) -> Union[L1RegressionActionHead, DiffusionActionHead]: + """ + Get action head for continuous value prediction. Args: - image: TF Tensor of shape (batch_size, H, W, C) or (H, W, C) and datatype tf.float32 with - values between [0,1]. - crop_scale: The area of the center crop with respect to the original image. - batch_size: Batch size. + cfg: Configuration object with model parameters + llm_dim: Dimension of the language model + + Returns: + Union[L1RegressionActionHead, DiffusionActionHead]: The initialized action head + + Raises: + AssertionError: If both L1 regression and diffusion are specified """ - # Convert from 3D Tensor (H, W, C) to 4D Tensor (batch_size, H, W, C) - assert image.shape.ndims == 3 or image.shape.ndims == 4 + assert not (cfg.use_l1_regression and cfg.use_diffusion), "Cannot use both L1 regression and diffusion action head!" + + # Initialize appropriate action head based on configuration + if cfg.use_l1_regression: + action_head = L1RegressionActionHead(input_dim=llm_dim, hidden_dim=llm_dim, action_dim=ACTION_DIM) + elif cfg.use_diffusion: + action_head = DiffusionActionHead( + input_dim=llm_dim, hidden_dim=llm_dim, action_dim=ACTION_DIM, num_diffusion_steps=cfg.num_diffusion_steps + ) + else: + raise ValueError("Either use_l1_regression or use_diffusion must be True") + + action_head = action_head.to(torch.bfloat16).to(DEVICE) + action_head.eval() + + # Find and load checkpoint (may be on Hugging Face Hub or stored locally) + if model_is_on_hf_hub(cfg.pretrained_checkpoint): + model_path_to_action_head_name = { + "moojink/openvla-7b-oft-finetuned-libero-spatial": "action_head--150000_checkpoint.pt", + "moojink/openvla-7b-oft-finetuned-libero-object": "action_head--150000_checkpoint.pt", + "moojink/openvla-7b-oft-finetuned-libero-goal": "action_head--50000_checkpoint.pt", + "moojink/openvla-7b-oft-finetuned-libero-10": "action_head--150000_checkpoint.pt", + } + if cfg.pretrained_checkpoint not in model_path_to_action_head_name.keys(): + raise ValueError("Unsupported HF Hub pretrained checkpoint found!") + # Download proprio projector directly from HF Hub + action_head_path = hf_hub_download( + repo_id=cfg.pretrained_checkpoint, filename=model_path_to_action_head_name[cfg.pretrained_checkpoint] + ) + state_dict = load_component_state_dict(action_head_path) + action_head.load_state_dict(state_dict) + else: + checkpoint_path = find_checkpoint_file(cfg.pretrained_checkpoint, "action_head") + state_dict = load_component_state_dict(checkpoint_path) + action_head.load_state_dict(state_dict) + + return action_head + + +def resize_image_for_policy(img: np.ndarray, resize_size: Union[int, Tuple[int, int]]) -> np.ndarray: + """ + Resize an image to match the policy's expected input size. + + Uses the same resizing scheme as in the training data pipeline for distribution matching. + + Args: + img: Numpy array containing the image + resize_size: Target size as int (square) or (height, width) tuple + + Returns: + np.ndarray: The resized image + """ + assert isinstance(resize_size, int) or isinstance(resize_size, tuple) + if isinstance(resize_size, int): + resize_size = (resize_size, resize_size) + + # Resize using the same pipeline as in RLDS dataset builder + img = tf.image.encode_jpeg(img) # Encode as JPEG + img = tf.io.decode_image(img, expand_animations=False, dtype=tf.uint8) # Decode back + img = tf.image.resize(img, resize_size, method="lanczos3", antialias=True) + img = tf.cast(tf.clip_by_value(tf.round(img), 0, 255), tf.uint8) + + return img.numpy() + + +def crop_and_resize(image: tf.Tensor, crop_scale: float, batch_size: int) -> tf.Tensor: + """ + Center-crop an image and resize it back to original dimensions. + + Uses the same logic as in the training data pipeline for distribution matching. + + Args: + image: TF Tensor of shape (batch_size, H, W, C) or (H, W, C) with values in [0,1] + crop_scale: Area of center crop relative to original image + batch_size: Batch size + + Returns: + tf.Tensor: The cropped and resized image + """ + # Handle 3D inputs by adding batch dimension if needed + assert image.shape.ndims in (3, 4), "Image must be 3D or 4D tensor" expanded_dims = False if image.shape.ndims == 3: image = tf.expand_dims(image, axis=0) expanded_dims = True - # Get height and width of crop + # Calculate crop dimensions (note: we use sqrt(crop_scale) for h/w) new_heights = tf.reshape(tf.clip_by_value(tf.sqrt(crop_scale), 0, 1), shape=(batch_size,)) new_widths = tf.reshape(tf.clip_by_value(tf.sqrt(crop_scale), 0, 1), shape=(batch_size,)) - # Get bounding box representing crop + # Create bounding box for the crop height_offsets = (1 - new_heights) / 2 width_offsets = (1 - new_widths) / 2 bounding_boxes = tf.stack( @@ -114,57 +577,236 @@ def crop_and_resize(image, crop_scale, batch_size): axis=1, ) - # Crop and then resize back up - image = tf.image.crop_and_resize(image, bounding_boxes, tf.range(batch_size), (224, 224)) + # Apply crop and resize + image = tf.image.crop_and_resize( + image, bounding_boxes, tf.range(batch_size), (OPENVLA_IMAGE_SIZE, OPENVLA_IMAGE_SIZE) + ) - # Convert back to 3D Tensor (H, W, C) + # Remove batch dimension if it was added if expanded_dims: image = image[0] return image -def get_vla_action(vla, processor, base_vla_name, obs, task_label, unnorm_key, center_crop=False): - """Generates an action with the VLA policy.""" - image = Image.fromarray(obs["full_image"]) - image = image.convert("RGB") +def center_crop_image(image: Union[np.ndarray, Image.Image]) -> Image.Image: + """ + Center crop an image to match training data distribution. - # (If trained with image augmentations) Center crop image and then resize back up to original size. - # IMPORTANT: Let's say crop scale == 0.9. To get the new height and width (post-crop), multiply - # the original height and width by sqrt(0.9) -- not 0.9! - if center_crop: - batch_size = 1 - crop_scale = 0.9 + Args: + image: Input image (PIL or numpy array) - # Convert to TF Tensor and record original data type (should be tf.uint8) + Returns: + Image.Image: Cropped PIL Image + """ + batch_size = 1 + crop_scale = 0.9 + + # Convert to TF Tensor if needed + if not isinstance(image, tf.Tensor): image = tf.convert_to_tensor(np.array(image)) - orig_dtype = image.dtype - # Convert to data type tf.float32 and values between [0,1] - image = tf.image.convert_image_dtype(image, tf.float32) + orig_dtype = image.dtype + + # Convert to float32 in range [0,1] + image = tf.image.convert_image_dtype(image, tf.float32) + + # Apply center crop and resize + image = crop_and_resize(image, crop_scale, batch_size) + + # Convert back to original data type + image = tf.clip_by_value(image, 0, 1) + image = tf.image.convert_image_dtype(image, orig_dtype, saturate=True) + + # Convert to PIL Image + return Image.fromarray(image.numpy()).convert("RGB") + + +def check_image_format(image: Any) -> None: + """ + Validate input image format. + + Args: + image: Image to check + + Raises: + AssertionError: If image format is invalid + """ + is_numpy_array = isinstance(image, np.ndarray) + has_correct_shape = len(image.shape) == 3 and image.shape[-1] == 3 + has_correct_dtype = image.dtype == np.uint8 + + assert is_numpy_array and has_correct_shape and has_correct_dtype, ( + "Incorrect image format detected! Make sure that the input image is a " + "numpy array with shape (H, W, 3) and dtype np.uint8!" + ) + + +def normalize_proprio(proprio: np.ndarray, norm_stats: Dict[str, Any]) -> np.ndarray: + """ + Normalize proprioception data to match training distribution. + + Args: + proprio: Raw proprioception data + norm_stats: Normalization statistics + + Returns: + np.ndarray: Normalized proprioception data + """ + if ACTION_PROPRIO_NORMALIZATION_TYPE == NormalizationType.BOUNDS: + mask = norm_stats.get("mask", np.ones_like(norm_stats["min"], dtype=bool)) + proprio_high, proprio_low = np.array(norm_stats["max"]), np.array(norm_stats["min"]) + elif ACTION_PROPRIO_NORMALIZATION_TYPE == NormalizationType.BOUNDS_Q99: + mask = norm_stats.get("mask", np.ones_like(norm_stats["q01"], dtype=bool)) + proprio_high, proprio_low = np.array(norm_stats["q99"]), np.array(norm_stats["q01"]) + else: + raise ValueError("Unsupported action/proprio normalization type detected!") + + normalized_proprio = np.clip( + np.where( + mask, + 2 * (proprio - proprio_low) / (proprio_high - proprio_low + 1e-8) - 1, + proprio, + ), + a_min=-1.0, + a_max=1.0, + ) + + return normalized_proprio + - # Crop and then resize back to original size - image = crop_and_resize(image, crop_scale, batch_size) +def prepare_images_for_vla(images: List[np.ndarray], cfg: Any) -> List[Image.Image]: + """ + Prepare images for VLA input by resizing and cropping as needed. + + Args: + images: List of input images as numpy arrays + cfg: Configuration object with parameters + + Returns: + List[Image.Image]: Processed images ready for the model + """ + processed_images = [] + + for image in images: + # Validate format + check_image_format(image) + + # Resize if needed + if image.shape != (OPENVLA_IMAGE_SIZE, OPENVLA_IMAGE_SIZE, 3): + image = resize_image_for_policy(image, OPENVLA_IMAGE_SIZE) - # Convert back to original data type - image = tf.clip_by_value(image, 0, 1) - image = tf.image.convert_image_dtype(image, orig_dtype, saturate=True) + # Convert to PIL image + pil_image = Image.fromarray(image).convert("RGB") - # Convert back to PIL Image - image = Image.fromarray(image.numpy()) - image = image.convert("RGB") + # Apply center crop if configured + if cfg.center_crop: + pil_image = center_crop_image(pil_image) + + processed_images.append(pil_image) + + return processed_images + + +def get_vla_action( + cfg: Any, + vla: torch.nn.Module, + processor: Any, + obs: Dict[str, Any], + task_label: str, + action_head: Optional[torch.nn.Module] = None, + proprio_projector: Optional[torch.nn.Module] = None, + noisy_action_projector: Optional[torch.nn.Module] = None, + use_film: bool = False, +) -> List[np.ndarray]: + """ + Generate action predictions with the VLA policy. + + Args: + cfg: Configuration object with parameters + vla: The VLA model + processor: Model processor for inputs + obs: Observation dictionary + task_label: Text description of the task + action_head: Optional action head for continuous actions + proprio_projector: Optional proprioception projector + noisy_action_projector: Optional noisy action projector for diffusion + use_film: Whether to use FiLM + + Returns: + List[np.ndarray]: Predicted actions + """ + # Collect all input images + all_images = [obs["full_image"]] + if cfg.num_images_in_input > 1: + all_images.extend([obs[k] for k in obs.keys() if "wrist" in k]) + + # Process images + all_images = prepare_images_for_vla(all_images, cfg) + + # Extract primary image and additional images + primary_image = all_images.pop(0) # Build VLA prompt - if "openvla-v01" in base_vla_name: # OpenVLA v0.1 - prompt = ( - f"{OPENVLA_V01_SYSTEM_PROMPT} USER: What action should the robot take to {task_label.lower()}? ASSISTANT:" + prompt = f"In: What action should the robot take to {task_label.lower()}?\nOut:" + + # Process primary image + inputs = processor(prompt, primary_image).to(DEVICE, dtype=torch.bfloat16) + + # Process additional wrist images if any + if all_images: + all_wrist_inputs = [ + processor(prompt, image_wrist).to(DEVICE, dtype=torch.bfloat16) for image_wrist in all_images + ] + # Concatenate all images + primary_pixel_values = inputs["pixel_values"] + all_wrist_pixel_values = [wrist_inputs["pixel_values"] for wrist_inputs in all_wrist_inputs] + inputs["pixel_values"] = torch.cat([primary_pixel_values] + all_wrist_pixel_values, dim=1) + + # Process proprioception data if used + proprio = None + if cfg.use_proprio: + proprio = obs["state"] + proprio_norm_stats = vla.norm_stats[cfg.unnorm_key]["proprio"] + obs["state"] = normalize_proprio(proprio, proprio_norm_stats) + proprio = obs["state"] + + # Generate action + if action_head is None: + # Standard VLA output (single-image inputs, discrete actions) + action, _ = vla.predict_action(**inputs, unnorm_key=cfg.unnorm_key, do_sample=False) + else: + # Custom action head for continuous actions + action, _ = vla.predict_action( + **inputs, + unnorm_key=cfg.unnorm_key, + do_sample=False, + proprio=proprio, + proprio_projector=proprio_projector, + noisy_action_projector=noisy_action_projector, + action_head=action_head, + use_film=use_film, ) - else: # OpenVLA - prompt = f"In: What action should the robot take to {task_label.lower()}?\nOut:" - # Process inputs. - inputs = processor(prompt, image).to(DEVICE, dtype=torch.bfloat16) + # Extract subset of actions for open loop steps + return [action[i] for i in range(min(len(action), cfg.num_open_loop_steps))] + - # Get action. - action = vla.predict_action(**inputs, unnorm_key=unnorm_key, do_sample=False) - return action +def get_action_from_server( + observation: Dict[str, Any], server_endpoint: str = "http://0.0.0.0:8777/act" +) -> Dict[str, Any]: + """ + Get VLA action from remote inference server. + + Args: + observation: Observation data to send to server + server_endpoint: URL of the inference server + + Returns: + Dict[str, Any]: Action response from server + """ + response = requests.post( + server_endpoint, + json=observation, + ) + return response.json() diff --git a/experiments/robot/robot_utils.py b/experiments/robot/robot_utils.py index 10e5289d8..64559e990 100644 --- a/experiments/robot/robot_utils.py +++ b/experiments/robot/robot_utils.py @@ -3,6 +3,7 @@ import os import random import time +from typing import Any, Dict, List, Optional, Union import numpy as np import torch @@ -12,22 +13,35 @@ get_vla_action, ) -# Initialize important constants and pretty-printing mode in NumPy. +# Initialize important constants ACTION_DIM = 7 DATE = time.strftime("%Y_%m_%d") DATE_TIME = time.strftime("%Y_%m_%d-%H_%M_%S") DEVICE = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") + +# Configure NumPy print settings np.set_printoptions(formatter={"float": lambda x: "{0:0.3f}".format(x)}) -# Initialize system prompt for OpenVLA v0.1. +# Initialize system prompt for OpenVLA v0.1 OPENVLA_V01_SYSTEM_PROMPT = ( "A chat between a curious user and an artificial intelligence assistant. " "The assistant gives helpful, detailed, and polite answers to the user's questions." ) +# Model image size configuration +MODEL_IMAGE_SIZES = { + "openvla": 224, + # Add other models as needed +} + -def set_seed_everywhere(seed: int): - """Sets the random seed for Python, NumPy, and PyTorch functions.""" +def set_seed_everywhere(seed: int) -> None: + """ + Set random seed for all random number generators for reproducibility. + + Args: + seed: The random seed to use + """ torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) np.random.seed(seed) @@ -37,66 +51,149 @@ def set_seed_everywhere(seed: int): os.environ["PYTHONHASHSEED"] = str(seed) -def get_model(cfg, wrap_diffusion_policy_for_droid=False): - """Load model for evaluation.""" +def get_model(cfg: Any, wrap_diffusion_policy_for_droid: bool = False) -> torch.nn.Module: + """ + Load and initialize model for evaluation based on configuration. + + Args: + cfg: Configuration object with model parameters + wrap_diffusion_policy_for_droid: Whether to wrap diffusion policy for DROID + + Returns: + torch.nn.Module: The loaded model + + Raises: + ValueError: If model family is not supported + """ if cfg.model_family == "openvla": model = get_vla(cfg) else: - raise ValueError("Unexpected `model_family` found in config.") + raise ValueError(f"Unsupported model family: {cfg.model_family}") + print(f"Loaded model: {type(model)}") return model -def get_image_resize_size(cfg): +def get_image_resize_size(cfg: Any) -> Union[int, tuple]: """ - Gets image resize size for a model class. - If `resize_size` is an int, then the resized image will be a square. - Else, the image will be a rectangle. - """ - if cfg.model_family == "openvla": - resize_size = 224 - else: - raise ValueError("Unexpected `model_family` found in config.") - return resize_size + Get image resize dimensions for a specific model. + If returned value is an int, the resized image will be a square. + If returned value is a tuple, the resized image will be a rectangle. + + Args: + cfg: Configuration object with model parameters + + Returns: + Union[int, tuple]: Image resize dimensions + + Raises: + ValueError: If model family is not supported + """ + if cfg.model_family not in MODEL_IMAGE_SIZES: + raise ValueError(f"Unsupported model family: {cfg.model_family}") + + return MODEL_IMAGE_SIZES[cfg.model_family] + + +def get_action( + cfg: Any, + model: torch.nn.Module, + obs: Dict[str, Any], + task_label: str, + processor: Optional[Any] = None, + action_head: Optional[torch.nn.Module] = None, + proprio_projector: Optional[torch.nn.Module] = None, + noisy_action_projector: Optional[torch.nn.Module] = None, + use_film: bool = False, +) -> Union[List[np.ndarray], np.ndarray]: + """ + Query the model to get action predictions. + + Args: + cfg: Configuration object with model parameters + model: The loaded model + obs: Observation dictionary + task_label: Text description of the task + processor: Model processor for inputs + action_head: Optional action head for continuous actions + proprio_projector: Optional proprioception projector + noisy_action_projector: Optional noisy action projector for diffusion + use_film: Whether to use FiLM + + Returns: + Union[List[np.ndarray], np.ndarray]: Predicted actions + + Raises: + ValueError: If model family is not supported + """ + with torch.no_grad(): + if cfg.model_family == "openvla": + action = get_vla_action( + cfg=cfg, + vla=model, + processor=processor, + obs=obs, + task_label=task_label, + action_head=action_head, + proprio_projector=proprio_projector, + noisy_action_projector=noisy_action_projector, + use_film=use_film, + ) + else: + raise ValueError(f"Unsupported model family: {cfg.model_family}") -def get_action(cfg, model, obs, task_label, processor=None): - """Queries the model to get an action.""" - if cfg.model_family == "openvla": - action = get_vla_action( - model, processor, cfg.pretrained_checkpoint, obs, task_label, cfg.unnorm_key, center_crop=cfg.center_crop - ) - assert action.shape == (ACTION_DIM,) - else: - raise ValueError("Unexpected `model_family` found in config.") return action -def normalize_gripper_action(action, binarize=True): +def normalize_gripper_action(action: np.ndarray, binarize: bool = True) -> np.ndarray: """ - Changes gripper action (last dimension of action vector) from [0,1] to [-1,+1]. - Necessary for some environments (not Bridge) because the dataset wrapper standardizes gripper actions to [0,1]. - Note that unlike the other action dimensions, the gripper action is not normalized to [-1,+1] by default by - the dataset wrapper. + Normalize gripper action from [0,1] to [-1,+1] range. + + This is necessary for some environments because the dataset wrapper + standardizes gripper actions to [0,1]. Note that unlike the other action + dimensions, the gripper action is not normalized to [-1,+1] by default. Normalization formula: y = 2 * (x - orig_low) / (orig_high - orig_low) - 1 + + Args: + action: Action array with gripper action in the last dimension + binarize: Whether to binarize gripper action to -1 or +1 + + Returns: + np.ndarray: Action array with normalized gripper action """ - # Just normalize the last action to [-1,+1]. + # Create a copy to avoid modifying the original + normalized_action = action.copy() + + # Normalize the last action dimension to [-1,+1] orig_low, orig_high = 0.0, 1.0 - action[..., -1] = 2 * (action[..., -1] - orig_low) / (orig_high - orig_low) - 1 + normalized_action[..., -1] = 2 * (normalized_action[..., -1] - orig_low) / (orig_high - orig_low) - 1 if binarize: - # Binarize to -1 or +1. - action[..., -1] = np.sign(action[..., -1]) + # Binarize to -1 or +1 + normalized_action[..., -1] = np.sign(normalized_action[..., -1]) - return action + return normalized_action -def invert_gripper_action(action): +def invert_gripper_action(action: np.ndarray) -> np.ndarray: """ - Flips the sign of the gripper action (last dimension of action vector). - This is necessary for some environments where -1 = open, +1 = close, since + Flip the sign of the gripper action (last dimension of action vector). + + This is necessary for environments where -1 = open, +1 = close, since the RLDS dataloader aligns gripper actions such that 0 = close, 1 = open. + + Args: + action: Action array with gripper action in the last dimension + + Returns: + np.ndarray: Action array with inverted gripper action """ - action[..., -1] = action[..., -1] * -1.0 - return action + # Create a copy to avoid modifying the original + inverted_action = action.copy() + + # Invert the gripper action + inverted_action[..., -1] *= -1.0 + + return inverted_action diff --git a/prismatic/extern/hf/modeling_prismatic.py b/prismatic/extern/hf/modeling_prismatic.py index 4a26c4871..9013a4f57 100644 --- a/prismatic/extern/hf/modeling_prismatic.py +++ b/prismatic/extern/hf/modeling_prismatic.py @@ -1,15 +1,9 @@ """ modeling_prismatic.py -Core HuggingFace-style PrismaticPreTrainedModel and PrismaticForConditionalGeneration class definitions, inheriting -from the default `transformers.PretrainedModel`. Meant to be standalone and self-contained, but exactly replicate the -logic in `prismatic.models.vlms.prismatic.py`. - -Note =>> for the time being, not adding the custom HF "docstring" formatting. - -References [LLaVa, IDEFICS-2]: - => https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava/modeling_llava.py - => https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py +Core HuggingFace-style PrismaticPreTrainedModel and PrismaticForConditionalGeneration class definitions. +Inherits from the default `transformers.PretrainedModel`. Meant to be standalone and self-contained, +but exactly replicate the logic in `prismatic.models.vlms.prismatic.py`. """ import logging @@ -27,16 +21,26 @@ from transformers import AutoModelForCausalLM, PretrainedConfig, PreTrainedModel from transformers.modeling_outputs import ModelOutput +from prismatic.training.train_utils import ( + get_current_action_mask, + get_next_actions_mask, +) +from prismatic.vla.constants import ( + ACTION_DIM, + ACTION_PROPRIO_NORMALIZATION_TYPE, + ACTION_TOKEN_BEGIN_IDX, + IGNORE_INDEX, + NUM_ACTIONS_CHUNK, + STOP_INDEX, + NormalizationType, +) + from .configuration_prismatic import OpenVLAConfig, PrismaticConfig -# Get Logger +# Set up logger logger = logging.getLogger(__name__) -# === PyTorch/HuggingFace Default IGNORE_INDEX (for CrossEntropyLoss labels) -IGNORE_INDEX = -100 - - # === Utility Functions for Monkey-Patching === def unpack_tuple(fn: Callable[[Any], Tuple[Any]]) -> Callable[[Any], Any]: def wrapper(*args: Any, **kwargs: Any) -> Any: @@ -61,6 +65,13 @@ def ls_apply_patch(ls_module: LayerScale): # === Prismatic Vision Backbone (nn.Module) Definitions (w/ Fused Backbone Support) === class PrismaticVisionBackbone(nn.Module): + """ + Vision backbone for Prismatic models that handles image feature extraction. + + Supports both single backbone (e.g., SigLIP) and fused backbone (e.g., SigLIP + DINOv2) configurations. + For fused backbones, features from both models are concatenated along the feature dimension. + """ + def __init__( self, use_fused_vision_backbone: bool, @@ -68,59 +79,152 @@ def __init__( timm_model_ids: List[str], timm_override_act_layers: List[Optional[str]], ) -> None: + """ + Initialize the vision backbone. + + Args: + use_fused_vision_backbone: Whether to use two backbones and fuse their features + image_sizes: List of image sizes for each backbone + timm_model_ids: List of TIMM model IDs to use for each backbone + timm_override_act_layers: List of activation layer overrides for each backbone + """ super().__init__() self.use_fused_vision_backbone = use_fused_vision_backbone + self.num_images_in_input = 1 # Default value, can be overridden later - # [Contract] Validate number of (fused) vision backbones, create "alpha" featurizer and Instantiate - # =>> Note :: Monkey-Patch the `forward()` function of the backbone to ensure FSDP-compatibility - # Hardcodes `get_intermediate_layers` to return the **SECOND-TO-LAST** layer patches! - assert len(timm_model_ids) <= 2, "Prismatic models only support up to 2 (fused) vision backbones!" - self.featurizer = timm.create_model( - timm_model_ids[0], - pretrained=False, - num_classes=0, - img_size=image_sizes[0], - act_layer=timm_override_act_layers[0], - ) - self.featurizer.forward = unpack_tuple( - partial(self.featurizer.get_intermediate_layers, n={len(self.featurizer.blocks) - 2}) + # Validate number of (fused) vision backbones + if len(timm_model_ids) > 2: + raise ValueError("Prismatic models only support up to 2 (fused) vision backbones!") + + # Create primary featurizer + self.featurizer = self._create_featurizer( + model_id=timm_model_ids[0], img_size=image_sizes[0], act_layer=timm_override_act_layers[0] ) self.embed_dim = self.featurizer.embed_dim - # If `use_fused_vision_backbone` =>> create "beta" featurizer + # Create secondary featurizer if using fused backbone if self.use_fused_vision_backbone: - self.fused_featurizer = timm.create_model( - timm_model_ids[1], - pretrained=False, - num_classes=0, - img_size=image_sizes[1], - act_layer=timm_override_act_layers[1], - ) - self.fused_featurizer.forward = unpack_tuple( - partial(self.fused_featurizer.get_intermediate_layers, n={len(self.fused_featurizer.blocks) - 2}) + self.fused_featurizer = self._create_featurizer( + model_id=timm_model_ids[1], img_size=image_sizes[1], act_layer=timm_override_act_layers[1] ) self.embed_dim += self.fused_featurizer.embed_dim - # Patch `vision_backbone.featurizer` and `vision_backbone.fused_featurizer` with HF-Compatible LayerScale + # Patch LayerScale modules for HF compatibility + self._patch_layer_scales() + + def _create_featurizer(self, model_id: str, img_size: int, act_layer: Optional[str]) -> nn.Module: + """ + Create a TIMM-based featurizer model with appropriate configurations. + + Args: + model_id: The TIMM model ID to load + img_size: Input image size for the model + act_layer: Override for the activation layer type + + Returns: + A configured featurizer model + """ + featurizer = timm.create_model( + model_id, + pretrained=False, + num_classes=0, + img_size=img_size, + act_layer=act_layer, + ) + + # Monkey-patch the forward function to extract the second-to-last layer features + num_blocks = len(featurizer.blocks) + featurizer.forward = unpack_tuple(partial(featurizer.get_intermediate_layers, n={num_blocks - 2})) + + return featurizer + + def _patch_layer_scales(self) -> None: + """ + Patch all LayerScale modules to be compatible with HF's parameter naming. + + HF Transformers overwrites parameters with names containing 'gamma', + so we need to rename and modify the forward method. + """ + # Patch primary featurizer for module in self.featurizer.modules(): if isinstance(module, LayerScale): ls_apply_patch(module) + # Patch secondary featurizer if it exists if self.use_fused_vision_backbone: for module in self.fused_featurizer.modules(): if isinstance(module, LayerScale): ls_apply_patch(module) + def get_num_patches(self) -> int: + """ + Returns the number of vision patches output by the vision backbone. + + Returns: + Number of patches per image + """ + return self.featurizer.patch_embed.num_patches + + def get_num_images_in_input(self) -> int: + """ + Returns the number of input images for the vision backbone. + + Returns: + Number of images expected in the input + """ + return self.num_images_in_input + + def set_num_images_in_input(self, num_images_in_input: int) -> None: + """ + Sets the number of input images for the vision backbone. + + Args: + num_images_in_input: Number of images to expect in the input + """ + self.num_images_in_input = num_images_in_input + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: - """Run image (`pixel_values`) through featurizer; if channel-stacked, then dispatch and sequence stack.""" - if not self.use_fused_vision_backbone: - return self.featurizer(pixel_values) + """ + Implements the forward pass for the vision backbone. + + If `self.use_fused_vision_backbone == True`, uses both SigLIP and DINOv2 transformers to extract visual features + (otherwise uses SigLIP only). Allows multi-image inputs (but only for fused vision backbone). + + Args: + pixel_values (torch.Tensor): Pixels for input image(s), (B, C, H, W). + """ + if self.num_images_in_input == 1: + if not self.use_fused_vision_backbone: + return self.featurizer(pixel_values) + + # Split `pixel_values :: [bsz, 2 * 3, resolution, resolution]` =>> featurize =>> channel stack + img, img_fused = torch.split(pixel_values, [3, 3], dim=1) + patches, patches_fused = self.featurizer(img), self.fused_featurizer(img_fused) + + return torch.cat([patches, patches_fused], dim=2) + + else: + assert self.use_fused_vision_backbone, "Multi-image inputs require using fused backbone!" + + # Split `pixel_values` into individual images (each with 6 channels: 3 for SigLIP + 3 for DINOv2) + images = torch.split(pixel_values, [6] * self.num_images_in_input, dim=1) - # Split `pixel_values :: [bsz, 2 * 3, resolution, resolution]` =>> featurize =>> channel stack - img, img_fused = torch.split(pixel_values, [3, 3], dim=1) - patches, patches_fused = self.featurizer(img), self.fused_featurizer(img_fused) + # Process each image and collect patches + all_patches = [] + for img in images: + # Split each image further into two stacks of channels (each with 3 channels) + img_regular, img_fused = torch.split(img, [3, 3], dim=1) - return torch.cat([patches, patches_fused], dim=2) + # Get patches from both SigLIP and DINOv2 vision transformers + patches = self.featurizer(img_regular) + patches_fused = self.fused_featurizer(img_fused) + + # Concatenate SigLIP and DINOv2 patches along the hidden dimension + combined_patches = torch.cat([patches, patches_fused], dim=2) + all_patches.append(combined_patches) + + # Concatenate all patches along the patch dimension + return torch.cat(all_patches, dim=1) # === Prismatic Projector (nn.Module) Definitions === @@ -250,6 +354,7 @@ def __init__(self, config: PrismaticConfig) -> None: ) self.vocab_size = config.text_config.vocab_size self.pad_token_id = config.pad_token_id + self.llm_dim = config.text_config.hidden_size # HF Boilerplate =>> initializes weights via `_init_weights()` and sets gradient checkpointing self.post_init() @@ -287,6 +392,109 @@ def resize_token_embeddings( return updated_embeddings + def _replace_input_embeddings(self, input_embeddings, all_actions_mask, noisy_action_features): + """ + Replace embeddings in input_embeddings at positions where all_actions_mask is True + with embeddings from noisy_action_features, using vectorized operations. + + Args: + input_embeddings: Tensor of shape (B, S, D) + all_actions_mask: Boolean tensor of shape (B, S) + noisy_action_features: Tensor of shape (B, K, D) where K is the number of True values in mask per sample + + Returns: + Modified input_embeddings tensor + """ + # Clone input to avoid modifying the original tensor + new_input_embeddings = input_embeddings.clone() + + # Create a tensor with the same shape of input_embeddings to hold the noisy action features + repositioned_noisy_action_features = torch.zeros_like(input_embeddings) + + # Create batch indices for splicing + batch_indices = torch.arange(input_embeddings.shape[0], device=input_embeddings.device) + batch_indices = batch_indices.unsqueeze(1).expand(-1, noisy_action_features.shape[1]) + + # Get indices where mask is True for each sample + masked_indices = torch.stack([torch.where(mask)[0] for mask in all_actions_mask]) + + # Move the noisy action features into their correct positions + repositioned_noisy_action_features[batch_indices, masked_indices] = noisy_action_features + + # Combine original input embeddings and noisy action embeddings using the mask + new_input_embeddings = torch.where( + all_actions_mask.unsqueeze(-1), repositioned_noisy_action_features, new_input_embeddings + ) + + return new_input_embeddings + + def _process_action_masks(self, labels): + """Helper to get action masks from labels""" + current_action_mask = get_current_action_mask(labels) + next_actions_mask = get_next_actions_mask(labels) + all_actions_mask = current_action_mask | next_actions_mask # (B, seq_len) + return all_actions_mask + + def _process_vision_features(self, pixel_values, language_embeddings=None, use_film=False): + """Process vision features with optional FiLM conditioning""" + if use_film: + # FiLM: Infuse language inputs into visual features + patch_features = self.vision_backbone(pixel_values, language_embeddings) # (bsz, 256 * num_images, D) + else: + patch_features = self.vision_backbone(pixel_values) # (bsz, 256 * num_images, D) + + # Project patch embeddings into language embedding space + return self.projector(patch_features) + + def _process_proprio_features(self, projected_patch_embeddings, proprio, proprio_projector): + """Process proprioceptive features and append to vision features""" + if proprio_projector is not None and proprio is not None: + # projected_patch_embeddings: (bsz, num_patches * num_images, llm_dim) + # proprio: (bsz, proprio_dim) or (propro_dim,) + proprio = proprio.reshape(projected_patch_embeddings.shape[0], -1) # (bsz, proprio_dim) + proprio_features = proprio_projector(proprio) # (bsz, llm_dim) + proprio_features = proprio_features.unsqueeze(dim=1) # (bsz, 1, llm_dim) + # For simplicity, just append proprio token to the end of projected vision patch tokens + return torch.cat((projected_patch_embeddings, proprio_features), dim=1) + return projected_patch_embeddings + + def _build_multimodal_attention(self, input_embeddings, projected_patch_embeddings, attention_mask): + """Build multimodal embeddings and attention mask""" + # Update attention mask + projected_patch_attention_mask = None + if attention_mask is not None: + projected_patch_attention_mask = torch.full( + (projected_patch_embeddings.shape[0], projected_patch_embeddings.shape[1]), + fill_value=True, + dtype=attention_mask.dtype, + device=attention_mask.device, + ) + + # Build multimodal embeddings & attention mask; insert embeddings after token (1:) + multimodal_embeddings = torch.cat( + [input_embeddings[:, :1, :], projected_patch_embeddings, input_embeddings[:, 1:, :]], dim=1 + ) + + multimodal_attention_mask = None + if attention_mask is not None: + multimodal_attention_mask = torch.cat( + [attention_mask[:, :1], projected_patch_attention_mask, attention_mask[:, 1:]], dim=1 + ) + + return multimodal_embeddings, multimodal_attention_mask + + def _build_multimodal_labels(self, labels, projected_patch_embeddings): + """Build multimodal labels with IGNORE_INDEX for patch embeddings""" + if labels is not None: + projected_patch_labels = torch.full( + (projected_patch_embeddings.shape[0], projected_patch_embeddings.shape[1]), + fill_value=IGNORE_INDEX, + dtype=labels.dtype, + device=labels.device, + ) + return torch.cat([labels[:, :1], projected_patch_labels, labels[:, 1:]], dim=1) + return None + # === Core Prismatic VLM `forward()` Logic === def forward( self, @@ -301,6 +509,12 @@ def forward( output_hidden_states: Optional[bool] = None, output_projector_features: Optional[bool] = None, return_dict: Optional[bool] = None, + proprio=None, + proprio_projector=None, + noisy_actions=None, + noisy_action_projector=None, + diffusion_timestep_embeddings=None, + use_film: bool = False, ) -> Union[Tuple, PrismaticCausalLMOutputWithPast]: """Run a forward pass through the VLM, returning a PrismaticCausalLMOutputWithPast instance.""" output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -316,11 +530,6 @@ def forward( # Instantiate Placeholder for Projector Features projected_patch_embeddings = None - # Note :: We only support forward passes with the following cases: - # => Cached Generation :: (input_ids.shape[1] == 1) and (past_key_values is not None) - # => Unimodal Forward :: (pixel_values is None) - # => Multimodal Forward :: (pixel_values is not None) and (input_ids/embeds.shape[0] == pixel_values.shape[0]) - # === Handle Generation with Cache (`input_ids.shape[1] == 1`) =>> requires `past_keys_values` === if input_ids.shape[1] == 1: assert input_ids.shape[0] == 1, "Generation is only currently supported for batch size of 1!" @@ -360,47 +569,66 @@ def forward( # === Handle Multimodal Forward === elif (input_ids.shape[0] == pixel_values.shape[0]) or (inputs_embeds.shape[0] == pixel_values.shape[0]): - assert past_key_values is None, "Unexpected key `past_key_values` provided during language-only forward!" + assert past_key_values is None, "Unexpected key `past_key_values` provided during multimodal forward!" - # Visual Feature Extraction - patch_features = self.vision_backbone(pixel_values) - - # Projection Logic =>> Update Attention Mask - projected_patch_embeddings = self.projector(patch_features) - projected_patch_attention_mask = None - if attention_mask is not None: - projected_patch_attention_mask = torch.full( - (projected_patch_embeddings.shape[0], projected_patch_embeddings.shape[1]), - fill_value=True, - dtype=attention_mask.dtype, - device=attention_mask.device, - ) + # Get input embeddings (from language model embeddings) + input_embeddings = self.get_input_embeddings()(input_ids) # (B, seq_len, D) + + # Extract action masks + all_actions_mask = self._process_action_masks(labels) - # Get Input Embeddings (from Language Model Embeddings) - input_embeddings = self.get_input_embeddings()(input_ids) + # Extract the language portion of the input embeddings (i.e. remove the action tokens portion) + language_embeddings = input_embeddings[~all_actions_mask].reshape( + input_embeddings.shape[0], -1, input_embeddings.shape[2] + ) # (B, lang_seq_len, llm_dim) - # Build Multimodal Embeddings & Attention Mask =>> Prismatic defaults to inserting after token (1:) - multimodal_embeddings = torch.cat( - [input_embeddings[:, :1, :], projected_patch_embeddings, input_embeddings[:, 1:, :]], dim=1 + # Get visual features + projected_patch_embeddings = self._process_vision_features(pixel_values, language_embeddings, use_film) + + # Add proprioceptive state if provided + projected_patch_embeddings = self._process_proprio_features( + projected_patch_embeddings, proprio, proprio_projector ) - multimodal_attention_mask = None - if attention_mask is not None: - multimodal_attention_mask = torch.cat( - [attention_mask[:, :1], projected_patch_attention_mask, attention_mask[:, 1:]], dim=1 + + # [Diffusion] Add diffusion timestep embedding if provided + if diffusion_timestep_embeddings is not None: + # For simplicity, just append diffusion timestep embedding to the end of projected vision patch tokens + projected_patch_embeddings = torch.cat( + (projected_patch_embeddings, diffusion_timestep_embeddings), dim=1 ) - # Build Labels (if specified) =>> Ignore Labels for Patch Embeddings - multimodal_labels = None - if labels is not None: - projected_patch_labels = torch.full( - (projected_patch_embeddings.shape[0], projected_patch_embeddings.shape[1]), - fill_value=IGNORE_INDEX, - dtype=labels.dtype, - device=labels.device, + # Process action embeddings + if noisy_actions is not None: + # Get mask corresponding to all action tokens + all_actions_mask = self._process_action_masks(labels) + + # Reshape noisy actions into individual action tokens + # noisy_actions: (B, chunk_len, action_dim) -> (B, chunk_len * action_dim, 1) + B = noisy_actions.shape[0] + noisy_actions = noisy_actions.reshape(B, -1).unsqueeze(-1) + + # Project noisy action tokens into language model embedding space + noisy_action_features = noisy_action_projector(noisy_actions) # (B, chunk_len * action_dim, llm_dim) + + # Replace embeddings of the action tokens with noisy action embeddings + input_embeddings = self._replace_input_embeddings( + input_embeddings, all_actions_mask, noisy_action_features ) - multimodal_labels = torch.cat([labels[:, :1], projected_patch_labels, labels[:, 1:]], dim=1) + else: + # Replace the embeddings of the action tokens with zeros + # (Later on, the positional embeddings will be added to them) + all_actions_mask = all_actions_mask.unsqueeze(-1) # (B, seq_len, 1) + input_embeddings = input_embeddings * ~all_actions_mask + + # Build multimodal embeddings & attention mask + multimodal_embeddings, multimodal_attention_mask = self._build_multimodal_attention( + input_embeddings, projected_patch_embeddings, attention_mask + ) + + # Build labels for multimodal sequence if needed + multimodal_labels = self._build_multimodal_labels(labels, projected_patch_embeddings) - # Dispatch to Language Model + # Dispatch to language model language_model_output = self.language_model( input_ids=None, attention_mask=multimodal_attention_mask, @@ -503,10 +731,244 @@ def __init__(self, config: OpenVLAConfig) -> None: # Compute vocab size for de-tokenization -- revert added "multiple of" self.vocab_size = self.config.text_config.vocab_size - self.config.pad_to_multiple_of + def _prepare_input_for_action_prediction(self, input_ids, attention_mask): + """Prepares input for action prediction by adding necessary tokens""" + # Add (ACTION_DIM * NUM_ACTIONS_CHUNK) placeholder tokens to input_ids to simulate action tokens + placeholder_action_token_ids = ( + torch.ones((input_ids.shape[0], ACTION_DIM * NUM_ACTIONS_CHUNK)).to(input_ids.device).to(input_ids.dtype) + ) + input_ids = torch.cat([input_ids, placeholder_action_token_ids], dim=-1) + + # Add stop token to sequence (needed in non-causal bi-directional self-attention, as it appears at train time) + stop_token_id = torch.ones((input_ids.shape[0], 1)).to(input_ids.device).to(input_ids.dtype) * STOP_INDEX + input_ids = torch.cat([input_ids, stop_token_id], dim=-1) + + # Extend the attention mask to fit the new shape of input + # Note: Only batch size == 1 supported right now + mask_extension = ( + torch.ones((attention_mask.shape[0], input_ids.shape[-1] - attention_mask.shape[-1])) + .to(attention_mask.device) + .to(attention_mask.dtype) + ) + attention_mask = torch.cat([attention_mask, mask_extension], dim=-1) + + return input_ids, attention_mask + + def _prepare_labels_for_action_prediction(self, labels, input_ids): + """Creates labels tensor for action prediction if not provided""" + # Extend labels tensor with fake action labels + ARBITRARY_ACTION_TOKEN_IDX = ACTION_TOKEN_BEGIN_IDX + 1 + labels_extension = ( + torch.ones((labels.shape[0], input_ids.shape[-1] - labels.shape[-1])).to(labels.device).to(labels.dtype) + * ARBITRARY_ACTION_TOKEN_IDX + ) + labels = torch.cat([labels, labels_extension], dim=-1) + + # Replace last label token with stop token + labels[:, -1] = STOP_INDEX + + return labels + + def _unnormalize_actions(self, normalized_actions, unnorm_key=None): + """Unnormalize actions using dataset statistics""" + action_norm_stats = self.get_action_stats(unnorm_key) + + if ACTION_PROPRIO_NORMALIZATION_TYPE == NormalizationType.BOUNDS: + mask = action_norm_stats.get("mask", np.ones_like(action_norm_stats["min"], dtype=bool)) + action_high, action_low = np.array(action_norm_stats["max"]), np.array(action_norm_stats["min"]) + elif ACTION_PROPRIO_NORMALIZATION_TYPE == NormalizationType.BOUNDS_Q99: + mask = action_norm_stats.get("mask", np.ones_like(action_norm_stats["q01"], dtype=bool)) + action_high, action_low = np.array(action_norm_stats["q99"]), np.array(action_norm_stats["q01"]) + else: + raise ValueError("Unsupported action/proprio normalization type detected!") + + actions = np.where( + mask, + 0.5 * (normalized_actions + 1) * (action_high - action_low + 1e-8) + action_low, + normalized_actions, + ) + + return actions + + def _run_diffusion_prediction( + self, + input_embeddings, + all_actions_mask, + noise, + action_head, + projected_patch_embeddings, + labels, + attention_mask, + NUM_PATCHES, + NUM_PROMPT_TOKENS, + noisy_action_projector, + ): + """Run diffusion-based action prediction""" + # Set diffusion timestep values + action_head.noise_scheduler.set_timesteps(action_head.num_diffusion_steps) + # Clone embedding for reuse in each timestep + orig_projected_patch_embeddings = projected_patch_embeddings.clone() + curr_noisy_actions = noise + + # Reverse diffusion: Iteratively denoise to generate action prediction + for t in action_head.noise_scheduler.timesteps: + # Get diffusion model's noise prediction (conditioned on VLA latent embedding, current noisy action + # embedding, and diffusion timestep embedding) + timesteps = torch.Tensor([t]).to(labels.device) + diffusion_timestep_embeddings = ( + action_head.time_encoder(timesteps).to(curr_noisy_actions.dtype).to(curr_noisy_actions.device) + ) # (B, llm_dim) + diffusion_timestep_embeddings = diffusion_timestep_embeddings.unsqueeze(1) # (B, 1, llm_dim) + + # [Diffusion] Replace the embeddings of the action tokens with noisy actions + # (Later on, the positional embeddings will be added to them) + + # For simplicity, append diffusion timestep embedding to the end of projected vision tokens + projected_patch_embeddings = torch.cat( + (orig_projected_patch_embeddings, diffusion_timestep_embeddings), dim=1 + ) + + # Reshape and project noisy actions into language embedding space + B = curr_noisy_actions.shape[0] + orig_curr_noisy_actions_shape = curr_noisy_actions.shape + curr_noisy_actions = curr_noisy_actions.reshape(B, -1).unsqueeze(-1) + noisy_action_features = noisy_action_projector(curr_noisy_actions) + curr_noisy_actions = curr_noisy_actions.reshape(orig_curr_noisy_actions_shape) + + # Replace action token embeddings with noisy action embeddings + input_embeddings = self._replace_input_embeddings( + input_embeddings.clone(), all_actions_mask, noisy_action_features + ) + + # Build multimodal embeddings and attention mask + multimodal_embeddings, multimodal_attention_mask = self._build_multimodal_attention( + input_embeddings, projected_patch_embeddings, attention_mask + ) + + # Forward pass through language model + language_model_output = self.language_model( + input_ids=None, + attention_mask=multimodal_attention_mask, + position_ids=None, + past_key_values=None, + inputs_embeds=multimodal_embeddings, + labels=None, + use_cache=None, + output_attentions=False, + output_hidden_states=True, + return_dict=True, + ) + + # Extract hidden states for action portion of response + last_hidden_states = language_model_output.hidden_states[-1] # (B, seq_len, D) + actions_hidden_states = last_hidden_states[ + :, + NUM_PATCHES + NUM_PROMPT_TOKENS : NUM_PATCHES + NUM_PROMPT_TOKENS + ACTION_DIM * NUM_ACTIONS_CHUNK, + :, + ] # (B, act_chunk_len, D) + + # Predict noise and update noisy actions: x_t -> x_{t-1} + noise_pred = action_head.predict_noise(actions_hidden_states) + curr_noisy_actions = action_head.noise_scheduler.step(noise_pred, t, curr_noisy_actions).prev_sample + + curr_noisy_actions = curr_noisy_actions.reshape(NUM_ACTIONS_CHUNK, ACTION_DIM) + + # Return final actions + return curr_noisy_actions.float().cpu().detach().numpy(), actions_hidden_states + + def _regression_or_discrete_prediction( + self, + input_embeddings, + all_actions_mask, + projected_patch_embeddings, + attention_mask, + labels, + NUM_PATCHES, + NUM_PROMPT_TOKENS, + action_head=None, + ): + """Run L1 regression-based continuous action prediction or discrete action tokens prediction.""" + # Zero out action token embeddings + all_actions_mask = all_actions_mask.unsqueeze(-1) # (B, seq_len, 1) + input_embeddings = input_embeddings * ~all_actions_mask + + # Build multimodal embeddings and attention mask + multimodal_embeddings, multimodal_attention_mask = self._build_multimodal_attention( + input_embeddings, projected_patch_embeddings, attention_mask + ) + + # Forward pass through language model + language_model_output = self.language_model( + input_ids=None, + attention_mask=multimodal_attention_mask, + position_ids=None, + past_key_values=None, + inputs_embeds=multimodal_embeddings, + labels=None, + use_cache=None, + output_attentions=False, + output_hidden_states=True, + return_dict=True, + ) + + # Extract hidden states for action tokens + last_hidden_states = language_model_output.hidden_states[-1] # (B, seq_len, D) + actions_hidden_states = last_hidden_states[ + :, + NUM_PATCHES + NUM_PROMPT_TOKENS : NUM_PATCHES + NUM_PROMPT_TOKENS + ACTION_DIM * NUM_ACTIONS_CHUNK, + :, + ] # (B, act_chunk_len, D) + + # Handle different prediction methods + if action_head is not None: + # L1 regression prediction + normalized_actions = action_head.predict_action(actions_hidden_states) + normalized_actions = normalized_actions.reshape(NUM_ACTIONS_CHUNK, ACTION_DIM) + normalized_actions = normalized_actions.float().cpu().detach().numpy() + else: + # Discrete token-based prediction + predicted_action_token_ids = ( + language_model_output.logits[ + :, + NUM_PATCHES + NUM_PROMPT_TOKENS : NUM_PATCHES + NUM_PROMPT_TOKENS + ACTION_DIM * NUM_ACTIONS_CHUNK, + ] + .argmax(dim=2) + .cpu() + .numpy() + ) + discretized_actions = self.vocab_size - predicted_action_token_ids + discretized_actions = np.clip(discretized_actions - 1, a_min=0, a_max=self.bin_centers.shape[0] - 1) + normalized_actions = self.bin_centers[discretized_actions] + normalized_actions = normalized_actions.reshape(NUM_ACTIONS_CHUNK, ACTION_DIM) + + return normalized_actions, actions_hidden_states + def predict_action( - self, input_ids: Optional[torch.LongTensor] = None, unnorm_key: Optional[str] = None, **kwargs: str + self, + input_ids: Optional[torch.LongTensor] = None, + unnorm_key: Optional[str] = None, + proprio=None, + proprio_projector=None, + action_head=None, + noisy_action_projector=None, + use_film: bool = False, + **kwargs: str, ) -> np.ndarray: - """Thin wrapper around .generate() that decodes predicted actions and unnormalizes them.""" + """Predict actions from input sequence, with options for different prediction methods. + + Args: + input_ids: Input token ids + unnorm_key: Key for unnormalization statistics + proprio: Proprioceptive features + proprio_projector: Projector for proprioceptive features + action_head: Optional head for L1 regression or diffusion-based prediction + noisy_action_projector: Projector for noisy actions in diffusion-based prediction + use_film: Whether to use FiLM conditioning + **kwargs: Additional arguments including pixel_values and attention_mask + + Returns: + Tuple of (unnormalized_actions, action_hidden_states) + """ # If the special empty token ('') does not already appear after the colon (':') token in the prompt # (after "OUT:" or "ASSISTANT:"), insert it to match the inputs seen at training time if not torch.all(input_ids[:, -1] == 29871): @@ -514,29 +976,92 @@ def predict_action( (input_ids, torch.unsqueeze(torch.Tensor([29871]).long(), dim=0).to(input_ids.device)), dim=1 ) - # Run VLA inference - generated_ids = self.generate(input_ids, max_new_tokens=self.get_action_dim(unnorm_key), **kwargs) + pixel_values = kwargs["pixel_values"] + attention_mask = kwargs["attention_mask"] - # Extract predicted action tokens and translate into (normalized) continuous actions - predicted_action_token_ids = generated_ids[0, -self.get_action_dim(unnorm_key) :].cpu().numpy() - discretized_actions = self.vocab_size - predicted_action_token_ids - discretized_actions = np.clip(discretized_actions - 1, a_min=0, a_max=self.bin_centers.shape[0] - 1) - normalized_actions = self.bin_centers[discretized_actions] + # Create fake labels tensor (needed for action mask) + labels = input_ids.clone() + labels[:] = IGNORE_INDEX - # Unnormalize actions - action_norm_stats = self.get_action_stats(unnorm_key) - mask = action_norm_stats.get("mask", np.ones_like(action_norm_stats["q01"], dtype=bool)) - action_high, action_low = np.array(action_norm_stats["q99"]), np.array(action_norm_stats["q01"]) - actions = np.where( - mask, - 0.5 * (normalized_actions + 1) * (action_high - action_low) + action_low, - normalized_actions, + # Get number of tokens in prompt (excluding the start token) + NUM_PROMPT_TOKENS = input_ids.shape[-1] - 1 # Subtract action tokens and stop token + + # Prepare inputs by adding necessary tokens + input_ids, attention_mask = self._prepare_input_for_action_prediction(input_ids, attention_mask) + + # Update labels tensor for action mask computation later + labels = self._prepare_labels_for_action_prediction(labels, input_ids) + + # Get input embeddings and action masks + input_embeddings = self.get_input_embeddings()(input_ids) + all_actions_mask = self._process_action_masks(labels) + + # Extract language embeddings + language_embeddings = input_embeddings[~all_actions_mask].reshape( + input_embeddings.shape[0], -1, input_embeddings.shape[2] ) - return actions + # Process vision features + projected_patch_embeddings = self._process_vision_features(pixel_values, language_embeddings, use_film) + + # Add proprioceptive features if provided + use_proprio = proprio_projector is not None and proprio is not None + if use_proprio: + proprio = torch.Tensor(proprio).to(projected_patch_embeddings.device, dtype=projected_patch_embeddings.dtype) + projected_patch_embeddings = self._process_proprio_features( + projected_patch_embeddings, proprio, proprio_projector + ) + + # Use diffusion if provided, otherwise use regression or discrete prediction + use_diffusion = noisy_action_projector is not None and hasattr(action_head, "noise_scheduler") + + # Calculate number of patches (including proprio token and/or diffusion timestep embedding if present) + NUM_PATCHES = self.vision_backbone.get_num_patches() * self.vision_backbone.get_num_images_in_input() + if use_proprio: + NUM_PATCHES += 1 + if use_diffusion: + NUM_PATCHES += 1 + + if use_diffusion: + # Sample random noise with shape equal to output action, used as the starting state for reverse diffusion + noise = torch.randn( + size=(1, NUM_ACTIONS_CHUNK, ACTION_DIM), device=input_embeddings.device, dtype=input_embeddings.dtype + ) + + # Run diffusion-based prediction + normalized_actions, actions_hidden_states = self._run_diffusion_prediction( + input_embeddings, + all_actions_mask, + noise, + action_head, + projected_patch_embeddings, + labels, + attention_mask, + NUM_PATCHES, + NUM_PROMPT_TOKENS, + noisy_action_projector, + ) + else: + # Run regression or discrete token-based prediction + normalized_actions, actions_hidden_states = self._regression_or_discrete_prediction( + input_embeddings, + all_actions_mask, + projected_patch_embeddings, + attention_mask, + labels, + NUM_PATCHES, + NUM_PROMPT_TOKENS, + action_head, + ) + + # Unnormalize predicted actions + actions = self._unnormalize_actions(normalized_actions, unnorm_key) + + return actions, actions_hidden_states @staticmethod def _check_unnorm_key(norm_stats: Dict[str, Dict[str, Any]], unnorm_key: Optional[str]) -> str: + """Validate and resolve the unnormalization key for action statistics""" if unnorm_key is None: assert len(norm_stats) == 1, ( f"Your model was trained on more than one dataset, " @@ -554,7 +1079,7 @@ def _check_unnorm_key(norm_stats: Dict[str, Dict[str, Any]], unnorm_key: Optiona def get_action_dim(self, unnorm_key: Optional[str] = None) -> int: """Get the dimensionality of the policy's action space.""" unnorm_key = self._check_unnorm_key(self.norm_stats, unnorm_key) - return len(self.norm_stats[unnorm_key]["action"]["q01"]) + return len(self.norm_stats[unnorm_key]["action"]["min"]) def get_action_stats(self, unnorm_key: Optional[str] = None) -> Dict[str, Any]: """Get all the logged statistics for the given dataset.""" diff --git a/prismatic/models/action_heads.py b/prismatic/models/action_heads.py new file mode 100644 index 000000000..b3043c078 --- /dev/null +++ b/prismatic/models/action_heads.py @@ -0,0 +1,211 @@ +"""Implementations of various action heads, which serve as alternatives to VLM sequential token prediction.""" + +import math + +import numpy as np +import torch +import torch.nn as nn +from diffusers.schedulers.scheduling_ddim import DDIMScheduler +from prismatic.vla.constants import ACTION_DIM, ACTION_TOKEN_BEGIN_IDX, IGNORE_INDEX, NUM_ACTIONS_CHUNK, PROPRIO_DIM, STOP_INDEX + + +class SinusoidalPositionalEncoding(nn.Module): + """ + Sine- and cosine-based positional encoding that produces embeddings of a batch of timesteps. + + For example, at train time, the input might be a batch of 32 randomly sampled diffusion timesteps -> shape (32,) + Then the output would be a batch of 32 timestep embeddings -> shape (32, D) + + Adapted from: https://github.com/real-stanford/diffusion_policy/blob/main/diffusion_policy/model/diffusion/positional_embedding.py + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim # dimensionality of the positional encoding + + def forward(self, x): + # x: (batch_size,) + device = x.device + assert self.dim % 2 == 0, f"# dimensions must be even but got {self.dim}" + half_dim = self.dim // 2 + exponent = torch.arange(half_dim, device=device) * -math.log(10000) / (half_dim - 1) # shape: (D/2,) + emb = torch.exp(exponent) # shape: (D/2,) + emb = x[:, None] * emb[None, :] # shape: (batch_size, 1) * (1, D/2) -> (batch_size, D/2) + emb = torch.cat((emb.sin(), emb.cos()), dim=-1) # shape: (batch_size, D) + return emb + + +class MLPResNetBlock(nn.Module): + """One MLP ResNet block with a residual connection.""" + def __init__(self, dim): + super().__init__() + self.dim = dim + self.ffn = nn.Sequential( # feedforward network, similar to the ones in Transformers + nn.LayerNorm(dim), + nn.Linear(dim, dim), + nn.ReLU(), + ) + + def forward(self, x): + # x: (batch_size, hidden_dim) + # We follow the module ordering of "Pre-Layer Normalization" feedforward networks in Transformers as + # described here: https://arxiv.org/pdf/2002.04745.pdf + identity = x + x = self.ffn(x) + x = x + identity + return x + + +class MLPResNet(nn.Module): + """MLP with residual connection blocks.""" + def __init__(self, num_blocks, input_dim, hidden_dim, output_dim): + super().__init__() + self.layer_norm1 = nn.LayerNorm(input_dim) + self.fc1 = nn.Linear(input_dim, hidden_dim) + self.relu = nn.ReLU() + self.mlp_resnet_blocks = nn.ModuleList() + for _ in range(num_blocks): + self.mlp_resnet_blocks.append(MLPResNetBlock(dim=hidden_dim)) + self.layer_norm2 = nn.LayerNorm(hidden_dim) + self.fc2 = nn.Linear(hidden_dim, output_dim) + + def forward(self, x): + # x: (batch_size, input_dim) + x = self.layer_norm1(x) # shape: (batch_size, input_dim) + x = self.fc1(x) # shape: (batch_size, hidden_dim) + x = self.relu(x) # shape: (batch_size, hidden_dim) + for block in self.mlp_resnet_blocks: + x = block(x) # shape: (batch_size, hidden_dim) + x = self.layer_norm2(x) # shape: (batch_size, hidden_dim) + x = self.fc2(x) # shape: (batch_size, output_dim) + return x + + +class L1RegressionActionHead(nn.Module): + """Simple MLP-based action head that generates continuous actions via L1 regression.""" + def __init__( + self, + input_dim=4096, + hidden_dim=4096, + action_dim=7, + ): + super().__init__() + self.action_dim = action_dim + self.model = MLPResNet( + num_blocks=2, input_dim=input_dim*ACTION_DIM, hidden_dim=hidden_dim, output_dim=action_dim + ) + + def predict_action(self, actions_hidden_states): + # actions_hidden_states: last hidden states of Transformer corresponding to action tokens in sequence + # - shape: (batch_size, chunk_len * action_dim, hidden_dim) + # ground_truth_actions: ground-truth actions + # - shape: (batch_size, chunk_len, action_dim) + batch_size = actions_hidden_states.shape[0] + device = actions_hidden_states.device + rearranged_actions_hidden_states = actions_hidden_states.reshape(batch_size, NUM_ACTIONS_CHUNK, -1) + action = self.model(rearranged_actions_hidden_states) + return action + + +class NoisePredictionModel(nn.Module): + """ + Diffusion noise prediction model that takes an observation embedding (which fuses the + noisy action, diffusion timestep, and image-language observation embeddings) and + outputs a noise prediction. + """ + + def __init__( + self, + transformer_hidden_dim, # Transformer hidden embedding size + hidden_dim, # MLP hidden size + action_dim=7, # action dimensionality + ): + super().__init__() + self.mlp_resnet = MLPResNet( + num_blocks=2, + input_dim=transformer_hidden_dim, + hidden_dim=hidden_dim, + output_dim=action_dim, + ) + + def forward( + self, + obs, + ): + # obs: observation embeddings to condition the generation on + # - shape: (batch_size, chunk_len, rearranged_hidden_dim=action_dim*hidden_dim) + # + # output: predicted noise + # - shape: (batch_size, action_dim) + output = self.mlp_resnet(obs) + return output + + +class DiffusionActionHead(nn.Module): + """ + Simple MLP-based action head that generates continuous actions via conditional denoising diffusion process. + + Loosely inspired by: https://github.com/real-stanford/diffusion_policy/blob/main/diffusion_policy/model/diffusion/transformer_for_diffusion.py + """ + + def __init__( + self, + input_dim=4096, + hidden_dim=4096, + action_dim=7, + num_diffusion_steps=100, + ): + super().__init__() + self.action_dim = action_dim + self.noise_predictor = NoisePredictionModel( + transformer_hidden_dim=hidden_dim*ACTION_DIM, hidden_dim=hidden_dim, action_dim=action_dim + ) + self.noise_scheduler = DDIMScheduler(num_train_timesteps=num_diffusion_steps, beta_schedule="squaredcos_cap_v2") + self.num_diffusion_steps = num_diffusion_steps + self.time_encoder = SinusoidalPositionalEncoding(dim=hidden_dim) + + def sample_noisy_actions(self, ground_truth_actions): + """ + Samples noise and applies noise to ground-truth actions to produce noisy actions, which are + used as input in the noise prediction network. Returns noise, noisy actions, and the + corresponding diffusion timestep embeddings. + """ + # ground_truth_actions: ground-truth actions + # - shape: (batch_size, chunk_len, action_dim) + batch_size = ground_truth_actions.shape[0] + device = ground_truth_actions.device + # Sample random noise with shape equal to actions, used for closed-form forward diffusion. + noise = torch.randn(size=(batch_size, NUM_ACTIONS_CHUNK, ACTION_DIM), device=device, dtype=ground_truth_actions.dtype) # (B, chunk_len, action_dim) + # Sample random diffusion timesteps (one for each action in batch). + timesteps = torch.randint( + low=0, high=self.noise_scheduler.config.num_train_timesteps, size=(batch_size,), device=device + ) + # Add noise to clean actions according to the magnitude at each diffusion timestep via + # closed-form forward diffusion. + noisy_actions = self.noise_scheduler.add_noise(ground_truth_actions, noise, timesteps) # (B, chunk_len, action_dim) + + # Get diffusion timestep embeddings as well + diffusion_timestep_embeddings = self.time_encoder(timesteps).to(noisy_actions.dtype).to(noisy_actions.device) # (B, llm_dim) + diffusion_timestep_embeddings = diffusion_timestep_embeddings.unsqueeze(1) # (B, 1, llm_dim) + + return_dict = dict( + noise=noise, + noisy_actions=noisy_actions, + diffusion_timestep_embeddings=diffusion_timestep_embeddings, + ) + + return return_dict + + def predict_noise(self, actions_hidden_states): + """ + Given a batch of last hidden Transformer layer embeddings (which fuse the vision-language observation embeddings, + noisy action embeddings, and diffusion timestep embedding), predicts the noise applied to the actions. + """ + # actions_hidden_states: last hidden states of Transformer corresponding to action tokens in sequence + # - shape: (batch_size, chunk_len * action_dim, hidden_dim) + batch_size = actions_hidden_states.shape[0] + device = actions_hidden_states.device + rearranged_actions_hidden_states = actions_hidden_states.reshape(batch_size, NUM_ACTIONS_CHUNK, -1) # (batch_size, chunk_len, action_dim * hidden_dim) + # Get diffusion model's noise prediction. + noise_pred = self.noise_predictor(rearranged_actions_hidden_states) + return noise_pred diff --git a/prismatic/models/film_vit_wrapper.py b/prismatic/models/film_vit_wrapper.py new file mode 100644 index 000000000..94618ca96 --- /dev/null +++ b/prismatic/models/film_vit_wrapper.py @@ -0,0 +1,276 @@ +"""Implementation of additional modules for the VLA's vision transformer.""" + +from functools import partial +from typing import Any, Callable, Sequence, Tuple, Union + +import torch +import torch.nn as nn +from timm.models.vision_transformer import VisionTransformer + + +class FiLMedVisionTransformerBlock(nn.Module): + """ + Wrapper for ViT blocks that adds components to implement FiLM language conditioning. + + Modulates visual feature embeddings via + x = (1 + gamma) * x + beta, + where x is visual feature and gamma and beta are learned projections of the average language embedding. + gamma and beta have D dimensions each, where D is the number of hidden dimensions in the ViT's features. + + NOTE #1 (Moo Jin): + In convolutional neural architectures, the "feature" in FiLM is an entire feature map, i.e., each channel in a + convolutional layer (so gamma and beta have C dimensions, where C is the number of channels). Therefore, FiLM's + scaling and shifting is applied across all spatial locations for conv nets -- i.e., it is spatially agnostic. + + For vision transformer architectures, you may consider individual patch embeddings as individual "features" at first + instinct, but this would make FiLM scaling and shifting spatially local. In order to make the modulation spatially + global like in convolutional architectures, we should apply the scaling and shifting to each dimension of each patch + embedding. I.e., gamma and beta should have D dimensions, where D is the number of dimensions in a visual embedding. + + NOTE #2 (Moo Jin): + x = (1 + gamma) * x + beta is used in the original FiLM paper as opposed to x = gamma * x + beta (see section 7.2 in + https://arxiv.org/pdf/1709.07871.pdf). Since gamma and beta are close to zero upon initialization, this leads to an + identity transformation at the beginning of training, which minimizes perturbation to the pretrained representation. + """ + + def __init__( + self, + block, + vision_dim: int, + llm_dim: int, + ): + """ + Initializes FiLM ViT block wrapper. + + Args: + block (timm.models.vision_transformer.Block): Vision transformer block. + vision_dim (int): Number of hidden dimensions in visual embeddings. + llm_dim (int): Number of hidden dimensions in language embeddings. + """ + super().__init__() + self.block = block + # Initialize gamma and beta projectors + self.scale = nn.Linear(llm_dim, vision_dim) + self.shift = nn.Linear(llm_dim, vision_dim) + + def forward(self, x, average_language_embedding): + """ + Overrides the vision transformer block forward pass to use FiLM. + + Args: + x (torch.Tensor): Visual input embeddings, (batch_size, vision_seq_len, vision_dim). + average_language_embedding (torch.Tensor): Average language embedding for task, (batch_size, llm_dim). + """ + # Project average language embedding to visual embedding space to get gamma and beta + gamma = self.scale(average_language_embedding) # (batch_size, vision_dim) + beta = self.shift(average_language_embedding) # (batch_size, vision_dim) + + # Pass visual inputs through attention portion of original block + x = x + self.block.drop_path1(self.block.ls1(self.block.attn(self.block.norm1(x)))) + + # Modulate intermediate visual representations via FiLM + x = x * (1 + gamma.view(gamma.shape[0], 1, gamma.shape[1])) + beta.view(beta.shape[0], 1, beta.shape[1]) + + # Pass visual inputs through feedforward portion of original block + x = x + self.block.drop_path2(self.block.ls2(self.block.mlp(self.block.norm2(x)))) + + return x + + +class NullVisionTransformerBlockWrapper(nn.Module): + """ + Null wrapper for ViT blocks that doesn't do anything; just calls the original block's forward function. + Useful if you want to use a block wrapper every X blocks instead of every block (e.g., to reduce the number of new + parameters introduced by a new wrapper). + """ + + def __init__( + self, + block, + ): + super().__init__() + self.block = block + + def forward(self, x, average_language_embedding): + return self.block(x) + + +def unpack_tuple(fn: Callable[[Any], Tuple[Any]]) -> Callable[[Any], Any]: + """Utility function for monkey-patching functions.""" + + def wrapper(*args: Any, **kwargs: Any) -> Any: + result = fn(*args, **kwargs) + return result[0] if isinstance(result, tuple) else result + + return wrapper + + +class FiLMedVisionTransformer(VisionTransformer): + """ + Wrapper for timm.models.vision_transformer.VisionTransformer that overrides functions to enable infusing language + embeddings into visual embeddings via FiLM. + """ + + def _intermediate_layers( + self, + x: torch.Tensor, + language_embeddings: torch.Tensor, + n: Union[int, Sequence] = 1, + ): + """ + Copy of timm.models.vision_transformer.VisionTransformer._intermediate_layers() with modifications + to take in language embeddings as additional input. + """ + outputs, num_blocks = [], len(self.blocks) + take_indices = set(range(num_blocks - n, num_blocks) if isinstance(n, int) else n) + + # forward pass + x = self.patch_embed(x) + x = self._pos_embed(x) + x = self.patch_drop(x) + x = self.norm_pre(x) + for i, blk in enumerate(self.blocks): + x = blk(x, language_embeddings) # Modified to receive language_embeddings + if i in take_indices: + outputs.append(x) + + return outputs + + def get_intermediate_layers( + self, + x: torch.Tensor, + language_embeddings: torch.Tensor, + n: Union[int, Sequence] = 1, + reshape: bool = False, + return_prefix_tokens: bool = False, + norm: bool = False, + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + """ + Copy of timm.models.vision_transformer.VisionTransformer.get_intermediate_layers() with modifications + to allow language embeddings as additional input. + """ + # take last n blocks if n is an int, if in is a sequence, select by matching indices + outputs = self._intermediate_layers(x, language_embeddings, n) + if norm: + outputs = [self.norm(out) for out in outputs] + prefix_tokens = [out[:, 0 : self.num_prefix_tokens] for out in outputs] + outputs = [out[:, self.num_prefix_tokens :] for out in outputs] + + if reshape: + grid_size = self.patch_embed.grid_size + outputs = [ + out.reshape(x.shape[0], grid_size[0], grid_size[1], -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + + if return_prefix_tokens: + return tuple(zip(outputs, prefix_tokens)) + return tuple(outputs) + + +class FiLMedPrismaticVisionBackbone(nn.Module): + """ + Wrapper for OpenVLA's vision backbone that implements feature-wise linear modulation (FiLM). + + Wraps the Vision Transformers in the vision backbone to enable language conditioning through FiLM. + Supports processing 1-3 images using dual vision backbones (SigLIP + DINOv2). + """ + + def __init__( + self, + vision_backbone, + llm_dim: int = 4096, # 4096 for Llama-2 7B + ) -> None: + """ + Initializes FiLM wrapper. + + Args: + vision_backbone (PrismaticVisionBackbone): Base vision backbone. + llm_dim (int): Dimension of language model embeddings. + """ + super().__init__() + self.vision_backbone = vision_backbone + self.llm_dim = llm_dim + + # Wrap vision transformers + self._wrap_vit(self.vision_backbone.featurizer) # SigLIP + if self.vision_backbone.use_fused_vision_backbone: + self._wrap_vit(self.vision_backbone.fused_featurizer) # DINOv2 + + def _wrap_vit(self, vit) -> None: + """ + Creates wrapper around an individual vision transformer to allow for infusion of language inputs. + + Args: + vit (VisionTransformer): Original vision transformer. + """ + # Wrap vision transformer blocks + block_wrappers = [] + for block in vit.blocks: + block_wrappers.append( + FiLMedVisionTransformerBlock(block=block, vision_dim=vit.num_features, llm_dim=self.llm_dim) + ) + vit.blocks = nn.Sequential(*block_wrappers) + + # Wrap vision transformer with new class that overrides functions used for forward pass + vit.__class__ = FiLMedVisionTransformer + vit.forward = unpack_tuple(partial(vit.get_intermediate_layers, n={len(vit.blocks) - 2})) + + def get_num_patches(self) -> int: + """Returns the number of vision patches output by the vision backbone.""" + return self.vision_backbone.get_num_patches() + + def get_num_images_in_input(self) -> int: + """Returns the number of input images for the vision backbone.""" + return self.vision_backbone.get_num_images_in_input() + + def set_num_images_in_input(self, num_images_in_input: int) -> None: + """Sets the number of input images for the vision backbone.""" + self.vision_backbone.set_num_images_in_input(num_images_in_input) + + def forward(self, pixel_values: torch.Tensor, language_embeddings: torch.Tensor) -> torch.Tensor: + """ + Implements the forward pass for the vision backbone with FiLM to infuse language inputs into visual features. + + Identical to PrismaticVisionBackbone.forward() except that language embeddings are also used as input. + + Args: + pixel_values (torch.Tensor): Pixels for input image(s), (B, C, H, W). + language_embeddings (torch.Tensor): Language embeddings for the task description, (B, seq_len, llm_dim). + """ + # For FiLM: Average the language embeddings of the task description + average_language_embedding = language_embeddings.mean(dim=1) + + if self.get_num_images_in_input() == 1: + if not self.vision_backbone.use_fused_vision_backbone: + return self.vision_backbone.featurizer(pixel_values, average_language_embedding) + + # Split `pixel_values :: [bsz, 2 * 3, resolution, resolution]` =>> featurize =>> channel stack + img, img_fused = torch.split(pixel_values, [3, 3], dim=1) + patches = self.vision_backbone.featurizer(img, average_language_embedding) + patches_fused = self.vision_backbone.fused_featurizer(img_fused, average_language_embedding) + + return torch.cat([patches, patches_fused], dim=2) + + else: + assert self.vision_backbone.use_fused_vision_backbone, "Multi-image inputs require using fused backbone!" + + # Split `pixel_values` into individual images (each with 6 channels: 3 for SigLIP + 3 for DINOv2) + images = torch.split(pixel_values, [6] * self.get_num_images_in_input(), dim=1) + + # Process each image and collect patches + all_patches = [] + for img in images: + # Split each image further into two stacks of channels (each with 3 channels) + img_regular, img_fused = torch.split(img, [3, 3], dim=1) + + # Get patches from both SigLIP and DINOv2 vision transformers + patches = self.vision_backbone.featurizer(img_regular, average_language_embedding) + patches_fused = self.vision_backbone.fused_featurizer(img_fused, average_language_embedding) + + # Concatenate SigLIP and DINOv2 patches along the hidden dimension + combined_patches = torch.cat([patches, patches_fused], dim=2) + all_patches.append(combined_patches) + + # Concatenate all patches along the patch dimension + return torch.cat(all_patches, dim=1) diff --git a/prismatic/models/projectors.py b/prismatic/models/projectors.py new file mode 100644 index 000000000..ea20dade1 --- /dev/null +++ b/prismatic/models/projectors.py @@ -0,0 +1,49 @@ +"""Implementation of additional projectors for additional inputs to the VLA models.""" +import torch +import torch.nn as nn + + +class ProprioProjector(nn.Module): + """ + Projects proprio state inputs into the LLM's embedding space. + """ + def __init__(self, llm_dim: int, proprio_dim: int) -> None: + super().__init__() + self.llm_dim = llm_dim + self.proprio_dim = proprio_dim + + self.fc1 = nn.Linear(self.proprio_dim, self.llm_dim, bias=True) + self.fc2 = nn.Linear(self.llm_dim, self.llm_dim, bias=True) + self.act_fn1 = nn.GELU() + + def forward(self, proprio: torch.Tensor = None) -> torch.Tensor: + # proprio: (bsz, proprio_dim) + projected_features = self.fc1(proprio) + projected_features = self.act_fn1(projected_features) + projected_features = self.fc2(projected_features) + return projected_features + + +class NoisyActionProjector(nn.Module): + """ + [Diffusion] Projects noisy action inputs into the LLM's embedding space. + + Note that since each action is tokenized into 7 tokens in OpenVLA (rather + than having 1 token per action), each noisy action token will have dimension 1 + instead of 7. + """ + def __init__(self, llm_dim: int) -> None: + super().__init__() + self.llm_dim = llm_dim + self.action_token_dim = 1 + + self.fc1 = nn.Linear(self.action_token_dim, self.llm_dim, bias=True) + self.fc2 = nn.Linear(self.llm_dim, self.llm_dim, bias=True) + self.act_fn1 = nn.GELU() + + def forward(self, noisy_actions: torch.Tensor = None) -> torch.Tensor: + # noisy_actions: (bsz, num_action_tokens=chunk_len*action_dim, 1) + projected_features = self.fc1(noisy_actions) + projected_features = self.act_fn1(projected_features) + projected_features = self.fc2(projected_features) + return projected_features diff --git a/prismatic/training/strategies/base_strategy.py b/prismatic/training/strategies/base_strategy.py index 018ee41cf..ba4fc9428 100644 --- a/prismatic/training/strategies/base_strategy.py +++ b/prismatic/training/strategies/base_strategy.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Callable, Optional +import numpy as np import torch import torch.distributed as dist from torch.utils.data import DataLoader, Dataset, DistributedSampler, IterableDataset @@ -21,11 +22,22 @@ from prismatic.models.vlms import PrismaticVLM from prismatic.overwatch import initialize_overwatch from prismatic.training.metrics import Metrics, VLAMetrics +from prismatic.training.train_utils import ( + compute_actions_l1_loss, + compute_token_accuracy, + get_current_action_mask, + get_next_actions_mask, +) from prismatic.util import check_bloat16_supported from prismatic.util.batching_utils import SplitModalitySampler from prismatic.util.data_utils import PaddedCollatorForActionPrediction, PaddedCollatorForLanguageModeling from prismatic.vla.action_tokenizer import ActionTokenizer +# HuggingFace Default / LLaMa-2 IGNORE_INDEX (for labels) +from prismatic.vla.constants import ACTION_DIM, ACTION_TOKEN_BEGIN_IDX, NUM_ACTIONS_CHUNK, IGNORE_INDEX +NEWLINE_INDEX = 13 # '\n' +STOP_INDEX = 2 # '' + # Initialize Overwatch =>> Wraps `logging.Logger` overwatch = initialize_overwatch(__name__) @@ -300,36 +312,48 @@ def run_vla_training( metrics.commit(loss=loss) loss.backward() - # === Compute Action Token Accuracy & L1 Loss === - - # To compute action token accuracy, we need to identify the locations of the action tokens - # in both `output.logits` and `batch["labels"]`. We know that when "right" padding, we - # insert `self.vlm.vision_backbone.num_patches` at index 1. - # - # Computing `action_prediction_accuracy` is then pretty straightforward: - # 1) Extract "aligned" predictions & labels - # 2) Compute boolean "mask" where "labels > 2" (where 2 is ID for `EOS_TOKEN`) - # => If masking out EOS, then it's just "labels != -100 (IGNORE_INDEX) - # 3) Compute masked accuracy as `(preds == logits) & mask` --> sum/divide by # unmasked! - action_preds = output.logits[:, self.vlm.vision_backbone.num_patches : -1].argmax(dim=2) - action_gt = batch["labels"][:, 1:].to(action_preds.device) - mask = action_gt > action_tokenizer.action_token_begin_idx + # Get predicted and ground-truth token IDs + predicted_token_ids = output.logits[:, self.vlm.vision_backbone.num_patches : -1].argmax(dim=2) + ground_truth_token_ids = batch["labels"][:, 1:].to(predicted_token_ids.device) + + ####################################################################### + # === Compute Current Action Token Accuracy & L1 Loss === + ####################################################################### + + # Get current action mask: Target the first ACTION_DIM non-ignore tokens + current_action_mask = get_current_action_mask(ground_truth_token_ids) # Compute Accuracy - correct_preds = (action_preds == action_gt) & mask - action_accuracy = correct_preds.sum().float() / mask.sum().float() + action_accuracy = compute_token_accuracy(predicted_token_ids, ground_truth_token_ids, mask=current_action_mask) # Compute L1 Loss on Predicted (Continuous) Actions - continuous_actions_pred = torch.tensor( - action_tokenizer.decode_token_ids_to_actions(action_preds[mask].cpu().numpy()) - ) - continuous_actions_gt = torch.tensor( - action_tokenizer.decode_token_ids_to_actions(action_gt[mask].cpu().numpy()) - ) - action_l1_loss = torch.nn.functional.l1_loss(continuous_actions_pred, continuous_actions_gt) + action_l1_loss = compute_actions_l1_loss(action_tokenizer, predicted_token_ids, ground_truth_token_ids, mask=current_action_mask) + + ####################################################################### + # === Compute Next Actions Token Accuracy & L1 Loss === + ####################################################################### + + # Get next actions mask: Target all tokens after the first ACTION_DIM non-ignore tokens (excluding the last token, which is the stop token) + next_actions_mask = get_next_actions_mask(ground_truth_token_ids) + + # Compute Accuracy + next_actions_accuracy = compute_token_accuracy(predicted_token_ids, ground_truth_token_ids, mask=next_actions_mask) + + # Compute L1 Loss on Predicted (Continuous) Actions + next_actions_l1_loss = compute_actions_l1_loss(action_tokenizer, predicted_token_ids, ground_truth_token_ids, mask=next_actions_mask) + + ####################################################################### + # === Log === + ####################################################################### # Commit Metrics - metrics.commit(action_accuracy=action_accuracy, l1_loss=action_l1_loss, update_step_time=True) + metrics.commit( + action_accuracy=action_accuracy, + l1_loss=action_l1_loss, + next_actions_accuracy=next_actions_accuracy, + next_actions_l1_loss=next_actions_l1_loss, + update_step_time=True, + ) # Compute metrics per dataset --> only on rank_zero since we don't log them on other workers anyways if overwatch.is_rank_zero(): @@ -338,21 +362,25 @@ def run_vla_training( for ds in datasets: ds_mask = torch.tensor([elem == ds for elem in batch["dataset_names"]]) action_accuracy_ds = correct_preds[ds_mask].sum().float() / mask[ds_mask].sum().float() - continuous_actions_pred_ds = torch.tensor( + pred_continuous_actions_ds = torch.tensor( action_tokenizer.decode_token_ids_to_actions( - action_preds[ds_mask][mask[ds_mask]].cpu().numpy() + predicted_token_ids[ds_mask][mask[ds_mask]].cpu().numpy() ) ) continuous_actions_gt_ds = torch.tensor( action_tokenizer.decode_token_ids_to_actions( - action_gt[ds_mask][mask[ds_mask]].cpu().numpy() + ground_truth_token_ids[ds_mask][mask[ds_mask]].cpu().numpy() ) ) action_l1_loss_ds = torch.nn.functional.l1_loss( - continuous_actions_pred_ds, continuous_actions_gt_ds + pred_continuous_actions_ds, continuous_actions_gt_ds ) metrics.commit_for_dataset( - dataset_name=ds.decode(), action_accuracy=action_accuracy_ds, l1_loss=action_l1_loss_ds + dataset_name=ds.decode(), + action_accuracy=action_accuracy_ds, + l1_loss=action_l1_loss_ds, + next_actions_accuracy=next_actions_accuracy, + next_actions_l1_loss=next_actions_l1_loss, ) # === Gradient Step === diff --git a/prismatic/training/train_utils.py b/prismatic/training/train_utils.py new file mode 100644 index 000000000..0c546885d --- /dev/null +++ b/prismatic/training/train_utils.py @@ -0,0 +1,56 @@ +"""Utils for training/fine-tuning scripts.""" + +import torch + +from prismatic.vla.constants import ACTION_DIM, ACTION_TOKEN_BEGIN_IDX, IGNORE_INDEX + + +def get_current_action_mask(token_ids): + # Create a tensor marking positions of IGNORE_INDEX + newline_positions = token_ids != IGNORE_INDEX + + # Calculate cumulative sum to identify regions between newlines + cumsum = torch.cumsum(newline_positions, dim=1) + + # Create the mask + mask = (1 <= cumsum) & (cumsum <= ACTION_DIM) + + # Extract the action part only + action_tokens_only_mask = token_ids > ACTION_TOKEN_BEGIN_IDX + mask = action_tokens_only_mask * mask + + return mask + + +def get_next_actions_mask(token_ids): + # Create a tensor marking positions of IGNORE_INDEX + newline_positions = token_ids != IGNORE_INDEX + + # Calculate cumulative sum to identify regions between newlines + cumsum = torch.cumsum(newline_positions, dim=1) + + # Create the mask + mask = cumsum > ACTION_DIM + + # Extract the action part only + action_tokens_only_mask = token_ids > ACTION_TOKEN_BEGIN_IDX + mask = action_tokens_only_mask * mask + + return mask + + +def compute_token_accuracy(predicted_token_ids, ground_truth_token_ids, mask): + correct_preds = (predicted_token_ids == ground_truth_token_ids) & mask + accuracy = correct_preds.sum().float() / mask.sum().float() + return accuracy + + +def compute_actions_l1_loss(action_tokenizer, predicted_token_ids, ground_truth_token_ids, mask): + pred_continuous_actions = torch.tensor( + action_tokenizer.decode_token_ids_to_actions(predicted_token_ids[mask].cpu().numpy()) + ) + true_continuous_actions = torch.tensor( + action_tokenizer.decode_token_ids_to_actions(ground_truth_token_ids[mask].cpu().numpy()) + ) + l1_loss = torch.nn.functional.l1_loss(pred_continuous_actions, true_continuous_actions) + return l1_loss diff --git a/prismatic/util/data_utils.py b/prismatic/util/data_utils.py index cbed9603e..141dbc741 100644 --- a/prismatic/util/data_utils.py +++ b/prismatic/util/data_utils.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import Callable, Dict, Sequence, Tuple +import numpy as np import torch from torch.nn.utils.rnn import pad_sequence @@ -123,19 +124,32 @@ def __call__(self, instances: Sequence[Dict[str, torch.Tensor]]) -> Dict[str, to # Stack all `pixel_values` --> depending on type is torch.Tensor or Dict[str, torch.Tensor] if isinstance(pixel_values[0], torch.Tensor): - pixel_values = torch.stack(pixel_values) - elif isinstance(pixel_values[0], dict): - pixel_values = { - k: torch.stack([pixel_values[idx][k] for idx in range(len(input_ids))]) for k in pixel_values[0] - } + if "pixel_values_wrist" in instances[0]: + pixel_values_wrist = [instance["pixel_values_wrist"] for instance in instances] + pixel_values = torch.cat((torch.stack(pixel_values), torch.stack(pixel_values_wrist)), dim=1) + else: + pixel_values = torch.stack(pixel_values) else: raise ValueError(f"Unsupported `pixel_values` type = {type(pixel_values)}") + # Stack all actions + actions = [torch.from_numpy(np.copy(instance["actions"])) for instance in instances] + actions = torch.stack(actions) + + # Stack proprio + if "proprio" in instances[0]: + proprio = [instance["proprio"] for instance in instances] + proprio = torch.Tensor(np.squeeze(np.stack(proprio))) + else: + proprio = None + output = dict( pixel_values=pixel_values, + proprio=proprio, input_ids=input_ids, attention_mask=attention_mask, labels=labels, + actions=actions, ) if dataset_names is not None: output["dataset_names"] = dataset_names diff --git a/prismatic/vla/constants.py b/prismatic/vla/constants.py new file mode 100644 index 000000000..229ae944a --- /dev/null +++ b/prismatic/vla/constants.py @@ -0,0 +1,86 @@ +""" +Important constants for VLA training and evaluation. + +Attempts to automatically identify the correct constants to set based on the Python command used to launch +training or evaluation. If it is unclear, defaults to using the LIBERO simulation benchmark constants. +""" +import sys +from enum import Enum + +# Llama 2 token constants +IGNORE_INDEX = -100 +ACTION_TOKEN_BEGIN_IDX = 31743 +STOP_INDEX = 2 # '' + + +# Defines supported normalization schemes for action and proprioceptive state. +class NormalizationType(str, Enum): + # fmt: off + NORMAL = "normal" # Normalize to Mean = 0, Stdev = 1 + BOUNDS = "bounds" # Normalize to Interval = [-1, 1] + BOUNDS_Q99 = "bounds_q99" # Normalize [quantile_01, ..., quantile_99] --> [-1, ..., 1] + # fmt: on + + +# Define constants for each robot platform +LIBERO_CONSTANTS = { + "NUM_ACTIONS_CHUNK": 8, + "ACTION_DIM": 7, + "PROPRIO_DIM": 8, + "ACTION_PROPRIO_NORMALIZATION_TYPE": NormalizationType.BOUNDS_Q99, +} + +ALOHA_CONSTANTS = { + "NUM_ACTIONS_CHUNK": 25, + "ACTION_DIM": 14, + "PROPRIO_DIM": 14, + "ACTION_PROPRIO_NORMALIZATION_TYPE": NormalizationType.BOUNDS, +} + +BRIDGE_CONSTANTS = { + "NUM_ACTIONS_CHUNK": 5, + "ACTION_DIM": 7, + "PROPRIO_DIM": 7, + "ACTION_PROPRIO_NORMALIZATION_TYPE": NormalizationType.BOUNDS_Q99, +} + + +# Function to detect robot platform from command line arguments +def detect_robot_platform(): + cmd_args = " ".join(sys.argv).lower() + + if "libero" in cmd_args: + return "LIBERO" + elif "aloha" in cmd_args: + return "ALOHA" + elif "bridge" in cmd_args: + return "BRIDGE" + else: + # Default to LIBERO if unclear + return "LIBERO" + + +# Determine which robot platform to use +ROBOT_PLATFORM = detect_robot_platform() + +# Set the appropriate constants based on the detected platform +if ROBOT_PLATFORM == "LIBERO": + constants = LIBERO_CONSTANTS +elif ROBOT_PLATFORM == "ALOHA": + constants = ALOHA_CONSTANTS +elif ROBOT_PLATFORM == "BRIDGE": + constants = BRIDGE_CONSTANTS + +# Assign constants to global variables +NUM_ACTIONS_CHUNK = constants["NUM_ACTIONS_CHUNK"] +ACTION_DIM = constants["ACTION_DIM"] +PROPRIO_DIM = constants["PROPRIO_DIM"] +ACTION_PROPRIO_NORMALIZATION_TYPE = constants["ACTION_PROPRIO_NORMALIZATION_TYPE"] + +# Print which robot platform constants are being used (for debugging) +print(f"Using {ROBOT_PLATFORM} constants:") +print(f" NUM_ACTIONS_CHUNK = {NUM_ACTIONS_CHUNK}") +print(f" ACTION_DIM = {ACTION_DIM}") +print(f" PROPRIO_DIM = {PROPRIO_DIM}") +print(f" ACTION_PROPRIO_NORMALIZATION_TYPE = {ACTION_PROPRIO_NORMALIZATION_TYPE}") +print("If needed, manually set the correct constants in `prismatic/vla/constants.py`!") diff --git a/prismatic/vla/datasets/datasets.py b/prismatic/vla/datasets/datasets.py index 539b4144d..06cadbf94 100644 --- a/prismatic/vla/datasets/datasets.py +++ b/prismatic/vla/datasets/datasets.py @@ -19,13 +19,9 @@ from prismatic.models.backbones.vision import ImageTransform from prismatic.util.data_utils import tree_map from prismatic.vla.action_tokenizer import ActionTokenizer +from prismatic.vla.constants import ACTION_DIM, ACTION_PROPRIO_NORMALIZATION_TYPE, ACTION_TOKEN_BEGIN_IDX, IGNORE_INDEX, NUM_ACTIONS_CHUNK, PROPRIO_DIM, STOP_INDEX from prismatic.vla.datasets.rlds import make_interleaved_dataset, make_single_dataset from prismatic.vla.datasets.rlds.oxe import OXE_NAMED_MIXTURES, get_oxe_dataset_kwargs_and_weights -from prismatic.vla.datasets.rlds.utils.data_utils import NormalizationType - -# HuggingFace Default / LLaMa-2 IGNORE_INDEX (for labels) -IGNORE_INDEX = -100 - @dataclass class RLDSBatchTransform: @@ -34,18 +30,31 @@ class RLDSBatchTransform: image_transform: ImageTransform prompt_builder_fn: Type[PromptBuilder] predict_stop_token: bool = True + use_wrist_image: bool = False + use_proprio: bool = False def __call__(self, rlds_batch: Dict[str, Any]) -> Dict[str, Any]: """Converts a RLDS batch to the format expected by the OpenVLA collator/models.""" - dataset_name, action = rlds_batch["dataset_name"], rlds_batch["action"][0] + dataset_name, current_action = rlds_batch["dataset_name"], rlds_batch["action"][0] img = Image.fromarray(rlds_batch["observation"]["image_primary"][0]) lang = rlds_batch["task"]["language_instruction"].decode().lower() + actions = rlds_batch["action"] # Construct Chat-based Prompt =>> Input is default query + language instruction, output are the action tokens prompt_builder = self.prompt_builder_fn("openvla") + + # Get future action chunk + future_actions = rlds_batch["action"][1:] + future_actions_string = ''.join(self.action_tokenizer(future_actions)) + + # Get action chunk string + current_action_string = self.action_tokenizer(current_action) + action_chunk_string = current_action_string + future_actions_string + action_chunk_len = len(action_chunk_string) + conversation = [ {"from": "human", "value": f"What action should the robot take to {lang}?"}, - {"from": "gpt", "value": self.action_tokenizer(action)}, + {"from": "gpt", "value": action_chunk_string}, ] for turn in conversation: prompt_builder.add_turn(turn["from"], turn["value"]) @@ -60,11 +69,26 @@ def __call__(self, rlds_batch: Dict[str, Any]) -> Dict[str, Any]: pixel_values = self.image_transform(img) # [CRITICAL] We do not want to take the loss for anything but the predicted action tokens! - labels[: -(len(action) + 1)] = IGNORE_INDEX + labels[: -(action_chunk_len + 1)] = IGNORE_INDEX if not self.predict_stop_token: labels[-1] = IGNORE_INDEX - return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels, dataset_name=dataset_name) + return_dict = dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels, dataset_name=dataset_name, actions=actions) + + # Add additional inputs + if self.use_wrist_image: + all_wrist_pixels = [] + for k in rlds_batch["observation"].keys(): + if "wrist" in k: + img_wrist = Image.fromarray(rlds_batch["observation"][k][0]) + pixel_values_wrist = self.image_transform(img_wrist) + all_wrist_pixels.append(pixel_values_wrist) + return_dict["pixel_values_wrist"] = torch.cat(all_wrist_pixels, dim=0) + if self.use_proprio and "proprio" in rlds_batch["observation"]: + proprio = rlds_batch["observation"]["proprio"] + return_dict["proprio"] = proprio + + return return_dict class RLDSDataset(IterableDataset): @@ -89,19 +113,24 @@ def __init__( mixture_spec = [(self.data_mix, 1.0)] # fmt: off + if "aloha" in self.data_mix: + load_camera_views = ("primary", "left_wrist", "right_wrist") + else: + load_camera_views = ("primary", "wrist") + per_dataset_kwargs, weights = get_oxe_dataset_kwargs_and_weights( self.data_root_dir, mixture_spec, - load_camera_views=("primary",), + load_camera_views=load_camera_views, load_depth=False, - load_proprio=False, + load_proprio=True, load_language=True, - action_proprio_normalization_type=NormalizationType.BOUNDS_Q99, + action_proprio_normalization_type=ACTION_PROPRIO_NORMALIZATION_TYPE, ) rlds_config = dict( traj_transform_kwargs=dict( window_size=1, # If we wanted to feed / predict more than one step - future_action_window_size=0, # For action chunking + future_action_window_size=NUM_ACTIONS_CHUNK-1, # For action chunking skip_unlabeled=True, # Skip trajectories without language labels goal_relabeling_strategy="uniform", # Goals are currently unused ), diff --git a/prismatic/vla/datasets/rlds/dataset.py b/prismatic/vla/datasets/rlds/dataset.py index e9bcd93eb..f07215a2d 100644 --- a/prismatic/vla/datasets/rlds/dataset.py +++ b/prismatic/vla/datasets/rlds/dataset.py @@ -16,10 +16,10 @@ import tensorflow_datasets as tfds from prismatic.overwatch import initialize_overwatch +from prismatic.vla.constants import ACTION_DIM, ACTION_PROPRIO_NORMALIZATION_TYPE, ACTION_TOKEN_BEGIN_IDX, IGNORE_INDEX, NUM_ACTIONS_CHUNK, PROPRIO_DIM, STOP_INDEX from prismatic.vla.datasets.rlds import obs_transforms, traj_transforms from prismatic.vla.datasets.rlds.utils import goal_relabeling, task_augmentation from prismatic.vla.datasets.rlds.utils.data_utils import ( - NormalizationType, allocate_threads, get_dataset_statistics, normalize_action_and_proprio, @@ -47,7 +47,7 @@ def make_dataset_from_rlds( depth_obs_keys: Dict[str, Optional[str]] = {}, state_obs_keys: List[Optional[str]] = (), language_key: Optional[str] = None, - action_proprio_normalization_type: NormalizationType = NormalizationType.NORMAL, + action_proprio_normalization_type: ACTION_PROPRIO_NORMALIZATION_TYPE, dataset_statistics: Optional[Union[dict, str]] = None, absolute_action_mask: Optional[List[bool]] = None, action_normalization_mask: Optional[List[bool]] = None, @@ -231,10 +231,7 @@ def restructure(traj): dataset_statistics["action"]["mask"] = np.array(action_normalization_mask) # construct the dataset - if "val" not in builder.info.splits: - split = "train[:95%]" if train else "train[95%:]" - else: - split = "train" if train else "val" + split = "train" if train else "val" dataset = dl.DLataset.from_rlds(builder, split=split, shuffle=shuffle, num_parallel_reads=num_parallel_reads) diff --git a/prismatic/vla/datasets/rlds/oxe/configs.py b/prismatic/vla/datasets/rlds/oxe/configs.py index 2b8dcb931..b8ab2b785 100644 --- a/prismatic/vla/datasets/rlds/oxe/configs.py +++ b/prismatic/vla/datasets/rlds/oxe/configs.py @@ -72,21 +72,21 @@ class ActionEncoding(IntEnum): "bridge_oxe": { # Version of Bridge V2 in Open X-Embodiment mixture "image_obs_keys": {"primary": "image", "secondary": "image_1", "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "bridge_orig": { # Original version of Bridge V2 from project website "image_obs_keys": {"primary": "image_0", "secondary": "image_1", "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "bridge_dataset": { # Original version of Bridge V2 from project website "image_obs_keys": {"primary": "image_0", "secondary": "image_1", "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -198,7 +198,7 @@ class ActionEncoding(IntEnum): "nyu_rot_dataset_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -209,7 +209,7 @@ class ActionEncoding(IntEnum): "wrist": "wrist_image", }, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -286,7 +286,7 @@ class ActionEncoding(IntEnum): "ucsd_pick_and_place_dataset_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -327,14 +327,14 @@ class ActionEncoding(IntEnum): "utokyo_pr2_opening_fridge_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "utokyo_pr2_tabletop_manipulation_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -359,7 +359,7 @@ class ActionEncoding(IntEnum): "robo_net": { "image_obs_keys": {"primary": "image", "secondary": "image1", "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -387,14 +387,14 @@ class ActionEncoding(IntEnum): "stanford_mask_vit_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "tokyo_u_lsmo_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -422,14 +422,14 @@ class ActionEncoding(IntEnum): "asu_table_top_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "stanford_robocook_converted_externally_to_rlds": { "image_obs_keys": {"primary": "image_1", "secondary": "image_2", "wrist": None}, "depth_obs_keys": {"primary": "depth_1", "secondary": "depth_2", "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -505,7 +505,7 @@ class ActionEncoding(IntEnum): "cmu_stretch": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["eef_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -594,42 +594,42 @@ class ActionEncoding(IntEnum): "tdroid_carrot_in_bowl": { # "put carrot in bowl" task, 50 demos @ 5 Hz control "image_obs_keys": {"primary": "static_image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": "static_depth_image", "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "tdroid_pour_corn_in_pot": { # "pour corn from red bowl into steel pot" task, 50 demos @ 5 Hz control "image_obs_keys": {"primary": "static_image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": "static_depth_image", "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "tdroid_flip_pot_upright": { # "flip pot upright" task, 10 demos @ 5 Hz control "image_obs_keys": {"primary": "static_image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": "static_depth_image", "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "tdroid_move_object_onto_plate": { # "move onto plate" task, 150 demos @ 5 Hz control "image_obs_keys": {"primary": "static_image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": "static_depth_image", "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "tdroid_knock_object_over": { # "knock over" task, 70 demos @ 5 Hz control "image_obs_keys": {"primary": "static_image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": "static_depth_image", "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "tdroid_cover_object_with_towel": { # "cover with towel" task, 45 demos @ 5 Hz control "image_obs_keys": {"primary": "static_image", "secondary": None, "wrist": None}, "depth_obs_keys": {"primary": "static_depth_image", "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, @@ -645,29 +645,58 @@ class ActionEncoding(IntEnum): "libero_spatial_no_noops": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": "wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "libero_object_no_noops": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": "wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "libero_goal_no_noops": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": "wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, "libero_10_no_noops": { "image_obs_keys": {"primary": "image", "secondary": None, "wrist": "wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, - "state_obs_keys": ["EEF_state", None, "gripper_state"], + "state_obs_keys": ["EEF_state", "gripper_state"], "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, + ### ALOHA fine-tuning datasets + "openvla_oft_aloha_fold_shorts_20_demos": { + "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, + "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, + "state_obs_keys": ["state"], + "state_encoding": StateEncoding.JOINT_BIMANUAL, + "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, + }, + "openvla_oft_aloha_fold_shirt_30_demos": { + "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, + "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, + "state_obs_keys": ["state"], + "state_encoding": StateEncoding.JOINT_BIMANUAL, + "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, + }, + "openvla_oft_aloha_scoop_x_into_bowl_45_demos": { + "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, + "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, + "state_obs_keys": ["state"], + "state_encoding": StateEncoding.JOINT_BIMANUAL, + "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, + }, + "openvla_oft_aloha_put_x_into_pot_300_demos": { + "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, + "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, + "state_obs_keys": ["state"], + "state_encoding": StateEncoding.JOINT_BIMANUAL, + "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, + }, } diff --git a/prismatic/vla/datasets/rlds/oxe/materialize.py b/prismatic/vla/datasets/rlds/oxe/materialize.py index 56d0d38fe..fd4103d8d 100644 --- a/prismatic/vla/datasets/rlds/oxe/materialize.py +++ b/prismatic/vla/datasets/rlds/oxe/materialize.py @@ -10,9 +10,9 @@ from typing import Any, Dict, List, Tuple from prismatic.overwatch import initialize_overwatch +from prismatic.vla.constants import ACTION_DIM, ACTION_PROPRIO_NORMALIZATION_TYPE, ACTION_TOKEN_BEGIN_IDX, IGNORE_INDEX, NUM_ACTIONS_CHUNK, PROPRIO_DIM, STOP_INDEX from prismatic.vla.datasets.rlds.oxe.configs import OXE_DATASET_CONFIGS, ActionEncoding from prismatic.vla.datasets.rlds.oxe.transforms import OXE_STANDARDIZATION_TRANSFORMS -from prismatic.vla.datasets.rlds.utils.data_utils import NormalizationType # Initialize Overwatch =>> Wraps `logging.Logger` overwatch = initialize_overwatch(__name__) @@ -25,12 +25,12 @@ def make_oxe_dataset_kwargs( load_depth: bool = False, load_proprio: bool = True, load_language: bool = True, - action_proprio_normalization_type: NormalizationType = NormalizationType.NORMAL, + action_proprio_normalization_type = ACTION_PROPRIO_NORMALIZATION_TYPE, ) -> Dict[str, Any]: """Generates config (kwargs) for given dataset from Open-X Embodiment.""" dataset_kwargs = deepcopy(OXE_DATASET_CONFIGS[dataset_name]) - if dataset_kwargs["action_encoding"] not in [ActionEncoding.EEF_POS, ActionEncoding.EEF_R6]: - raise ValueError(f"Cannot load `{dataset_name}`; only EEF_POS & EEF_R6 actions supported!") + if dataset_kwargs["action_encoding"] not in [ActionEncoding.EEF_POS, ActionEncoding.EEF_R6, ActionEncoding.JOINT_POS_BIMANUAL]: + raise ValueError(f"Cannot load `{dataset_name}`; only EEF_POS & EEF_R6 & JOINT_POS_BIMANUAL actions supported!") # [Contract] For EEF_POS & EEF_R6 actions, only the last action dimension (gripper) is absolute! # Normalize all action dimensions *except* the gripper @@ -40,6 +40,9 @@ def make_oxe_dataset_kwargs( elif dataset_kwargs["action_encoding"] is ActionEncoding.EEF_R6: dataset_kwargs["absolute_action_mask"] = [False] * 9 + [True] dataset_kwargs["action_normalization_mask"] = [True] * 9 + [False] + elif dataset_kwargs["action_encoding"] is ActionEncoding.JOINT_POS_BIMANUAL: + dataset_kwargs["absolute_action_mask"] = [True] * 14 + dataset_kwargs["action_normalization_mask"] = [True] * 14 dataset_kwargs["action_proprio_normalization_type"] = action_proprio_normalization_type # Adjust Loaded Camera Views @@ -83,7 +86,7 @@ def get_oxe_dataset_kwargs_and_weights( load_depth: bool = False, load_proprio: bool = True, load_language: bool = True, - action_proprio_normalization_type: NormalizationType = NormalizationType.NORMAL, + action_proprio_normalization_type = ACTION_PROPRIO_NORMALIZATION_TYPE, ) -> Tuple[Dict[str, Any], List[float]]: """ Generates dataset kwargs for a given dataset mix from the Open X-Embodiment dataset. The returned kwargs diff --git a/prismatic/vla/datasets/rlds/oxe/mixtures.py b/prismatic/vla/datasets/rlds/oxe/mixtures.py index aca03da44..feb47c8a0 100644 --- a/prismatic/vla/datasets/rlds/oxe/mixtures.py +++ b/prismatic/vla/datasets/rlds/oxe/mixtures.py @@ -206,5 +206,19 @@ "libero_10_no_noops": [ ("libero_10_no_noops", 1.0), ], -} + + # === ALOHA Fine-Tuning Datasets === + "openvla_oft_aloha_fold_shorts_20_demos": [ + ("openvla_oft_aloha_fold_shorts_20_demos", 1.0), + ], + "openvla_oft_aloha_fold_shirt_30_demos": [ + ("openvla_oft_aloha_fold_shirt_30_demos", 1.0), + ], + "openvla_oft_aloha_scoop_x_into_bowl_45_demos": [ + ("openvla_oft_aloha_scoop_x_into_bowl_45_demos", 1.0), + ], + "openvla_oft_aloha_put_x_into_pot_300_demos": [ + ("openvla_oft_aloha_put_x_into_pot_300_demos", 1.0), + ], # fmt: on +} diff --git a/prismatic/vla/datasets/rlds/oxe/transforms.py b/prismatic/vla/datasets/rlds/oxe/transforms.py index cc9c68712..405f80b86 100644 --- a/prismatic/vla/datasets/rlds/oxe/transforms.py +++ b/prismatic/vla/datasets/rlds/oxe/transforms.py @@ -841,6 +841,11 @@ def libero_dataset_transform(trajectory: Dict[str, Any]) -> Dict[str, Any]: return trajectory +def aloha_dataset_transform(trajectory: Dict[str, Any]) -> Dict[str, Any]: + # Don't need to do anything because dataset is already in the correct format + return trajectory + + # === Registry === OXE_STANDARDIZATION_TRANSFORMS = { "bridge_oxe": bridge_oxe_dataset_transform, @@ -919,4 +924,9 @@ def libero_dataset_transform(trajectory: Dict[str, Any]) -> Dict[str, Any]: "libero_object_no_noops": libero_dataset_transform, "libero_goal_no_noops": libero_dataset_transform, "libero_10_no_noops": libero_dataset_transform, + ### ALOHA fine-tuning datasets + "openvla_oft_aloha_fold_shorts_20_demos": aloha_dataset_transform, + "openvla_oft_aloha_fold_shirt_30_demos": aloha_dataset_transform, + "openvla_oft_aloha_scoop_x_into_bowl_45_demos": aloha_dataset_transform, + "openvla_oft_aloha_put_x_into_pot_300_demos": aloha_dataset_transform, } diff --git a/prismatic/vla/datasets/rlds/traj_transforms.py b/prismatic/vla/datasets/rlds/traj_transforms.py index 82cc43a2e..d2ae695ab 100644 --- a/prismatic/vla/datasets/rlds/traj_transforms.py +++ b/prismatic/vla/datasets/rlds/traj_transforms.py @@ -24,24 +24,22 @@ def chunk_act_obs(traj: Dict, window_size: int, future_action_window_size: int = """ traj_len = tf.shape(traj["action"])[0] action_dim = traj["action"].shape[-1] - chunk_indices = tf.broadcast_to(tf.range(-window_size + 1, 1), [traj_len, window_size]) + tf.broadcast_to( - tf.range(traj_len)[:, None], [traj_len, window_size] + effective_traj_len = traj_len - future_action_window_size + chunk_indices = tf.broadcast_to(tf.range(-window_size + 1, 1), [effective_traj_len, window_size]) + tf.broadcast_to( + tf.range(effective_traj_len)[:, None], [effective_traj_len, window_size] ) action_chunk_indices = tf.broadcast_to( tf.range(-window_size + 1, 1 + future_action_window_size), - [traj_len, window_size + future_action_window_size], + [effective_traj_len, window_size + future_action_window_size], ) + tf.broadcast_to( - tf.range(traj_len)[:, None], - [traj_len, window_size + future_action_window_size], + tf.range(effective_traj_len)[:, None], + [effective_traj_len, window_size + future_action_window_size], ) floored_chunk_indices = tf.maximum(chunk_indices, 0) - if "timestep" in traj["task"]: - goal_timestep = traj["task"]["timestep"] - else: - goal_timestep = tf.fill([traj_len], traj_len - 1) + goal_timestep = tf.fill([effective_traj_len], traj_len - 1) floored_action_chunk_indices = tf.minimum(tf.maximum(action_chunk_indices, 0), goal_timestep[:, None]) @@ -51,22 +49,10 @@ def chunk_act_obs(traj: Dict, window_size: int, future_action_window_size: int = # indicates whether an entire observation is padding traj["observation"]["pad_mask"] = chunk_indices >= 0 - # if no absolute_action_mask was provided, assume all actions are relative - if "absolute_action_mask" not in traj and future_action_window_size > 0: - logging.warning( - "future_action_window_size > 0 but no absolute_action_mask was provided. " - "Assuming all actions are relative for the purpose of making neutral actions." - ) - absolute_action_mask = traj.get("absolute_action_mask", tf.zeros([traj_len, action_dim], dtype=tf.bool)) - neutral_actions = tf.where( - absolute_action_mask[:, None, :], - traj["action"], # absolute actions are repeated (already done during chunking) - tf.zeros_like(traj["action"]), # relative actions are zeroed - ) - - # actions past the goal timestep become neutral - action_past_goal = action_chunk_indices > goal_timestep[:, None] - traj["action"] = tf.where(action_past_goal[:, :, None], neutral_actions, traj["action"]) + # Truncate other elements of the trajectory dict + traj["task"] = tf.nest.map_structure(lambda x: tf.gather(x, tf.range(effective_traj_len)), traj["task"]) + traj["dataset_name"] = tf.gather(traj["dataset_name"], tf.range(effective_traj_len)) + traj["absolute_action_mask"] = tf.gather(traj["absolute_action_mask"], tf.range(effective_traj_len)) return traj diff --git a/prismatic/vla/datasets/rlds/utils/data_utils.py b/prismatic/vla/datasets/rlds/utils/data_utils.py index 7b0e5ae9c..41b61bd12 100644 --- a/prismatic/vla/datasets/rlds/utils/data_utils.py +++ b/prismatic/vla/datasets/rlds/utils/data_utils.py @@ -7,7 +7,6 @@ import hashlib import json import os -from enum import Enum from typing import Any, Callable, Dict, List, Optional, Tuple import dlimp as dl @@ -16,6 +15,7 @@ from tqdm import tqdm from prismatic.overwatch import initialize_overwatch +from prismatic.vla.constants import NormalizationType # Initialize Overwatch =>> Wraps `logging.Logger` overwatch = initialize_overwatch(__name__) @@ -45,15 +45,6 @@ def to_padding(tensor: tf.Tensor) -> tf.Tensor: raise ValueError(f"Cannot generate padding for tensor of type {tensor.dtype}.") -# Defines supported normalization schemes for action and proprioceptive state. -class NormalizationType(str, Enum): - # fmt: off - NORMAL = "normal" # Normalize to Mean = 0, Stdev = 1 - BOUNDS = "bounds" # Normalize to Interval = [-1, 1] - BOUNDS_Q99 = "bounds_q99" # Normalize [quantile_01, ..., quantile_99] --> [-1, ..., 1] - # fmt: on - - # === State / Action Processing Primitives === diff --git a/pyproject.toml b/pyproject.toml index f72cae071..562e9ba27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,17 +3,17 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "openvla" +name = "openvla-oft" authors = [ {name = "Moo Jin Kim", email="moojink@stanford.edu"}, - {name = "Karl Pertsch", email="pertsch@berkeley.edu"}, - {name = "Siddharth Karamcheti", email="skaramcheti@cs.stanford.edu"}, + {name = "Chelsea Finn", email="cbfinn@cs.stanford.edu"}, + {name = "Percy Liang", email="pliang@cs.stanford.edu"}, ] -description = "OpenVLA: Vision-Language-Action Models for Robotics" -version = "0.0.3" +description = "Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success" +version = "0.0.1" readme = "README.md" requires-python = ">=3.8" -keywords = ["vision-language-actions models", "multimodal pretraining", "robot learning"] +keywords = ["vision-language-actions models", "fine-tuning", "robot learning"] license = {file = "LICENSE"} classifiers = [ "Development Status :: 3 - Alpha", @@ -47,12 +47,17 @@ dependencies = [ "torch==2.2.0", "torchvision==0.17.0", "torchaudio==2.2.0", - "transformers==4.40.1", + "transformers @ git+https://github.com/moojink/transformers-openvla-oft.git", # IMPORTANT: Use this fork for bidirectional attn (for parallel decoding) "wandb", "tensorflow==2.15.0", "tensorflow_datasets==4.9.3", "tensorflow_graphics==2021.12.3", - "dlimp @ git+https://github.com/moojink/dlimp_openvla" + "dlimp @ git+https://github.com/moojink/dlimp_openvla", + "diffusers", + "imageio", + "uvicorn", + "fastapi", + "json-numpy", ] [project.optional-dependencies] @@ -69,9 +74,9 @@ sagemaker = [ ] [project.urls] -homepage = "https://github.com/openvla/openvla" -repository = "https://github.com/openvla/openvla" -documentation = "https://github.com/openvla/openvla" +homepage = "https://github.com/moojink/openvla-oft" +repository = "https://github.com/moojink/openvla-oft" +documentation = "https://github.com/moojink/openvla-oft" [tool.setuptools.packages.find] where = ["."] diff --git a/vla-scripts/deploy.py b/vla-scripts/deploy.py index c70a9f279..def1bc373 100644 --- a/vla-scripts/deploy.py +++ b/vla-scripts/deploy.py @@ -1,30 +1,7 @@ """ deploy.py -Provide a lightweight server/client implementation for deploying OpenVLA models (through the HF AutoClass API) over a -REST API. This script implements *just* the server, with specific dependencies and instructions below. - -Note that for the *client*, usage just requires numpy/json-numpy, and requests; example usage below! - -Dependencies: - => Server (runs OpenVLA model on GPU): `pip install uvicorn fastapi json-numpy` - => Client: `pip install requests json-numpy` - -Client (Standalone) Usage (assuming a server running on 0.0.0.0:8000): - -``` -import requests -import json_numpy -json_numpy.patch() -import numpy as np - -action = requests.post( - "http://0.0.0.0:8000/act", - json={"image": np.zeros((256, 256, 3), dtype=np.uint8), "instruction": "do something"} -).json() - -Note that if your server is not accessible on the open web, you can use ngrok, or forward ports to your client via ssh: - => `ssh -L 8000:localhost:8000 ssh USER@` +Starts VLA server which the client can query to get robot actions. """ import os.path @@ -35,6 +12,7 @@ json_numpy.patch() import json import logging +import numpy as np import traceback from dataclasses import dataclass from pathlib import Path @@ -48,61 +26,69 @@ from PIL import Image from transformers import AutoModelForVision2Seq, AutoProcessor -# === Utilities === -SYSTEM_PROMPT = ( - "A chat between a curious user and an artificial intelligence assistant. " - "The assistant gives helpful, detailed, and polite answers to the user's questions." +from experiments.robot.openvla_utils import ( + get_vla, + get_vla_action, + get_action_head, + get_processor, + get_proprio_projector, +) +from experiments.robot.robot_utils import ( + get_image_resize_size, ) +from prismatic.vla.constants import ACTION_DIM, ACTION_TOKEN_BEGIN_IDX, IGNORE_INDEX, NUM_ACTIONS_CHUNK, PROPRIO_DIM, STOP_INDEX def get_openvla_prompt(instruction: str, openvla_path: Union[str, Path]) -> str: - if "v01" in openvla_path: - return f"{SYSTEM_PROMPT} USER: What action should the robot take to {instruction.lower()}? ASSISTANT:" - else: - return f"In: What action should the robot take to {instruction.lower()}?\nOut:" + return f"In: What action should the robot take to {instruction.lower()}?\nOut:" # === Server Interface === class OpenVLAServer: - def __init__(self, openvla_path: Union[str, Path], attn_implementation: Optional[str] = "flash_attention_2") -> Path: + def __init__(self, cfg) -> Path: """ - A simple server for OpenVLA models; exposes `/act` to predict an action for a given image + instruction. - => Takes in {"image": np.ndarray, "instruction": str, "unnorm_key": Optional[str]} - => Returns {"action": np.ndarray} + A simple server for OpenVLA models; exposes `/act` to predict an action for a given observation + instruction. """ - self.openvla_path, self.attn_implementation = openvla_path, attn_implementation - self.device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") - - # Load VLA Model using HF AutoClasses - self.processor = AutoProcessor.from_pretrained(self.openvla_path, trust_remote_code=True) - self.vla = AutoModelForVision2Seq.from_pretrained( - self.openvla_path, - attn_implementation=attn_implementation, - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - trust_remote_code=True, - ).to(self.device) - - # [Hacky] Load Dataset Statistics from Disk (if passing a path to a fine-tuned model) - if os.path.isdir(self.openvla_path): - with open(Path(self.openvla_path) / "dataset_statistics.json", "r") as f: - self.vla.norm_stats = json.load(f) - - def predict_action(self, payload: Dict[str, Any]) -> str: + self.cfg = cfg + + # Load model + self.vla = get_vla(cfg) + + # Load proprio projector + self.proprio_projector = None + if cfg.use_proprio: + self.proprio_projector = get_proprio_projector(cfg, self.vla.llm_dim, PROPRIO_DIM) + + # Load continuous action head + self.action_head = None + if cfg.use_l1_regression or cfg.use_diffusion: + self.action_head = get_action_head(cfg, self.vla.llm_dim) + + # Check that the model contains the action un-normalization key + assert cfg.unnorm_key in self.vla.norm_stats, f"Action un-norm key {cfg.unnorm_key} not found in VLA `norm_stats`!" + + # Get Hugging Face processor + self.processor = None + self.processor = get_processor(cfg) + + # Get expected image dimensions + self.resize_size = get_image_resize_size(cfg) + + + def get_server_action(self, payload: Dict[str, Any]) -> str: try: if double_encode := "encoded" in payload: # Support cases where `json_numpy` is hard to install, and numpy arrays are "double-encoded" as strings assert len(payload.keys()) == 1, "Only uses encoded payload!" payload = json.loads(payload["encoded"]) - # Parse payload components - image, instruction = payload["image"], payload["instruction"] - unnorm_key = payload.get("unnorm_key", None) + observation = payload + instruction = observation["instruction"] + + action = get_vla_action( + self.cfg, self.vla, self.processor, observation, instruction, action_head=self.action_head, proprio_projector=self.proprio_projector, use_film=self.cfg.use_film, + ) - # Run VLA Inference - prompt = get_openvla_prompt(instruction, self.openvla_path) - inputs = self.processor(prompt, Image.fromarray(image).convert("RGB")).to(self.device, dtype=torch.bfloat16) - action = self.vla.predict_action(**inputs, unnorm_key=unnorm_key, do_sample=False) if double_encode: return JSONResponse(json_numpy.dumps(action)) else: @@ -111,33 +97,56 @@ def predict_action(self, payload: Dict[str, Any]) -> str: logging.error(traceback.format_exc()) logging.warning( "Your request threw an error; make sure your request complies with the expected format:\n" - "{'image': np.ndarray, 'instruction': str}\n" - "You can optionally an `unnorm_key: str` to specific the dataset statistics you want to use for " - "de-normalizing the output actions." + "{'observation': dict, 'instruction': str}\n" ) return "error" - def run(self, host: str = "0.0.0.0", port: int = 8000) -> None: + def run(self, host: str = "0.0.0.0", port: int = 8777) -> None: self.app = FastAPI() - self.app.post("/act")(self.predict_action) + self.app.post("/act")(self.get_server_action) uvicorn.run(self.app, host=host, port=port) @dataclass class DeployConfig: # fmt: off - openvla_path: Union[str, Path] = "openvla/openvla-7b" # HF Hub Path (or path to local run directory) # Server Configuration host: str = "0.0.0.0" # Host IP Address - port: int = 8000 # Host Port - + port: int = 8777 # Host Port + + ################################################################################################################# + # Model-specific parameters + ################################################################################################################# + model_family: str = "openvla" # Model family + pretrained_checkpoint: Union[str, Path] = "" # Pretrained checkpoint path + + use_l1_regression: bool = True # If True, uses continuous action head with L1 regression objective + use_diffusion: bool = False # If True, uses continuous action head with diffusion modeling objective (DDIM) + num_diffusion_steps: int = 50 # (When `diffusion==True`) Number of diffusion steps for inference + use_film: bool = False # If True, uses FiLM to infuse language inputs into visual features + num_images_in_input: int = 3 # Number of images in the VLA input (default: 3) + use_proprio: bool = True # Whether to include proprio state in input + + center_crop: bool = True # Center crop? (if trained w/ random crop image aug) + num_open_loop_steps: int = 25 # Number of actions to execute open-loop before requerying policy + + unnorm_key: Union[str, Path] = "" # Action un-normalization key + use_relative_actions: bool = False # Whether to use relative actions (delta joint angles) + + load_in_8bit: bool = False # (For OpenVLA only) Load with 8-bit quantization + load_in_4bit: bool = False # (For OpenVLA only) Load with 4-bit quantization + + ################################################################################################################# + # Utils + ################################################################################################################# + seed: int = 7 # Random Seed (for reproducibility) # fmt: on @draccus.wrap() def deploy(cfg: DeployConfig) -> None: - server = OpenVLAServer(cfg.openvla_path) + server = OpenVLAServer(cfg) server.run(cfg.host, port=cfg.port) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index ec51a6b3c..445ae018b 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -1,175 +1,844 @@ """ finetune.py -Simple script for parameter-efficient fine-tuning of OpenVLA models loaded through the HuggingFace AutoClasses, using -HuggingFace PEFT library for low-rank adaptation (LoRA). - -Notes & Benchmarks: - - Requires PEFT (`pip install peft==0.11.1`) - - LoRA fine-tuning (see parameters below -- no quantization, LoRA rank = 32, target_modules = all-linear): - + One 48 GB GPU can fit a Batch Size of 12 - + One 80 GB GPU can fit a Batch Size of 24 - -Run with: - - [Single Node Multi-GPU (= $K) ]: torchrun --standalone --nnodes 1 --nproc-per-node $K vla-scripts/finetune.py - - [Override Config Values]: torchrun --standalone --nnodes 1 --nproc-per-node $K vla-scripts/finetune.py \ - --data_root_dir \ - --dataset_name \ - --run_root_dir \ - ... +Fine-tunes OpenVLA via LoRA. """ import os +import time from collections import deque from dataclasses import dataclass from pathlib import Path -from typing import Optional +from typing import Dict, Optional, Tuple, Type import draccus import torch import torch.distributed as dist +import torch.nn as nn import tqdm from accelerate import PartialState -from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training +from huggingface_hub import HfApi, snapshot_download +from peft import LoraConfig, PeftModel, get_peft_model from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import AdamW +from torch.optim.lr_scheduler import MultiStepLR from torch.utils.data import DataLoader -from transformers import AutoModelForVision2Seq, AutoProcessor, BitsAndBytesConfig -from transformers import AutoConfig, AutoImageProcessor +from transformers import AutoConfig, AutoImageProcessor, AutoModelForVision2Seq, AutoProcessor from transformers.modeling_outputs import CausalLMOutputWithPast import wandb -from prismatic.models.backbones.llm.prompting import PurePromptBuilder, VicunaV15ChatPromptBuilder -from prismatic.util.data_utils import PaddedCollatorForActionPrediction -from prismatic.vla.action_tokenizer import ActionTokenizer -from prismatic.vla.datasets import RLDSBatchTransform, RLDSDataset -from prismatic.vla.datasets.rlds.utils.data_utils import save_dataset_statistics + +from experiments.robot.openvla_utils import ( + check_model_logic_mismatch, + model_is_on_hf_hub, + update_auto_map, +) from prismatic.extern.hf.configuration_prismatic import OpenVLAConfig from prismatic.extern.hf.modeling_prismatic import OpenVLAForActionPrediction from prismatic.extern.hf.processing_prismatic import PrismaticImageProcessor, PrismaticProcessor +from prismatic.models.action_heads import DiffusionActionHead, L1RegressionActionHead +from prismatic.models.backbones.llm.prompting import PurePromptBuilder +from prismatic.models.film_vit_wrapper import FiLMedPrismaticVisionBackbone +from prismatic.models.projectors import ( + NoisyActionProjector, + ProprioProjector, +) +from prismatic.training.train_utils import ( + compute_actions_l1_loss, + compute_token_accuracy, + get_current_action_mask, + get_next_actions_mask, +) +from prismatic.util.data_utils import PaddedCollatorForActionPrediction +from prismatic.vla.action_tokenizer import ActionTokenizer +from prismatic.vla.constants import ( + ACTION_DIM, + ACTION_PROPRIO_NORMALIZATION_TYPE, + NUM_ACTIONS_CHUNK, + PROPRIO_DIM, +) +from prismatic.vla.datasets import RLDSBatchTransform, RLDSDataset +from prismatic.vla.datasets.rlds.utils.data_utils import save_dataset_statistics # Sane Defaults os.environ["TOKENIZERS_PARALLELISM"] = "false" -# # === Utilities === -# # fmt: off -# def create_vision_transform(vla: nn.Module, input_size: int) -> Callable[[Image.Image], torch.Tensor]: -# """Gets image transform for the vision encoder.""" -# data_cfg = timm.data.resolve_model_data_config(vla.vision_backbone) -# data_cfg["input_size"] = (3, input_size, input_size) -# return timm.data.create_transform( -# input_size=data_cfg["input_size"], -# interpolation=data_cfg["interpolation"], -# mean=data_cfg["mean"], -# std=data_cfg["std"], -# crop_pct=1.0, # Set to 1.0 to disable cropping -# crop_mode="center", # Default crop mode --> no-op when `crop_pct == 1.0` -# is_training=False, # Disable image_aug when loading transform; handled by RLDS dataloader -# ) -# -# # fmt: on - - @dataclass class FinetuneConfig: # fmt: off - vla_path: str = "openvla/openvla-7b" # Path to OpenVLA model (on HuggingFace Hub) - - # Directory Paths - data_root_dir: Path = Path("datasets/open-x-embodiment") # Path to Open-X dataset directory - dataset_name: str = "droid_wipe" # Name of fine-tuning dataset (e.g., `droid_wipe`) - run_root_dir: Path = Path("runs") # Path to directory to store logs & checkpoints - adapter_tmp_dir: Path = Path("adapter-tmp") # Temporary directory for LoRA weights before fusing - - # Fine-tuning Parameters - batch_size: int = 16 # Fine-tuning batch size - max_steps: int = 200_000 # Max number of fine-tuning steps - save_steps: int = 5000 # Interval for checkpoint saving - learning_rate: float = 5e-4 # Fine-tuning learning rate - grad_accumulation_steps: int = 1 # Gradient accumulation steps - image_aug: bool = True # Whether to train with image augmentations - shuffle_buffer_size: int = 100_000 # Dataloader shuffle buffer size (can reduce if OOM) - save_latest_checkpoint_only: bool = True # Whether to save only one checkpoint per run and - # continually overwrite the latest checkpoint - # (If False, saves all checkpoints) - - # LoRA Arguments - use_lora: bool = True # Whether to use LoRA fine-tuning - lora_rank: int = 32 # Rank of LoRA weight matrix - lora_dropout: float = 0.0 # Dropout applied to LoRA weights - use_quantization: bool = False # Whether to 4-bit quantize VLA for LoRA fine-tuning - # => CAUTION: Reduces memory but hurts performance - - # Tracking Parameters - wandb_project: str = "openvla" # Name of W&B project to log to (use default!) - wandb_entity: str = "stanford-voltron" # Name of entity to log under - run_id_note: Optional[str] = None # Extra note for logging, Weights & Biases + vla_path: str = "openvla/openvla-7b" # Path to OpenVLA model (on HuggingFace Hub or stored locally) + + # Dataset + data_root_dir: Path = Path("datasets/rlds") # Directory containing RLDS datasets + dataset_name: str = "aloha_scoop_x_into_bowl" # Name of fine-tuning dataset (e.g., `aloha_scoop_x_into_bowl`) + run_root_dir: Path = Path("runs") # Path to directory to store logs & checkpoints + shuffle_buffer_size: int = 100_000 # Dataloader shuffle buffer size (can reduce if OOM errors occur) + + # Algorithm and architecture + use_l1_regression: bool = True # If True, trains continuous action head with L1 regression objective + use_diffusion: bool = False # If True, trains continuous action head with diffusion modeling objective (DDIM) + num_diffusion_steps: int = 50 # (When `diffusion==True`) Number of diffusion steps for training + use_film: bool = False # If True, uses FiLM to infuse language inputs into visual features + num_images_in_input: int = 1 # Number of images in the VLA input (default: 1) + use_proprio: bool = False # If True, includes robot proprioceptive state in input + + # Training configuration + batch_size: int = 8 # Batch size per device (total batch size = batch_size * num GPUs) + learning_rate: float = 5e-4 # Learning rate + lr_warmup_steps: int = 0 # Number of steps to warm up learning rate (from 10% to 100%) + num_steps_before_decay: int = 100_000 # Number of steps before LR decays by 10x + grad_accumulation_steps: int = 1 # Number of gradient accumulation steps + max_steps: int = 200_000 # Max number of training steps + use_val_set: bool = False # If True, uses validation set and log validation metrics + val_freq: int = 10_000 # (When `use_val_set==True`) Validation set logging frequency in steps + val_time_limit: int = 180 # (When `use_val_set==True`) Time limit for computing validation metrics + save_freq: int = 10_000 # Checkpoint saving frequency in steps + save_latest_checkpoint_only: bool = False # If True, saves only 1 checkpoint, overwriting latest checkpoint + # (If False, saves all checkpoints) + resume: bool = False # If True, resumes from checkpoint + resume_step: Optional[int] = None # (When `resume==True`) Step number that we are resuming from + image_aug: bool = True # If True, trains with image augmentations (HIGHLY RECOMMENDED) + diffusion_sample_freq: int = 50 # (When `use_diffusion==True`) Frequency for sampling in steps + + # LoRA + use_lora: bool = True # If True, uses LoRA fine-tuning + lora_rank: int = 32 # Rank of LoRA weight matrix + lora_dropout: float = 0.0 # Dropout applied to LoRA weights + merge_lora_during_training: bool = True # If True, merges LoRA weights and saves result during training + # Note: Merging can be very slow on some machines. If so, set to + # False and merge final checkpoint offline! + + # Logging + wandb_entity: str = "your-wandb-entity" # Name of WandB entity + wandb_project: str = "your-wandb-project" # Name of WandB project + run_id_note: Optional[str] = None # Extra note to add to end of run ID for logging + run_id_override: Optional[str] = None # Optional string to override the run ID with + wandb_log_freq: int = 10 # WandB logging frequency in steps # fmt: on +def remove_ddp_in_checkpoint(state_dict) -> dict: + """ + Removes the 'module.' prefix from parameter names in a PyTorch model state dictionary that was saved using + DistributedDataParallel (DDP). + + When a model is trained using PyTorch's DistributedDataParallel, the saved state dictionary contains parameters + prefixed with 'module.'. This function removes these prefixes to make the state dictionary compatible when + loading into models that are not yet wrapped in DDP. + + Args: + state_dict (dict): PyTorch model state dictionary. + + Returns: + dict: A new state dictionary with the same contents but with 'module.' prefixes removed from parameter names. + Parameters without the 'module.' prefix remain unchanged. + """ + new_state_dict = {} + for k, v in state_dict.items(): + if k[:7] == "module.": + new_state_dict[k[7:]] = v + else: + new_state_dict[k] = v + return new_state_dict + + +def get_run_id(cfg) -> str: + """ + Generates or retrieves an identifier string for an experiment run. + + Args: + cfg (FinetuneConfig): Training configuration. + + Returns: + str: Experiment run ID. + """ + if cfg.run_id_override is not None: + # Override the run ID with the user-provided ID + run_id = cfg.run_id_override + elif cfg.resume: + # Override run ID with the previous resumed run's ID + run_id = cfg.vla_path.split("/")[-1] + # Remove the "--XXX_chkpt" suffix from the run ID if it exists + if "chkpt" in run_id.split("--")[-1]: + run_id = "--".join(run_id.split("--")[:-1]) + else: + run_id = ( + f"{cfg.vla_path.split('/')[-1]}+{cfg.dataset_name}" + f"+b{cfg.batch_size * cfg.grad_accumulation_steps}" + f"+lr-{cfg.learning_rate}" + ) + if cfg.use_lora: + run_id += f"+lora-r{cfg.lora_rank}+dropout-{cfg.lora_dropout}" + if cfg.image_aug: + run_id += "--image_aug" + if cfg.run_id_note is not None: + run_id += f"--{cfg.run_id_note}" + return run_id + + +def load_checkpoint(module_name: str, path: str, step: int, device: str = "cpu") -> dict: + """ + Loads a checkpoint for a given module. + + Args: + module_name (str): Name of model component to load checkpoint for. + path (str): Path to checkpoint directory. + step (int): Gradient step number of saved checkpoint. + device (str): String specifying how to remap storage locations (default = "cpu"). + + Returns: + dict: PyTorch model state dictionary. + """ + checkpoint_path = os.path.join(path, f"{module_name}--{step}_checkpoint.pt") + print(f"Loading checkpoint: {checkpoint_path}") + state_dict = torch.load(checkpoint_path, weights_only=True, map_location=device) + return remove_ddp_in_checkpoint(state_dict) + + +def wrap_ddp(module: nn.Module, device_id: int, find_unused: bool = False) -> DDP: + """ + Wrap a module with DistributedDataParallel. + + Args: + module (nn.Module): PyTorch module. + device_id (str): Device ID. + find_unused (bool): Whether to detect parameters without gradients in distributed training. + + Returns: + DistributedDataParallel: PyTorch module wrapped with DDP. + """ + return DDP(module, device_ids=[device_id], find_unused_parameters=find_unused, gradient_as_bucket_view=True) + + +def count_parameters(module: nn.Module, name: str) -> None: + """ + Counts and prints the number of trainable parameters in a module. + + Args: + module (nn.Module): PyTorch module. + module_name (str): Name of model component. + + Returns: + None. + """ + num_params = sum(p.numel() for p in module.parameters() if p.requires_grad) + print(f"# trainable params in {name}: {num_params}") + + +def init_module( + module_class: Type[nn.Module], + module_name: str, + cfg: FinetuneConfig, + device_id: int, + module_args: dict, + to_bf16: bool = False, + find_unused_params: bool = False, +) -> DDP: + """ + Initializes a module, optionally loads checkpoint, moves to device, and wraps with DDP. + + Args: + module_class (Type[nn.Module]): Class of PyTorch module to initialize. + module_name (str): Name of model component to load checkpoint for. + cfg (FinetuneConfig): Training configuration. + device_id (str): Device ID. + module_args (dict): Args for initializing the module. + to_bf16 (bool): Whether to convert to torch.bfloat16 data type. + find_unused_params (bool): Whether to detect parameters without gradients in distributed training. + + Returns: + DistributedDataParallel: PyTorch module wrapped with DDP. + """ + module = module_class(**module_args) + count_parameters(module, module_name) + + if cfg.resume: + state_dict = load_checkpoint(module_name, cfg.vla_path, cfg.resume_step) + module.load_state_dict(state_dict) + + if to_bf16: + module = module.to(torch.bfloat16) + module = module.to(device_id) + + return wrap_ddp(module, device_id, find_unused_params) + + +def run_forward_pass( + vla, + action_head, + noisy_action_projector, + proprio_projector, + batch, + action_tokenizer, + device_id, + use_l1_regression, + use_diffusion, + use_proprio, + use_film, + num_patches, + compute_diffusion_l1=False, + num_diffusion_steps=None, +) -> Tuple[torch.Tensor, Dict[str, float]]: + """ + Compute model forward pass and metrics for both training and validation. + + Args: + vla (OpenVLAForActionPrediction): Vision-language-action policy. + action_head (nn.Module): Action head module. + noisy_action_projector (nn.Module): Noisy action projector module (only used for diffusion). + proprio_projector (nn.Module): Proprioceptive state projector module. + batch (dict): Input batch. + action_tokenizer (ActionTokenizer): Action tokenizer. + device_id (str): Device ID. + use_l1_regression (bool): Whether to use L1 regression. + use_diffusion (bool): Whether to use diffusion. + use_proprio (bool): Whether to use proprioceptive state as input. + use_film (bool): Whether to use FiLM for better language following. + num_patches (int): Number of vision patches. + compute_diffusion_l1 (bool): Whether to sample actions and compute L1 loss for diffusion (do this once every + diffusion_sample_freq steps during training; do it every batch for validation) + num_diffusion_steps (int): Number of diffusion steps (only used for diffusion). + + Returns: + tuple: (loss, metrics_dict) + loss: The loss tensor with gradient for backpropagation. + metrics_dict: Dictionary of computed metrics (detached values for logging). + """ + metrics = {} + + # Get ground-truth action labels + ground_truth_actions = batch["actions"].to(device_id).to(torch.bfloat16) + + # [Only for diffusion] Sample noisy actions used as input for noise predictor network + if use_diffusion: + noisy_dict = action_head.module.sample_noisy_actions(ground_truth_actions) + noise, noisy_actions, diffusion_timestep_embeddings = ( + noisy_dict["noise"], + noisy_dict["noisy_actions"], + noisy_dict["diffusion_timestep_embeddings"], + ) + else: + noise, noisy_actions, diffusion_timestep_embeddings = None, None, None + + # VLA forward pass + with torch.autocast("cuda", dtype=torch.bfloat16): + output: CausalLMOutputWithPast = vla( + input_ids=batch["input_ids"].to(device_id), + attention_mask=batch["attention_mask"].to(device_id), + pixel_values=batch["pixel_values"].to(torch.bfloat16).to(device_id), + labels=batch["labels"], + output_hidden_states=True, + proprio=batch["proprio"] if use_proprio else None, + proprio_projector=proprio_projector if use_proprio else None, + noisy_actions=noisy_actions if use_diffusion else None, + noisy_action_projector=noisy_action_projector if use_diffusion else None, + diffusion_timestep_embeddings=diffusion_timestep_embeddings if use_diffusion else None, + use_film=use_film, + ) + + # Get action masks needed for logging + ground_truth_token_ids = batch["labels"][:, 1:].to(device_id) + current_action_mask = get_current_action_mask(ground_truth_token_ids) + next_actions_mask = get_next_actions_mask(ground_truth_token_ids) + + # Compute metrics for discrete action representation (next-token prediction) + if not (use_l1_regression or use_diffusion): + loss = output.loss + predicted_token_ids = output.logits[:, num_patches:-1].argmax(dim=2) + curr_action_accuracy = compute_token_accuracy( + predicted_token_ids, ground_truth_token_ids, mask=current_action_mask + ) + curr_action_l1_loss = compute_actions_l1_loss( + action_tokenizer, predicted_token_ids, ground_truth_token_ids, mask=current_action_mask + ) + next_actions_accuracy = compute_token_accuracy( + predicted_token_ids, ground_truth_token_ids, mask=next_actions_mask + ) + next_actions_l1_loss = compute_actions_l1_loss( + action_tokenizer, predicted_token_ids, ground_truth_token_ids, mask=next_actions_mask + ) + metrics.update( + { + "loss_value": loss.item(), # Detached value for logging + "curr_action_accuracy": curr_action_accuracy.item(), + "curr_action_l1_loss": curr_action_l1_loss.item(), + "next_actions_accuracy": next_actions_accuracy.item(), + "next_actions_l1_loss": next_actions_l1_loss.item(), + } + ) + # Compute metrics for continuous action representations (L1 regression | diffusion) + else: + # Get last layer hidden states + last_hidden_states = output.hidden_states[-1] # (B, seq_len, D) + # Get hidden states for text portion of prompt+response (after the vision patches) + text_hidden_states = last_hidden_states[:, num_patches:-1] + # Get hidden states for action portion of response + batch_size = batch["input_ids"].shape[0] + actions_hidden_states = ( + text_hidden_states[current_action_mask | next_actions_mask] + .reshape(batch_size, NUM_ACTIONS_CHUNK * ACTION_DIM, -1) + .to(torch.bfloat16) + ) # (B, act_chunk_len, D) + + if use_l1_regression: + # Predict action + predicted_actions = action_head.module.predict_action(actions_hidden_states) + # Get full L1 loss + loss = torch.nn.L1Loss()(ground_truth_actions, predicted_actions) + + if use_diffusion: + # Predict noise + noise_pred = action_head.module.predict_noise(actions_hidden_states) + # Get diffusion noise prediction MSE loss + noise_pred = noise_pred.reshape(noise.shape) + loss = nn.functional.mse_loss(noise_pred, noise, reduction="mean") + + # Only sample actions and compute L1 losses if specified + if compute_diffusion_l1: + with torch.no_grad(): + predicted_actions = run_diffusion_sampling( + vla=vla, + action_head=action_head, + noisy_action_projector=noisy_action_projector, + proprio_projector=proprio_projector, + batch=batch, + batch_size=batch_size, + num_patches=num_patches, + actions_shape=ground_truth_actions.shape, + device_id=device_id, + current_action_mask=current_action_mask, + next_actions_mask=next_actions_mask, + use_proprio=use_proprio, + use_film=use_film, + ) + + metrics.update( + { + "loss_value": loss.item(), # Detached value for logging + } + ) + + # Get detailed L1 losses for logging + should_log_l1_loss = not use_diffusion or (use_diffusion and compute_diffusion_l1) + if should_log_l1_loss: + ground_truth_curr_action = ground_truth_actions[:, 0] + predicted_curr_action = predicted_actions[:, 0] + ground_truth_next_actions = ground_truth_actions[:, 1:] + predicted_next_actions = predicted_actions[:, 1:] + curr_action_l1_loss = torch.nn.L1Loss()(ground_truth_curr_action, predicted_curr_action) + next_actions_l1_loss = torch.nn.L1Loss()(ground_truth_next_actions, predicted_next_actions) + metrics.update( + { + "curr_action_l1_loss": curr_action_l1_loss.item(), + "next_actions_l1_loss": next_actions_l1_loss.item(), + } + ) + + # Return both the loss tensor (with gradients) and the metrics dictionary (with detached values) + return loss, metrics + + +def run_diffusion_sampling( + vla, + action_head, + noisy_action_projector, + proprio_projector, + batch, + batch_size, + num_patches, + actions_shape, + device_id, + current_action_mask, + next_actions_mask, + use_proprio, + use_film, +) -> torch.Tensor: + """ + Run diffusion sampling (reverse diffusion) to generate actions. + + Args: + vla (OpenVLAForActionPrediction): Vision-language-action policy. + action_head (nn.Module): Action head module. + noisy_action_projector (nn.Module): Noisy action projector module (only used for diffusion). + proprio_projector (nn.Module): Proprioceptive state projector module. + batch (dict): Input batch. + batch_size (int): Batch size. + num_patches (int): Number of vision patches. + actions_shape (tuple): Shape of ground-truth actions. + device_id (str): Device ID. + current_action_mask (torch.Tensor): Mask for current action. + next_actions_mask (torch.Tensor): Mask for next actions. + use_proprio (bool): Whether to use proprioceptive state as input. + use_film (bool): Whether to use FiLM for better language following. + + Returns: + torch.Tensor: Predicted actions. + """ + # Sample random noisy action, used as the starting point for reverse diffusion + noise = torch.randn( + size=(batch_size, NUM_ACTIONS_CHUNK, ACTION_DIM), + device=device_id, + dtype=torch.bfloat16, + ) # (B, chunk_len, action_dim) + + # Set diffusion timestep values + action_head.module.noise_scheduler.set_timesteps(action_head.module.num_diffusion_steps) + + # Reverse diffusion: Iteratively denoise to generate action, conditioned on observation + curr_noisy_actions = noise + for t in action_head.module.noise_scheduler.timesteps: + # Get diffusion model's noise prediction (conditioned on VLA latent embedding, current noisy action embedding, + # and diffusion timestep embedding) + timesteps = torch.Tensor([t]).repeat(batch_size).to(device_id) + diffusion_timestep_embeddings = ( + action_head.module.time_encoder(timesteps).to(curr_noisy_actions.dtype).to(curr_noisy_actions.device) + ) # (B, llm_dim) + diffusion_timestep_embeddings = diffusion_timestep_embeddings.unsqueeze(1) # (B, 1, llm_dim) + + with torch.autocast("cuda", dtype=torch.bfloat16): + output = vla( + input_ids=batch["input_ids"].to(device_id), + attention_mask=batch["attention_mask"].to(device_id), + pixel_values=batch["pixel_values"].to(torch.bfloat16).to(device_id), + labels=batch["labels"], + output_hidden_states=True, + proprio=batch["proprio"] if use_proprio else None, + proprio_projector=proprio_projector if use_proprio else None, + noisy_actions=curr_noisy_actions, + noisy_action_projector=noisy_action_projector, + diffusion_timestep_embeddings=diffusion_timestep_embeddings, + use_film=use_film, + ) + # Get last layer hidden states + last_hidden_states = output.hidden_states[-1] # (B, seq_len, D) + # Get hidden states for text portion of prompt+response (after the vision patches) + text_hidden_states = last_hidden_states[:, num_patches:-1] + # Get hidden states for action portion of response + actions_hidden_states = text_hidden_states[current_action_mask | next_actions_mask].reshape( + batch_size, NUM_ACTIONS_CHUNK * ACTION_DIM, -1 + ) # (B, act_chunk_len, D) + actions_hidden_states = actions_hidden_states.to(torch.bfloat16) + # Predict noise + noise_pred = action_head.module.predict_noise(actions_hidden_states) + + # Compute the action at the previous diffusion timestep: x_t -> x_{t-1} + curr_noisy_actions = action_head.module.noise_scheduler.step(noise_pred, t, curr_noisy_actions).prev_sample + + return curr_noisy_actions.reshape(actions_shape) + + +def compute_smoothened_metrics(metrics_deques) -> dict: + """ + Compute smoothened metrics from recent deques. + + Args: + metrics_deques (dict): Dictionary of deques containing recent metrics. + + Returns: + dict: Dictionary of smoothened metrics. + """ + smoothened_metrics = {} + for name, deque in metrics_deques.items(): + if deque and len(deque) > 0: + smoothened_metrics[name] = sum(deque) / len(deque) + return smoothened_metrics + + +def log_metrics_to_wandb(metrics, prefix, step, wandb_entity) -> None: + """ + Log metrics to Weights & Biases. + + Args: + metrics (dict): Dictionary of metrics to log + prefix (str): Prefix for metric names + step (int): Training step + wandb_entity (str): W&B entity instance + + Returns: + None. + """ + log_dict = {} + for name, value in metrics.items(): + # Map loss_value to Loss for better readability in W&B + if name == "loss_value": + log_dict[f"{prefix}/Loss"] = value + # Keep other metrics as is + else: + log_dict[f"{prefix}/{name.replace('_', ' ').title()}"] = value + wandb_entity.log(log_dict, step=step) + + +def save_training_checkpoint( + cfg, + run_dir, + log_step, + vla, + processor, + proprio_projector, + noisy_action_projector, + action_head, + train_dataset, + distributed_state, +) -> None: + """ + Save all training checkpoints including model components, LoRA adapter, and dataset statistics. + + Args: + cfg (FinetuneConfig): Training configuration. + run_dir (Path): Experiment run directory path. + log_step (int): Current logging step. + vla (OpenVLAForActionPrediction): Vision-language-action policy. + processor (PrismaticProcessor): OpenVLA inputs processor. + proprio_projector (nn.Module): Proprioceptive state projector module. + noisy_action_projector (nn.Module): Noisy action projector module (only used for diffusion). + action_head (nn.Module): Action head module. + train_dataset (RLDSDataset): Training dataset. + distributed_state (PartialState): Distributed training state. + + Returns: + None. + """ + # Determine checkpoint paths and naming + if cfg.save_latest_checkpoint_only: + checkpoint_dir = run_dir + checkpoint_name_suffix = "latest_checkpoint.pt" + else: + checkpoint_dir = Path(str(run_dir) + f"--{log_step}_chkpt") + checkpoint_name_suffix = f"{log_step}_checkpoint.pt" + + adapter_dir = checkpoint_dir / "lora_adapter" + + # Create directories and save dataset statistics (main process only) + if distributed_state.is_main_process: + os.makedirs(checkpoint_dir, exist_ok=True) + os.makedirs(adapter_dir, exist_ok=True) + save_dataset_statistics(train_dataset.dataset_statistics, checkpoint_dir) + print(f"Saving Model Checkpoint for Step {log_step}") + + # Wait for directories to be created + dist.barrier() + + # Save model components (main process only) + if distributed_state.is_main_process: + # Save processor and LoRA adapter + processor.save_pretrained(checkpoint_dir) + vla.module.save_pretrained(adapter_dir) + + # Save other components + if cfg.use_proprio and proprio_projector is not None: + torch.save(proprio_projector.state_dict(), checkpoint_dir / f"proprio_projector--{checkpoint_name_suffix}") + + if cfg.use_diffusion and noisy_action_projector is not None: + torch.save( + noisy_action_projector.state_dict(), checkpoint_dir / f"noisy_action_projector--{checkpoint_name_suffix}" + ) + + if (cfg.use_l1_regression or cfg.use_diffusion) and action_head is not None: + torch.save(action_head.state_dict(), checkpoint_dir / f"action_head--{checkpoint_name_suffix}") + + if cfg.use_film: + # To be safe, just save the entire vision backbone (not just FiLM components) + torch.save( + vla.module.vision_backbone.state_dict(), checkpoint_dir / f"vision_backbone--{checkpoint_name_suffix}" + ) + + # Wait for model components to be saved + dist.barrier() + + # Merge LoRA weights into base model and save resulting model checkpoint + # Note: Can be very slow on some devices; if so, we recommend merging offline + if cfg.use_lora and cfg.merge_lora_during_training: + base_vla = AutoModelForVision2Seq.from_pretrained( + cfg.vla_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True + ) + merged_vla = PeftModel.from_pretrained(base_vla, adapter_dir) + merged_vla = merged_vla.merge_and_unload() + + if distributed_state.is_main_process: + merged_vla.save_pretrained(checkpoint_dir) + print(f"Saved merged model for Step {log_step} at: {checkpoint_dir}") + + # Wait for merged model to be saved + dist.barrier() + + +def run_validation( + vla, + action_head, + noisy_action_projector, + proprio_projector, + val_dataloader, + action_tokenizer, + device_id, + cfg, + num_patches, + log_step, + distributed_state, + val_time_limit, +) -> None: + """ + Compute validation set metrics for logging. + + Args: + vla (OpenVLAForActionPrediction): Vision-language-action policy. + action_head (nn.Module): Action head module. + noisy_action_projector (nn.Module): Noisy action projector module (only used for diffusion). + proprio_projector (nn.Module): Proprioceptive state projector module. + val_dataloader (DataLoader): Validation data loader. + action_tokenizer (ActionTokenizer): Action tokenizer. + device_id (str): Device ID. + cfg (FinetuneConfig): Training configuration. + num_patches (int): Number of vision patches. + log_step (int): Current logging step. + distributed_state (PartialState): Distributed training state. + val_time_limit (int): Time limit for computing validation metrics. + + Returns: + None. + """ + val_start_time = time.time() + vla.eval() + val_batches_count = 0 + + # List to store validation metrics + all_val_metrics = [] + + with torch.no_grad(): + for batch in val_dataloader: + # Always compute L1 loss for validation, even for diffusion + _, metrics = run_forward_pass( + vla=vla, + action_head=action_head, + noisy_action_projector=noisy_action_projector, + proprio_projector=proprio_projector, + batch=batch, + action_tokenizer=action_tokenizer, + device_id=device_id, + use_l1_regression=cfg.use_l1_regression, + use_diffusion=cfg.use_diffusion, + use_proprio=cfg.use_proprio, + use_film=cfg.use_film, + num_patches=num_patches, + compute_diffusion_l1=True, + num_diffusion_steps=cfg.num_diffusion_steps if cfg.use_diffusion else None, + ) + + # Add the loss value to the metrics + metrics["loss"] = metrics["loss_value"] + all_val_metrics.append(metrics) + val_batches_count += 1 + + # Cut testing on validation set short if it exceeds time limit + if time.time() - val_start_time > val_time_limit: + break + + # Compute average validation metrics + avg_val_metrics = {} + for metric_name in all_val_metrics[0].keys(): + values = [metrics[metric_name] for metrics in all_val_metrics if metric_name in metrics] + if values: + avg_val_metrics[metric_name] = sum(values) / len(values) + + # Add batch count to metrics + avg_val_metrics["val_batches_count"] = val_batches_count + + # Log validation metrics to W&B + if distributed_state.is_main_process: + log_metrics_to_wandb(avg_val_metrics, "VLA Val", log_step, wandb) + + @draccus.wrap() def finetune(cfg: FinetuneConfig) -> None: + """ + Fine-tunes base VLA on demonstration dataset via LoRA. + + Allows toggling different action representations (discrete vs. continuous), different learning objectives + (next-token prediction vs. L1 regression vs. diffusion), FiLM. Also allows for additional model inputs, + such as additional camera images and robot proprioceptive state. Assumes parallel action generation with + action chunking. + + Args: + cfg (FinetuneConfig): Training configuration. + + Returns: + None. + """ + assert cfg.use_lora, "Only LoRA fine-tuning is supported. Please set --use_lora=True!" + assert not (cfg.use_l1_regression and cfg.use_diffusion), ( + "Cannot do both L1 regression and diffusion. Please pick one of them!" + ) + + # Trim trailing forward slash ('/') in VLA path if it exists + cfg.vla_path = cfg.vla_path.rstrip("/") print(f"Fine-tuning OpenVLA Model `{cfg.vla_path}` on `{cfg.dataset_name}`") - # [Validate] Ensure GPU Available & Set Device / Distributed Context - assert torch.cuda.is_available(), "Fine-tuning assumes at least one GPU is available!" + # Get experiment run ID + run_id = get_run_id(cfg) + + # Create experiment run directory + run_dir = cfg.run_root_dir / run_id + os.makedirs(run_dir, exist_ok=True) + + # GPU setup distributed_state = PartialState() - torch.cuda.set_device(device_id := distributed_state.local_process_index) + device_id = distributed_state.local_process_index + torch.cuda.set_device(device_id) torch.cuda.empty_cache() - # Configure Unique Experiment ID & Log Directory - exp_id = ( - f"{cfg.vla_path.split('/')[-1]}+{cfg.dataset_name}" - f"+b{cfg.batch_size * cfg.grad_accumulation_steps}" - f"+lr-{cfg.learning_rate}" + # Initialize wandb logging + if distributed_state.is_main_process: + wandb.init(entity=cfg.wandb_entity, project=cfg.wandb_project, name=f"ft+{run_id}") + + # Print detected constants + print( + "Detected constants:\n" + f"\tNUM_ACTIONS_CHUNK: {NUM_ACTIONS_CHUNK}\n" + f"\tACTION_DIM: {ACTION_DIM}\n" + f"\tPROPRIO_DIM: {PROPRIO_DIM}\n" + f"\tACTION_PROPRIO_NORMALIZATION_TYPE: {ACTION_PROPRIO_NORMALIZATION_TYPE}" ) - if cfg.use_lora: - exp_id += f"+lora-r{cfg.lora_rank}+dropout-{cfg.lora_dropout}" - if cfg.use_quantization: - exp_id += "+q-4bit" - if cfg.run_id_note is not None: - exp_id += f"--{cfg.run_id_note}" - if cfg.image_aug: - exp_id += "--image_aug" - - # Start =>> Build Directories - run_dir, adapter_dir = cfg.run_root_dir / exp_id, cfg.adapter_tmp_dir / exp_id - os.makedirs(run_dir, exist_ok=True) - # Quantization Config =>> only if LoRA fine-tuning - quantization_config = None - if cfg.use_quantization: - assert cfg.use_lora, "Quantized training only supported for LoRA fine-tuning!" - quantization_config = BitsAndBytesConfig( - load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4" - ) + # Two options: + # (1) Base model is on Hugging Face Hub + # - Then download it and record the path to the download directory + # (2) Base model is stored locally + # - Then register model config in HF Auto Classes + # In both cases, we want to check whether any changes have been made to + # the `modeling_prismatic.py` file in this codebase; if so, we will copy + # the file to the downloaded or locally stored checkpoint directory so + # that the user's changes to the VLA class logic go into effect + if model_is_on_hf_hub(cfg.vla_path): + # Download model directly from Hugging Face Hub + vla_download_path = snapshot_download(repo_id=cfg.vla_path) + # Overwrite VLA path + cfg.vla_path = vla_download_path + else: + # Register OpenVLA model to HF Auto Classes (not needed if the model is on HF Hub) + AutoConfig.register("openvla", OpenVLAConfig) + AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) + AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) + AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) - # Register OpenVLA model to HF Auto Classes (not needed if the model is on HF Hub) - AutoConfig.register("openvla", OpenVLAConfig) - AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) - AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) - AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) + # Update config.json and sync model files + update_auto_map(cfg.vla_path) + check_model_logic_mismatch(cfg.vla_path) - # Load OpenVLA Processor and Model using HF AutoClasses + # Load processor and VLA processor = AutoProcessor.from_pretrained(cfg.vla_path, trust_remote_code=True) vla = AutoModelForVision2Seq.from_pretrained( cfg.vla_path, torch_dtype=torch.bfloat16, - quantization_config=quantization_config, low_cpu_mem_usage=True, trust_remote_code=True, - ) + ).to(device_id) - # Device Placement =>> note that BitsAndBytes automatically handles for quantized training - if cfg.use_quantization: - vla = prepare_model_for_kbit_training(vla) - else: - vla = vla.to(device_id) + # Set number of images in VLA input + vla.vision_backbone.set_num_images_in_input(cfg.num_images_in_input) - # [LoRA] Wrap Model w/ PEFT `LoraConfig` =>> by default we set `target_modules=all-linear` + # LoRA setup if cfg.use_lora: lora_config = LoraConfig( r=cfg.lora_rank, @@ -181,13 +850,96 @@ def finetune(cfg: FinetuneConfig) -> None: vla = get_peft_model(vla, lora_config) vla.print_trainable_parameters() - # Wrap VLA in PyTorch DDP Wrapper for Multi-GPU Training - vla = DDP(vla, device_ids=[device_id], find_unused_parameters=True, gradient_as_bucket_view=True) + # FiLM setup + if cfg.use_film: + count_parameters(vla.vision_backbone, "vla.vision_backbone (original)") + # Wrap vision backbone with FiLM wrapper + # Important: For this, must specify `vla.model.vision_backbone` instead of just `vla.vision_backbone`, since the + # latter would cause the new wrapped backbone to be saved as a new attribute of `vla` instead of overwriting the + # original one (due to the LoRA wrapper) + vla.model.vision_backbone = FiLMedPrismaticVisionBackbone( + vision_backbone=vla.model.vision_backbone, + llm_dim=vla.llm_dim, + ) + count_parameters(vla.vision_backbone, "vla.vision_backbone (post-wrap)") + if cfg.resume: + state_dict = load_checkpoint("vision_backbone", cfg.vla_path, cfg.resume_step) + vla.model.vision_backbone.load_state_dict(state_dict) + vla.model.vision_backbone = vla.model.vision_backbone.to(device_id) + + # Wrap VLA with DDP + vla = wrap_ddp(vla, device_id, find_unused=True) + + # If applicable, instantiate proprio projector + if cfg.use_proprio: + proprio_projector = init_module( + ProprioProjector, + "proprio_projector", + cfg, + device_id, + {"llm_dim": vla.module.llm_dim, "proprio_dim": PROPRIO_DIM}, + ) + + # If applicable, instantiate continuous action head for L1 regression + if cfg.use_l1_regression: + action_head = init_module( + L1RegressionActionHead, + "action_head", + cfg, + device_id, + {"input_dim": vla.module.llm_dim, "hidden_dim": vla.module.llm_dim, "action_dim": ACTION_DIM}, + to_bf16=True, + ) - # Create Optimizer =>> note that we default to a simple constant learning rate! + # If applicable, instantiate diffusion action head and noisy action projector + if cfg.use_diffusion: + action_head = init_module( + DiffusionActionHead, + "action_head", + cfg, + device_id, + { + "input_dim": vla.module.llm_dim, + "hidden_dim": vla.module.llm_dim, + "action_dim": ACTION_DIM, + "num_diffusion_steps": cfg.num_diffusion_steps, + }, + to_bf16=True, + ) + noisy_action_projector = init_module( + NoisyActionProjector, "noisy_action_projector", cfg, device_id, {"llm_dim": vla.module.llm_dim} + ) + + # Get number of vision patches + NUM_PATCHES = vla.module.vision_backbone.get_num_patches() * vla.module.vision_backbone.get_num_images_in_input() + # If we have proprio inputs, a single proprio embedding is appended to the end of the vision patch embeddings + if cfg.use_proprio: + NUM_PATCHES += 1 + # For diffusion, a single diffusion timestep embedding is appended to the end of the vision patch embeddings + if cfg.use_diffusion: + NUM_PATCHES += 1 + + # Instantiate optimizer trainable_params = [param for param in vla.parameters() if param.requires_grad] + if cfg.use_l1_regression or cfg.use_diffusion: + trainable_params += [param for param in action_head.parameters() if param.requires_grad] + if cfg.use_diffusion: + trainable_params += [param for param in noisy_action_projector.parameters() if param.requires_grad] + if cfg.use_proprio: + trainable_params += [param for param in proprio_projector.parameters() if param.requires_grad] + print(f"# total trainable params: {sum(p.numel() for p in trainable_params)}") optimizer = AdamW(trainable_params, lr=cfg.learning_rate) + # Record original learning rate + original_lr = optimizer.param_groups[0]["lr"] + + # Create learning rate scheduler + scheduler = MultiStepLR( + optimizer, + milestones=[cfg.num_steps_before_decay], # Number of steps after which LR will change + gamma=0.1, # Multiplicative factor of learning rate decay + ) + # Create Action Tokenizer action_tokenizer = ActionTokenizer(processor.tokenizer) @@ -199,20 +951,27 @@ def finetune(cfg: FinetuneConfig) -> None: # --- # from prismatic.vla.datasets import DummyDataset # - # vla_dataset = DummyDataset( + # train_dataset = DummyDataset( # action_tokenizer, # processor.tokenizer, # image_transform=processor.image_processor.apply_transform, - # prompt_builder_fn=PurePromptBuilder if "v01" not in cfg.vla_path else VicunaV15ChatPromptBuilder, + # prompt_builder_fn=PurePromptBuilder, # ) # --- + + # We assume that the model takes as input one third-person camera image and 1 or 2 optional wrist camera image(s) + use_wrist_image = cfg.num_images_in_input > 1 + + # Create training and optional validation datasets batch_transform = RLDSBatchTransform( action_tokenizer, processor.tokenizer, image_transform=processor.image_processor.apply_transform, - prompt_builder_fn=PurePromptBuilder if "v01" not in cfg.vla_path else VicunaV15ChatPromptBuilder, + prompt_builder_fn=PurePromptBuilder, + use_wrist_image=use_wrist_image, + use_proprio=cfg.use_proprio, ) - vla_dataset = RLDSDataset( + train_dataset = RLDSDataset( cfg.data_root_dir, cfg.dataset_name, batch_transform, @@ -220,45 +979,74 @@ def finetune(cfg: FinetuneConfig) -> None: shuffle_buffer_size=cfg.shuffle_buffer_size, image_aug=cfg.image_aug, ) + if cfg.use_val_set: + val_dataset = RLDSDataset( + cfg.data_root_dir, + cfg.dataset_name, + batch_transform, + resize_resolution=tuple(vla.module.config.image_sizes), + shuffle_buffer_size=cfg.shuffle_buffer_size // 10, + image_aug=cfg.image_aug, + train=False, + ) - # [Important] Save Dataset Statistics =>> used to de-normalize actions for inference! + # [Important] Save dataset statistics so that we can unnormalize actions during inference if distributed_state.is_main_process: - save_dataset_statistics(vla_dataset.dataset_statistics, run_dir) + save_dataset_statistics(train_dataset.dataset_statistics, run_dir) - # Create Collator and DataLoader + # Create collator and dataloader collator = PaddedCollatorForActionPrediction( processor.tokenizer.model_max_length, processor.tokenizer.pad_token_id, padding_side="right" ) dataloader = DataLoader( - vla_dataset, + train_dataset, batch_size=cfg.batch_size, sampler=None, collate_fn=collator, - num_workers=0, # Important =>> Set to 0 if using RLDS; TFDS rolls its own parallelism! + num_workers=0, # Important: Set to 0 if using RLDS, which uses its own parallelism ) - - # Initialize Logging =>> W&B - if distributed_state.is_main_process: - wandb.init(entity=cfg.wandb_entity, project=cfg.wandb_project, name=f"ft+{exp_id}") + if cfg.use_val_set: + val_batch_size = cfg.batch_size + val_dataloader = DataLoader( + val_dataset, + batch_size=val_batch_size, + sampler=None, + collate_fn=collator, + num_workers=0, # Important: Set to 0 if using RLDS, which uses its own parallelism + ) # Deque to store recent train metrics (used for computing smoothened metrics for gradient accumulation) - recent_losses = deque(maxlen=cfg.grad_accumulation_steps) - recent_action_accuracies = deque(maxlen=cfg.grad_accumulation_steps) - recent_l1_losses = deque(maxlen=cfg.grad_accumulation_steps) + recent_metrics = { + "loss_value": deque(maxlen=cfg.grad_accumulation_steps), + "curr_action_accuracy": deque(maxlen=cfg.grad_accumulation_steps), + "curr_action_l1_loss": deque(maxlen=cfg.grad_accumulation_steps), + "next_actions_accuracy": deque(maxlen=cfg.grad_accumulation_steps), + "next_actions_l1_loss": deque(maxlen=cfg.grad_accumulation_steps), + } - # Train! + # Start training with tqdm.tqdm(total=cfg.max_steps, leave=False) as progress: vla.train() optimizer.zero_grad() for batch_idx, batch in enumerate(dataloader): - with torch.autocast("cuda", dtype=torch.bfloat16): - output: CausalLMOutputWithPast = vla( - input_ids=batch["input_ids"].to(device_id), - attention_mask=batch["attention_mask"].to(device_id), - pixel_values=batch["pixel_values"].to(torch.bfloat16).to(device_id), - labels=batch["labels"], - ) - loss = output.loss + # Compute training metrics and loss + compute_diffusion_l1 = cfg.use_diffusion and batch_idx % cfg.diffusion_sample_freq == 0 + loss, metrics = run_forward_pass( + vla=vla, + action_head=action_head, + noisy_action_projector=noisy_action_projector if cfg.use_diffusion else None, + proprio_projector=proprio_projector if cfg.use_proprio else None, + batch=batch, + action_tokenizer=action_tokenizer, + device_id=device_id, + use_l1_regression=cfg.use_l1_regression, + use_diffusion=cfg.use_diffusion, + use_proprio=cfg.use_proprio, + use_film=cfg.use_film, + num_patches=NUM_PATCHES, + compute_diffusion_l1=compute_diffusion_l1, + num_diffusion_steps=cfg.num_diffusion_steps if cfg.use_diffusion else None, + ) # Normalize loss to account for gradient accumulation normalized_loss = loss / cfg.grad_accumulation_steps @@ -266,105 +1054,82 @@ def finetune(cfg: FinetuneConfig) -> None: # Backward pass normalized_loss.backward() - # Compute Accuracy and L1 Loss for Logging - action_logits = output.logits[:, vla.module.vision_backbone.featurizer.patch_embed.num_patches : -1] - action_preds = action_logits.argmax(dim=2) - action_gt = batch["labels"][:, 1:].to(action_preds.device) - mask = action_gt > action_tokenizer.action_token_begin_idx - - # Compute Accuracy - correct_preds = (action_preds == action_gt) & mask - action_accuracy = correct_preds.sum().float() / mask.sum().float() - - # Compute L1 Loss on Predicted (Continuous) Actions - continuous_actions_pred = torch.tensor( - action_tokenizer.decode_token_ids_to_actions(action_preds[mask].cpu().numpy()) - ) - continuous_actions_gt = torch.tensor( - action_tokenizer.decode_token_ids_to_actions(action_gt[mask].cpu().numpy()) - ) - action_l1_loss = torch.nn.functional.l1_loss(continuous_actions_pred, continuous_actions_gt) - # Store recent train metrics - recent_losses.append(loss.item()) - recent_action_accuracies.append(action_accuracy.item()) - recent_l1_losses.append(action_l1_loss.item()) + for metric_name, value in metrics.items(): + if metric_name in recent_metrics: + recent_metrics[metric_name].append(value) # Compute gradient step index gradient_step_idx = batch_idx // cfg.grad_accumulation_steps # Compute smoothened train metrics - # =>> Equal to current step metrics when not using gradient accumulation - # =>> Otherwise, equal to the average of metrics observed over micro-batches used for gradient accumulation - smoothened_loss = sum(recent_losses) / len(recent_losses) - smoothened_action_accuracy = sum(recent_action_accuracies) / len(recent_action_accuracies) - smoothened_l1_loss = sum(recent_l1_losses) / len(recent_l1_losses) - - # Push Metrics to W&B (every 10 gradient steps) - if distributed_state.is_main_process and gradient_step_idx % 10 == 0: + smoothened_metrics = compute_smoothened_metrics(recent_metrics) + + # Push Metrics to W&B (every wandb_log_freq gradient steps) + log_step = gradient_step_idx if not cfg.resume else cfg.resume_step + gradient_step_idx + if distributed_state.is_main_process and log_step % cfg.wandb_log_freq == 0: + log_metrics_to_wandb(smoothened_metrics, "VLA Train", log_step, wandb) + + # [If applicable] Linearly warm up learning rate from 10% to 100% of original + if cfg.lr_warmup_steps > 0: + lr_progress = min((gradient_step_idx + 1) / cfg.lr_warmup_steps, 1.0) # Cap at 1.0 + current_lr = original_lr * (0.1 + 0.9 * lr_progress) + for param_group in optimizer.param_groups: + param_group["lr"] = current_lr + + if distributed_state.is_main_process and gradient_step_idx % cfg.wandb_log_freq == 0: + # Log the learning rate + # Make sure to do this AFTER any learning rate modifications (e.g., warmup/decay) wandb.log( { - "train_loss": smoothened_loss, - "action_accuracy": smoothened_action_accuracy, - "l1_loss": smoothened_l1_loss, + "VLA Train/Learning Rate": scheduler.get_last_lr()[0], }, - step=gradient_step_idx, + step=log_step, ) - # Optimizer Step + # Optimizer and LR scheduler step if (batch_idx + 1) % cfg.grad_accumulation_steps == 0: optimizer.step() + scheduler.step() optimizer.zero_grad() progress.update() - # Save Model Checkpoint =>> by default, only keeps the latest checkpoint, continually overwriting it! - if gradient_step_idx > 0 and gradient_step_idx % cfg.save_steps == 0: - if distributed_state.is_main_process: - print(f"Saving Model Checkpoint for Step {gradient_step_idx}") - - # If LoRA, we first save adapter weights, then merge into full model; otherwise, default save! - save_dir = adapter_dir if cfg.use_lora else run_dir - - # Save Processor & Weights - processor.save_pretrained(run_dir) - vla.module.save_pretrained(save_dir) - - # Wait for processor and adapter weights to be saved by main process - dist.barrier() - - # Merge LoRA weights into model backbone for faster inference - # =>> Note that merging is slow and can be done post-hoc to speed up training - if cfg.use_lora: - base_vla = AutoModelForVision2Seq.from_pretrained( - cfg.vla_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True - ) - merged_vla = PeftModel.from_pretrained(base_vla, adapter_dir) - merged_vla = merged_vla.merge_and_unload() - if distributed_state.is_main_process: - if cfg.save_latest_checkpoint_only: - # Overwrite latest checkpoint - merged_vla.save_pretrained(run_dir) - - print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {run_dir}") - else: - # Prepare to save checkpoint in new directory - checkpoint_dir = Path(str(run_dir) + f"--{gradient_step_idx}_chkpt") - os.makedirs(checkpoint_dir, exist_ok=True) - - # Save dataset statistics to new directory - save_dataset_statistics(vla_dataset.dataset_statistics, checkpoint_dir) - - # Save processor and model weights to new directory - processor.save_pretrained(checkpoint_dir) - merged_vla.save_pretrained(checkpoint_dir) - - print(f"Saved Model Checkpoint for Step {gradient_step_idx} at: {checkpoint_dir}") + # Save model checkpoint: either keep latest checkpoint only or all checkpoints + if gradient_step_idx > 0 and log_step % cfg.save_freq == 0: + save_training_checkpoint( + cfg=cfg, + run_dir=run_dir, + log_step=log_step, + vla=vla, + processor=processor, + proprio_projector=proprio_projector if cfg.use_proprio else None, + noisy_action_projector=noisy_action_projector if cfg.use_diffusion else None, + action_head=action_head if (cfg.use_l1_regression or cfg.use_diffusion) else None, + train_dataset=train_dataset, + distributed_state=distributed_state, + ) - # Block on Main Process Checkpointing - dist.barrier() + # Test model on validation set + if cfg.use_val_set and log_step > 0 and log_step % cfg.val_freq == 0: + run_validation( + vla=vla, + action_head=action_head, + noisy_action_projector=noisy_action_projector if cfg.use_diffusion else None, + proprio_projector=proprio_projector if cfg.use_proprio else None, + val_dataloader=val_dataloader, + action_tokenizer=action_tokenizer, + device_id=device_id, + cfg=cfg, + num_patches=NUM_PATCHES, + log_step=log_step, + distributed_state=distributed_state, + val_time_limit=cfg.val_time_limit, + ) + # Set model back to training mode after validation + vla.train() # Stop training when max_steps is reached - if gradient_step_idx == cfg.max_steps: + if log_step == cfg.max_steps: print(f"Max step {cfg.max_steps} reached! Stopping training...") break diff --git a/vla-scripts/merge_lora_weights_and_save.py b/vla-scripts/merge_lora_weights_and_save.py new file mode 100644 index 000000000..8c38c10e9 --- /dev/null +++ b/vla-scripts/merge_lora_weights_and_save.py @@ -0,0 +1,73 @@ +""" +Loads a checkpoint that only has a LoRA adapter (no merged model) and merges the adapter +into the base OpenVLA model. Saves the final checkpoint in the same directory. + +Make sure to specify the correct base checkpoint when running this script. For example, +- if you fine-tuned the default OpenVLA-7B model without modifications, then `--base_checkpoint=="openvla/openvla-7b"` +- if you fine-tuned a different model or resumed fine-tuning from a different checkpoint, then specify that base checkpoint +- if you fine-tuned the default OpenVLA-7B model with modifications to `modeling_prismatic.py` (OpenVLA class definition), + then the base checkpoint path should point to the checkpoint containing the modifications + +Usage: + python vla-scripts/merge_lora_weights_and_save.py \ + --base_checkpoint openvla/openvla-7b \ + --lora_finetuned_checkpoint_dir /PATH/TO/CHECKPOINT/DIR/ +""" + +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Union + +import draccus +import torch +from peft import PeftModel +from transformers import AutoConfig, AutoImageProcessor, AutoModelForVision2Seq, AutoProcessor + +from prismatic.extern.hf.configuration_prismatic import OpenVLAConfig +from prismatic.extern.hf.modeling_prismatic import OpenVLAForActionPrediction +from prismatic.extern.hf.processing_prismatic import PrismaticImageProcessor, PrismaticProcessor + + +@dataclass +class ConvertConfig: + # fmt: off + + base_checkpoint: Union[str, Path] = "" # Base model checkpoint path/dir (either openvla/openvla-7b or whichever model you fine-tuned / resumed training from) + lora_finetuned_checkpoint_dir: Union[str, Path] = "" # Checkpoint directory containing the LoRA adapter + + # fmt: on + + +@draccus.wrap() +def main(cfg: ConvertConfig) -> None: + # Register OpenVLA model to HF Auto Classes (not needed if the model is on HF Hub) + AutoConfig.register("openvla", OpenVLAConfig) + AutoImageProcessor.register(OpenVLAConfig, PrismaticImageProcessor) + AutoProcessor.register(OpenVLAConfig, PrismaticProcessor) + AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) + + # Load Model using HF AutoClasses + print(f"Loading base model: {cfg.base_checkpoint}") + vla = AutoModelForVision2Seq.from_pretrained( + cfg.base_checkpoint, + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + trust_remote_code=True, + ) + + # Load LoRA weights and merge into base model, then save final checkpoint + print("Merging LoRA weights into base model...") + start_time = time.time() + merged_vla = PeftModel.from_pretrained(vla, os.path.join(cfg.lora_finetuned_checkpoint_dir, "lora_adapter")).to( + "cuda" + ) + merged_vla = merged_vla.merge_and_unload() + merged_vla.save_pretrained(cfg.lora_finetuned_checkpoint_dir) + print(f"\nMerging complete! Time elapsed (sec): {time.time() - start_time}") + print(f"\nSaved merged model checkpoint at:\n{cfg.lora_finetuned_checkpoint_dir}") + + +if __name__ == "__main__": + main() From 9bccb2a342000bfab11707c5bdfe93713fba2596 Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Fri, 28 Feb 2025 07:36:26 -0800 Subject: [PATCH 21/58] Update README: Add arXiv link --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0017aa13a..2ee296135 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success **Project website: https://openvla-oft.github.io/** -**Paper: TODO** + +**Paper: https://arxiv.org/abs/2502.19645** + **Summary video: https://youtu.be/T3Zkkr_NTSA** ## System Requirements @@ -83,8 +85,12 @@ If you run into any issues, please open a new GitHub issue. If you do not receiv ## Citation -If you use our code in your work, please cite [our paper](TODO): +If you use our code in your work, please cite [our paper](https://arxiv.org/abs/2502.19645): ```bibtex -TODO +@article{kim25finetuning, +title={Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success}, +author={{Moo Jin} Kim and Chelsea Finn and Percy Liang}, +journal = {arXiv preprint arXiv:2502.19645}, +year={2025},} ``` From 40d6c8906ec6ee224b48ebdf8feb4a733ae9f91f Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Fri, 28 Feb 2025 21:38:40 -0800 Subject: [PATCH 22/58] Update finetune.py: Sync model files on master process only Prevents race conditions in multi-GPU training runs. --- vla-scripts/finetune.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 445ae018b..d9cdde8f1 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -823,8 +823,12 @@ def finetune(cfg: FinetuneConfig) -> None: AutoModelForVision2Seq.register(OpenVLAConfig, OpenVLAForActionPrediction) # Update config.json and sync model files - update_auto_map(cfg.vla_path) - check_model_logic_mismatch(cfg.vla_path) + if distributed_state.is_main_process: + update_auto_map(cfg.vla_path) + check_model_logic_mismatch(cfg.vla_path) + + # Wait for model files to be synced + dist.barrier() # Load processor and VLA processor = AutoProcessor.from_pretrained(cfg.vla_path, trust_remote_code=True) From a45877f5d291bc9e063cff3bd92909203a502c7b Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Sat, 1 Mar 2025 00:33:48 -0800 Subject: [PATCH 23/58] Update ALOHA.md: Add ALOHA fine-tuning & eval instructions --- ALOHA.md | 123 +++++++++++++++++- .../robot/aloha/requirements_aloha.txt | 26 ++++ prismatic/vla/datasets/rlds/oxe/configs.py | 15 ++- prismatic/vla/datasets/rlds/oxe/mixtures.py | 22 ++-- prismatic/vla/datasets/rlds/oxe/transforms.py | 9 +- 5 files changed, 172 insertions(+), 23 deletions(-) create mode 100644 experiments/robot/aloha/requirements_aloha.txt diff --git a/ALOHA.md b/ALOHA.md index 1072c26ac..c9dd218a8 100644 --- a/ALOHA.md +++ b/ALOHA.md @@ -3,8 +3,8 @@ ## Relevant Files Evaluation -* `experiments/robot/aloha/`: ALOHA eval files - * `run_aloha_eval.py`: ALOHA eval script +* `experiments/robot/aloha/`: ALOHA training and eval files + * `run_aloha_eval.py`: ALOHA eval script (CLIENT SIDE; see "SERVER SIDE" below) * `aloha_utils.py`: ALOHA eval utils * Other ALOHA robot environment files copied from the original [ALOHA GitHub repo](https://github.com/tonyzhaozh/aloha): * `constants.py` @@ -13,18 +13,127 @@ Evaluation * `experiments/robot/`: General eval utils files * `openvla_utils.py`: OpenVLA-specific eval utils * `robot_utils.py`: Other eval utils +* `vla-scripts/deploy.py`: VLA server deploy script (SERVER SIDE) + +Note: Unlike the LIBERO evaluation setup, we use a server-client interface here. This is particularly useful if the user's machine which commands the robot does not have access to a local GPU with sufficient specs to run the fine-tuned VLA policies. Training +* `experiments/robot/aloha/`: ALOHA training and eval files + * * `vla-scripts/finetune.py`: VLA fine-tuning script ## Setup -(Coming soon!) +Set up a conda environment for training policies and deploying them on the VLA server (see instructions in [SETUP.md](SETUP.md)). + +## Fine-Tuning on ALOHA Robot Data + +We assume that you have collected a set of expert demonstrations on the ALOHA robot already. + +First, use our `preprocess_split_aloha_data.py` script to preprocess the raw ALOHA dataset: downsize images from 480x640 to 256x256 and split into training and validation sets. Below are examples for the `put X into pot` task in our paper: + +```bash +python experiments/robot/aloha/preprocess_split_aloha_data.py \ + --dataset_path /scr/moojink/data/aloha1_raw/put_green_pepper_into_pot/ \ + --out_base_dir /scr/moojink/data/aloha1_preprocessed/ \ + --percent_val 0.05 +python experiments/robot/aloha/preprocess_split_aloha_data.py \ + --dataset_path /scr/moojink/data/aloha1_raw/put_red_pepper_into_pot/ \ + --out_base_dir /scr/moojink/data/aloha1_preprocessed/ \ + --percent_val 0.05 +python experiments/robot/aloha/preprocess_split_aloha_data.py \ + --dataset_path /scr/moojink/data/aloha1_raw/put_yellow_corn_into_pot/ \ + --out_base_dir /scr/moojink/data/aloha1_preprocessed/ \ + --percent_val 0.05 +``` + +Then, convert the preprocessed ALOHA datasets into a single RLDS dataset that is compatible with OpenVLA fine-tuning. This process is the same as in the original OpenVLA repo. See instructions for converting to RLDS [here](https://github.com/moojink/rlds_dataset_builder) (a sample ALOHA preprocessed-to-RLDS conversion script is available [here](https://github.com/moojink/rlds_dataset_builder/blob/main/aloha1_put_X_into_pot_300_demos/aloha1_put_X_into_pot_300_demos_dataset_builder.py); this script converts the three preprocessed datasets above into one unified RLDS dataset, with train/val splits). + +After converting to RLDS, register the dataset with our dataloader by adding an entry for it in `configs.py` ([here](prismatic/vla/datasets/rlds/oxe/configs.py#L680)), `transforms.py` ([here](prismatic/vla/datasets/rlds/oxe/transforms.py#L928)), and `mixtures.py` ([here](prismatic/vla/datasets/rlds/oxe/mixtures.py#L216)). For reference, in each of these files, there are sample entries for the ALOHA datasets that we used in our paper. + +Before fine-tuning, set the desired ALOHA action chunk size in [`prismatic/vla/constants.py`](prismatic/vla/constants.py) (see `NUM_ACTIONS_CHUNK` in `ALOHA_CONSTANTS`). We set it to 25 by default because we used a control frequency of 25 Hz in our ALOHA setup to reduce storage costs and training time (while still maintaining smoothness in the robot's motions). If you use 50 Hz, we recommend setting `NUM_ACTIONS_CHUNK` to `50`. In general, 1 second-long action chunks are a good default. + +Now begin fine-tuning! Below is a sample command to fine-tune OpenVLA using our OFT+ recipe on the `put X into pot` task above ("+" in "OFT+" means FiLM is included for enhanced language grounding). Replace `X` in the first line with the number of GPUs available to you. + +```bash +torchrun --standalone --nnodes 1 --nproc-per-node X vla-scripts/finetune.py \ + --vla_path openvla/openvla-7b \ + --data_root_dir /PATH/TO/RLDS/DATASETS/DIR/ \ + --dataset_name aloha1_put_X_into_pot_300_demos \ + --run_root_dir /YOUR/CHECKPOINTS/AND/LOG/DIR/ \ + --use_l1_regression True \ + --use_diffusion False \ + --use_film True \ + --num_images_in_input 3 \ + --use_proprio True \ + --batch_size 4 \ + --learning_rate 5e-4 \ + --num_steps_before_decay 50000 \ + --max_steps 100005 \ + --use_val_set True \ + --val_freq 10000 \ + --save_freq 10000 \ + --save_latest_checkpoint_only False \ + --image_aug True \ + --lora_rank 32 \ + --wandb_entity "YOUR_WANDB_ENTITY" \ + --wandb_project "YOUR_WANDB_PROJECT" \ + --run_id_note parallel_dec--25_acts_chunk--continuous_acts--L1_regression--3rd_person_img--left_right_wrist_imgs--proprio_state--film +``` + +The above training command should reproduce our OpenVLA-OFT+ results on the `put X into pot` task if `X = 8` and the 100K step checkpoint is evaluated. It will fine-tune OpenVLA using 3 input images (1 third-person image + 2 wrist camera images). Note that we use learning rate decay after a certain point (50K steps in the command above) since doing so speeds up training convergence (train L1 loss spikes down from our experience). + +Best practices for fine-tuning: +* In general, we recommend fine-tuning until training L1 loss goes below 0.01 and starts to plateau. + * One way to achieve this is to fine-tune using our default learning rate of `5e-4` until the loss starts to decrease very slowly, and then decay the learning rate by 10x to `5e-5` (which should make the loss spike down) and train until the training L1 loss finally plateaus. +* Depending on your dataset size, you may need to adjust some hyperparameters. For example, if you use a large dataset with over 300 demos, you may need to decay the learning rate later and train for longer for best performance. Decaying too earlier can lead to a suboptimal policy. +* If your task does not require good langauge grounding (e.g., if there is only one language instruction), FiLM is not necessary; consider setting `--use_film False` to train fewer model parameters. +* Please be sure to test your policy with the same device/GPU used to train it! Otherwise, performance may drop substantially. You may be able to avoid the performance drop if you merge the LoRA weights into the base model on the downstream device used for testing (e.g., if you train on H100 and then merge on A100 before testing on A100). You can see our script [vla-scripts/merge_lora_weights_and_save.py](vla-scripts/merge_lora_weights_and_save.py) for merging the LoRA adapter into the base model offline. It's okay if you already merged LoRA weights into the base OpenVLA model during fine-tuning; you can always redownload the base model and merge again as long as you still have the LoRA adapter (`merge_lora_weights_and_save.py` will handle this for you). + +If you run into any issues, please open a new GitHub issue. + +## Launching ALOHA Robot Evaluations + +On the machine that you will use to command the robot, set up a second lightweight conda environment that will be used to run the robot environment, query the VLA server, and execute actions in the environment: + +```bash +# Create and activate client conda environment +# NOTE: We set `python=3.8.10` (different from server conda env) to be compatible with ROS Noetic! +conda create -n openvla-oft-aloha python=3.8.10 -y +conda activate openvla-oft-aloha + +# Install PyTorch +# Use a command specific to your machine: https://pytorch.org/get-started/locally/ +pip3 install torch torchvision torchaudio + +# Install packages needed for the ALOHA robot environment +pip install -r experiments/robot/aloha/requirements_aloha.txt +``` + +Launch the VLA server on the machine that has the GPU you will use to run model inference (using the `openvla-oft` conda environment). Below is a sample command for this (change as needed): -## Launching LIBERO Evaluations +```bash +python vla-scripts/deploy.py \ + --pretrained_checkpoint /PATH/TO/FINETUNED/MODEL/CHECKPOINT/DIR/ \ + --use_l1_regression True \ + --use_film True \ + --num_images_in_input 3 \ + --use_proprio True + --center_crop True \ + --num_open_loop_steps 25 \ + --unnorm_key aloha1_put_X_into_pot_300_demos \ +``` -(Coming soon!) +Then, run the ALOHA evaluation script. Specify the VLA server URL or IP address in the `vla_server_url` argument. Below is a sample command: -## Fine-Tuning on LIBERO Datasets +```bash +python experiments/robot/aloha/run_aloha_eval.py \ + --center_crop True \ + --num_open_loop_steps 25 \ + --use_vla_server True \ + --vla_server_url \ + --num_rollouts_planned \ + --max_steps +``` -(Coming soon!) +If you run into any issues, please open a new GitHub issue. diff --git a/experiments/robot/aloha/requirements_aloha.txt b/experiments/robot/aloha/requirements_aloha.txt new file mode 100644 index 000000000..c84c6d08c --- /dev/null +++ b/experiments/robot/aloha/requirements_aloha.txt @@ -0,0 +1,26 @@ +numpy<2 +draccus +torchvision +torch +pyquaternion +pyyaml +rospkg +pexpect +mujoco==2.3.7 +dm_control==1.0.14 +opencv-python +matplotlib +einops +packaging +h5py +traitlets +ipdb +IPython +modern_robotics +Pillow +termcolor +imageio[ffmpeg] +uvicorn +fastapi +requests +json_numpy diff --git a/prismatic/vla/datasets/rlds/oxe/configs.py b/prismatic/vla/datasets/rlds/oxe/configs.py index b8ab2b785..3222e023b 100644 --- a/prismatic/vla/datasets/rlds/oxe/configs.py +++ b/prismatic/vla/datasets/rlds/oxe/configs.py @@ -670,29 +670,36 @@ class ActionEncoding(IntEnum): "state_encoding": StateEncoding.POS_EULER, "action_encoding": ActionEncoding.EEF_POS, }, + "libero_4_task_suites_no_noops": { + "image_obs_keys": {"primary": "image", "secondary": None, "wrist": "wrist_image"}, + "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, + "state_obs_keys": ["EEF_state", "gripper_state"], + "state_encoding": StateEncoding.POS_EULER, + "action_encoding": ActionEncoding.EEF_POS, + }, ### ALOHA fine-tuning datasets - "openvla_oft_aloha_fold_shorts_20_demos": { + "aloha1_fold_shorts_20_demos": { "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, "state_obs_keys": ["state"], "state_encoding": StateEncoding.JOINT_BIMANUAL, "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, }, - "openvla_oft_aloha_fold_shirt_30_demos": { + "aloha1_fold_shirt_30_demos": { "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, "state_obs_keys": ["state"], "state_encoding": StateEncoding.JOINT_BIMANUAL, "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, }, - "openvla_oft_aloha_scoop_x_into_bowl_45_demos": { + "aloha1_scoop_X_into_bowl_45_demos": { "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, "state_obs_keys": ["state"], "state_encoding": StateEncoding.JOINT_BIMANUAL, "action_encoding": ActionEncoding.JOINT_POS_BIMANUAL, }, - "openvla_oft_aloha_put_x_into_pot_300_demos": { + "aloha1_put_X_into_pot_300_demos": { "image_obs_keys": {"primary": "image", "secondary": None, "left_wrist": "left_wrist_image", "right_wrist": "right_wrist_image"}, "depth_obs_keys": {"primary": None, "secondary": None, "wrist": None}, "state_obs_keys": ["state"], diff --git a/prismatic/vla/datasets/rlds/oxe/mixtures.py b/prismatic/vla/datasets/rlds/oxe/mixtures.py index feb47c8a0..c5a2862fd 100644 --- a/prismatic/vla/datasets/rlds/oxe/mixtures.py +++ b/prismatic/vla/datasets/rlds/oxe/mixtures.py @@ -206,19 +206,25 @@ "libero_10_no_noops": [ ("libero_10_no_noops", 1.0), ], + "libero_4_task_suites_no_noops": [ + ("libero_spatial_no_noops", 1.0), + ("libero_object_no_noops", 1.0), + ("libero_goal_no_noops", 1.0), + ("libero_10_no_noops", 1.0), + ], # === ALOHA Fine-Tuning Datasets === - "openvla_oft_aloha_fold_shorts_20_demos": [ - ("openvla_oft_aloha_fold_shorts_20_demos", 1.0), + "aloha1_fold_shorts_20_demos": [ + ("aloha1_fold_shorts_20_demos", 1.0), ], - "openvla_oft_aloha_fold_shirt_30_demos": [ - ("openvla_oft_aloha_fold_shirt_30_demos", 1.0), + "aloha1_fold_shirt_30_demos": [ + ("aloha1_fold_shirt_30_demos", 1.0), ], - "openvla_oft_aloha_scoop_x_into_bowl_45_demos": [ - ("openvla_oft_aloha_scoop_x_into_bowl_45_demos", 1.0), + "aloha1_scoop_X_into_bowl_45_demos": [ + ("aloha1_scoop_X_into_bowl_45_demos", 1.0), ], - "openvla_oft_aloha_put_x_into_pot_300_demos": [ - ("openvla_oft_aloha_put_x_into_pot_300_demos", 1.0), + "aloha1_put_X_into_pot_300_demos": [ + ("aloha1_put_X_into_pot_300_demos", 1.0), ], # fmt: on } diff --git a/prismatic/vla/datasets/rlds/oxe/transforms.py b/prismatic/vla/datasets/rlds/oxe/transforms.py index 405f80b86..bf848e98f 100644 --- a/prismatic/vla/datasets/rlds/oxe/transforms.py +++ b/prismatic/vla/datasets/rlds/oxe/transforms.py @@ -924,9 +924,10 @@ def aloha_dataset_transform(trajectory: Dict[str, Any]) -> Dict[str, Any]: "libero_object_no_noops": libero_dataset_transform, "libero_goal_no_noops": libero_dataset_transform, "libero_10_no_noops": libero_dataset_transform, + "libero_4_task_suites_no_noops": libero_dataset_transform, ### ALOHA fine-tuning datasets - "openvla_oft_aloha_fold_shorts_20_demos": aloha_dataset_transform, - "openvla_oft_aloha_fold_shirt_30_demos": aloha_dataset_transform, - "openvla_oft_aloha_scoop_x_into_bowl_45_demos": aloha_dataset_transform, - "openvla_oft_aloha_put_x_into_pot_300_demos": aloha_dataset_transform, + "aloha1_fold_shorts_20_demos": aloha_dataset_transform, + "aloha1_fold_shirt_30_demos": aloha_dataset_transform, + "aloha1_scoop_X_into_bowl_45_demos": aloha_dataset_transform, + "aloha1_put_X_into_pot_300_demos": aloha_dataset_transform, } From 588f9f96af3771af8f6ce6fc81a38bd6de46a6d3 Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Fri, 7 Mar 2025 10:42:55 -0800 Subject: [PATCH 24/58] Update LIBERO.md: Add lfs to git clone command --- LIBERO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LIBERO.md b/LIBERO.md index fcadfc6fd..851af31cd 100644 --- a/LIBERO.md +++ b/LIBERO.md @@ -32,7 +32,7 @@ and LIBERO-10 datasets in RLDS data format (~10 GB total). You can use these to train other methods. This step is optional since we provide pretrained OpenVLA-OFT checkpoints below. Note that these are the same datasets used in the original OpenVLA project. If needed, see details on how to download the original non-RLDS datasets [here](https://github.com/openvla/openvla?tab=readme-ov-file#libero-setup). ```bash -git clone git@hf.co:datasets/openvla/modified_libero_rlds +git lfs clone git@hf.co:datasets/openvla/modified_libero_rlds ``` ## Launching LIBERO Evaluations From 2fde5e960abff00786d12c49a596cf0ab5e5aa2f Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Fri, 7 Mar 2025 10:47:35 -0800 Subject: [PATCH 25/58] Update ALOHA eval instructions and VRAM usage note --- ALOHA.md | 36 ++++++++++++++++++----- README.md | 2 +- experiments/robot/aloha/run_aloha_eval.py | 1 + 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ALOHA.md b/ALOHA.md index c9dd218a8..c4f5631a8 100644 --- a/ALOHA.md +++ b/ALOHA.md @@ -19,7 +19,7 @@ Note: Unlike the LIBERO evaluation setup, we use a server-client interface here. Training * `experiments/robot/aloha/`: ALOHA training and eval files - * + * `preprocess_split_aloha_data.py`: ALOHA data preprocessing script * `vla-scripts/finetune.py`: VLA fine-tuning script ## Setup @@ -30,7 +30,7 @@ Set up a conda environment for training policies and deploying them on the VLA s We assume that you have collected a set of expert demonstrations on the ALOHA robot already. -First, use our `preprocess_split_aloha_data.py` script to preprocess the raw ALOHA dataset: downsize images from 480x640 to 256x256 and split into training and validation sets. Below are examples for the `put X into pot` task in our paper: +First, use our `preprocess_split_aloha_data.py` script to preprocess the raw ALOHA dataset: downsize images from 480x640 to 256x256 and split into training and validation sets. Below are examples for the `put X into pot` task in our paper (which has 3 possible target objects, 1 per episode): ```bash python experiments/robot/aloha/preprocess_split_aloha_data.py \ @@ -49,9 +49,9 @@ python experiments/robot/aloha/preprocess_split_aloha_data.py \ Then, convert the preprocessed ALOHA datasets into a single RLDS dataset that is compatible with OpenVLA fine-tuning. This process is the same as in the original OpenVLA repo. See instructions for converting to RLDS [here](https://github.com/moojink/rlds_dataset_builder) (a sample ALOHA preprocessed-to-RLDS conversion script is available [here](https://github.com/moojink/rlds_dataset_builder/blob/main/aloha1_put_X_into_pot_300_demos/aloha1_put_X_into_pot_300_demos_dataset_builder.py); this script converts the three preprocessed datasets above into one unified RLDS dataset, with train/val splits). -After converting to RLDS, register the dataset with our dataloader by adding an entry for it in `configs.py` ([here](prismatic/vla/datasets/rlds/oxe/configs.py#L680)), `transforms.py` ([here](prismatic/vla/datasets/rlds/oxe/transforms.py#L928)), and `mixtures.py` ([here](prismatic/vla/datasets/rlds/oxe/mixtures.py#L216)). For reference, in each of these files, there are sample entries for the ALOHA datasets that we used in our paper. +After converting to RLDS, register the dataset (which, for the example task above, would be called `aloha1_put_X_into_pot_300_demos`) with our dataloader by adding an entry for it in `configs.py` ([here](prismatic/vla/datasets/rlds/oxe/configs.py#L680)), `transforms.py` ([here](prismatic/vla/datasets/rlds/oxe/transforms.py#L928)), and `mixtures.py` ([here](prismatic/vla/datasets/rlds/oxe/mixtures.py#L216)). For reference, in each of these files, there are sample entries for the ALOHA datasets that we used in our paper. -Before fine-tuning, set the desired ALOHA action chunk size in [`prismatic/vla/constants.py`](prismatic/vla/constants.py) (see `NUM_ACTIONS_CHUNK` in `ALOHA_CONSTANTS`). We set it to 25 by default because we used a control frequency of 25 Hz in our ALOHA setup to reduce storage costs and training time (while still maintaining smoothness in the robot's motions). If you use 50 Hz, we recommend setting `NUM_ACTIONS_CHUNK` to `50`. In general, 1 second-long action chunks are a good default. +Before fine-tuning, set the desired ALOHA action chunk size in [`prismatic/vla/constants.py`](prismatic/vla/constants.py) (see `NUM_ACTIONS_CHUNK` in `ALOHA_CONSTANTS`). We set it to 25 by default because we used a control frequency of 25 Hz in our ALOHA setup to reduce storage costs and training time (while still maintaining smoothness in the robot's motions). If you use 50 Hz, we recommend setting `NUM_ACTIONS_CHUNK` to `50`. In general, 1 second-long action chunks are a good default. Do NOT modify `ACTION_PROPRIO_NORMALIZATION_TYPE`: Since the ALOHA robot action space is absolute joint angles, we do not want to use a normalization scheme that clips outlier values (like the Q1-Q99 normalization we used with the relative end-effector pose actions for LIBERO), since that would prevent the model from outputting certain robot joint angles that are crucial for solving the task. Now begin fine-tuning! Below is a sample command to fine-tune OpenVLA using our OFT+ recipe on the `put X into pot` task above ("+" in "OFT+" means FiLM is included for enhanced language grounding). Replace `X` in the first line with the number of GPUs available to you. @@ -94,18 +94,30 @@ If you run into any issues, please open a new GitHub issue. ## Launching ALOHA Robot Evaluations -On the machine that you will use to command the robot, set up a second lightweight conda environment that will be used to run the robot environment, query the VLA server, and execute actions in the environment: +In the primary conda environment (`openvla-oft`) which you will use to launch the VLA server, install a few packages for the server-client interface: + +```bash +conda activate openvla-oft +pip install uvicorn fastapi json-numpy +``` + +On the machine that you will use to command the robot, set up a second conda environment that will be used to run the robot environment, query the VLA server, and execute actions in the environment: ```bash # Create and activate client conda environment # NOTE: We set `python=3.8.10` (different from server conda env) to be compatible with ROS Noetic! -conda create -n openvla-oft-aloha python=3.8.10 -y +conda create -n openvla-oft-aloha python=3.10 -y conda activate openvla-oft-aloha # Install PyTorch # Use a command specific to your machine: https://pytorch.org/get-started/locally/ pip3 install torch torchvision torchaudio +# Clone openvla-oft repo and pip install to download dependencies +git clone https://github.com/moojink/openvla-oft.git +cd openvla-oft +pip install -e . + # Install packages needed for the ALOHA robot environment pip install -r experiments/robot/aloha/requirements_aloha.txt ``` @@ -118,10 +130,10 @@ python vla-scripts/deploy.py \ --use_l1_regression True \ --use_film True \ --num_images_in_input 3 \ - --use_proprio True + --use_proprio True \ --center_crop True \ --num_open_loop_steps 25 \ - --unnorm_key aloha1_put_X_into_pot_300_demos \ + --unnorm_key aloha1_put_X_into_pot_300_demos ``` Then, run the ALOHA evaluation script. Specify the VLA server URL or IP address in the `vla_server_url` argument. Below is a sample command: @@ -137,3 +149,11 @@ python experiments/robot/aloha/run_aloha_eval.py \ ``` If you run into any issues, please open a new GitHub issue. + +## Troubleshooting Tips + +* Tip #1: If you run into a ROS error such as `ImportError: /lib/x86_64-linux-gnu/libp11-kit.so.0: undefined symbol: ffi_type_pointer, version LIBFFI_BASE_7.0`, try running the following command in your client conda environment (`openvla-oft-aloha`): + + ``` + conda install -c conda-forge libffi + ``` diff --git a/README.md b/README.md index 2ee296135..8e8ab5b2e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Inference: * 1 GPU with ~16 GB VRAM for LIBERO sim benchmark tasks -* 1 GPU with ~20 GB VRAM for ALOHA robot tasks +* 1 GPU with ~25 GB VRAM for ALOHA robot tasks Training: * Between 1-8 GPUs with 27-80 GB, depending on the desired training setup (with default bfloat16 data type). See [this FAQ on our project website](https://openvla-oft.github.io/#train-compute) for details. diff --git a/experiments/robot/aloha/run_aloha_eval.py b/experiments/robot/aloha/run_aloha_eval.py index 177883487..520f5af9a 100644 --- a/experiments/robot/aloha/run_aloha_eval.py +++ b/experiments/robot/aloha/run_aloha_eval.py @@ -103,6 +103,7 @@ def setup_logging(cfg: GenerateConfig): def log_message(message: str, log_file=None): """Log a message to console and optionally to a log file.""" + print(message) logger.info(message) if log_file: log_file.write(message + "\n") From 0b75f6f77d4716ba7e4701b5b0e94dd10baa236f Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Fri, 7 Mar 2025 11:17:43 -0800 Subject: [PATCH 26/58] Update openvla_utils.py: Add torch.inference_mode() to get_vla_action() Also update ALOHA VRAM usage in README.md --- experiments/robot/openvla_utils.py | 104 +++++++++++++++-------------- 1 file changed, 53 insertions(+), 51 deletions(-) diff --git a/experiments/robot/openvla_utils.py b/experiments/robot/openvla_utils.py index 30d63b884..570d1cff5 100644 --- a/experiments/robot/openvla_utils.py +++ b/experiments/robot/openvla_utils.py @@ -736,57 +736,59 @@ def get_vla_action( Returns: List[np.ndarray]: Predicted actions """ - # Collect all input images - all_images = [obs["full_image"]] - if cfg.num_images_in_input > 1: - all_images.extend([obs[k] for k in obs.keys() if "wrist" in k]) - - # Process images - all_images = prepare_images_for_vla(all_images, cfg) - - # Extract primary image and additional images - primary_image = all_images.pop(0) - - # Build VLA prompt - prompt = f"In: What action should the robot take to {task_label.lower()}?\nOut:" - - # Process primary image - inputs = processor(prompt, primary_image).to(DEVICE, dtype=torch.bfloat16) - - # Process additional wrist images if any - if all_images: - all_wrist_inputs = [ - processor(prompt, image_wrist).to(DEVICE, dtype=torch.bfloat16) for image_wrist in all_images - ] - # Concatenate all images - primary_pixel_values = inputs["pixel_values"] - all_wrist_pixel_values = [wrist_inputs["pixel_values"] for wrist_inputs in all_wrist_inputs] - inputs["pixel_values"] = torch.cat([primary_pixel_values] + all_wrist_pixel_values, dim=1) - - # Process proprioception data if used - proprio = None - if cfg.use_proprio: - proprio = obs["state"] - proprio_norm_stats = vla.norm_stats[cfg.unnorm_key]["proprio"] - obs["state"] = normalize_proprio(proprio, proprio_norm_stats) - proprio = obs["state"] - - # Generate action - if action_head is None: - # Standard VLA output (single-image inputs, discrete actions) - action, _ = vla.predict_action(**inputs, unnorm_key=cfg.unnorm_key, do_sample=False) - else: - # Custom action head for continuous actions - action, _ = vla.predict_action( - **inputs, - unnorm_key=cfg.unnorm_key, - do_sample=False, - proprio=proprio, - proprio_projector=proprio_projector, - noisy_action_projector=noisy_action_projector, - action_head=action_head, - use_film=use_film, - ) + with torch.inference_mode(): + + # Collect all input images + all_images = [obs["full_image"]] + if cfg.num_images_in_input > 1: + all_images.extend([obs[k] for k in obs.keys() if "wrist" in k]) + + # Process images + all_images = prepare_images_for_vla(all_images, cfg) + + # Extract primary image and additional images + primary_image = all_images.pop(0) + + # Build VLA prompt + prompt = f"In: What action should the robot take to {task_label.lower()}?\nOut:" + + # Process primary image + inputs = processor(prompt, primary_image).to(DEVICE, dtype=torch.bfloat16) + + # Process additional wrist images if any + if all_images: + all_wrist_inputs = [ + processor(prompt, image_wrist).to(DEVICE, dtype=torch.bfloat16) for image_wrist in all_images + ] + # Concatenate all images + primary_pixel_values = inputs["pixel_values"] + all_wrist_pixel_values = [wrist_inputs["pixel_values"] for wrist_inputs in all_wrist_inputs] + inputs["pixel_values"] = torch.cat([primary_pixel_values] + all_wrist_pixel_values, dim=1) + + # Process proprioception data if used + proprio = None + if cfg.use_proprio: + proprio = obs["state"] + proprio_norm_stats = vla.norm_stats[cfg.unnorm_key]["proprio"] + obs["state"] = normalize_proprio(proprio, proprio_norm_stats) + proprio = obs["state"] + + # Generate action + if action_head is None: + # Standard VLA output (single-image inputs, discrete actions) + action, _ = vla.predict_action(**inputs, unnorm_key=cfg.unnorm_key, do_sample=False) + else: + # Custom action head for continuous actions + action, _ = vla.predict_action( + **inputs, + unnorm_key=cfg.unnorm_key, + do_sample=False, + proprio=proprio, + proprio_projector=proprio_projector, + noisy_action_projector=noisy_action_projector, + action_head=action_head, + use_film=use_film, + ) # Extract subset of actions for open loop steps return [action[i] for i in range(min(len(action), cfg.num_open_loop_steps))] From 7f9efa6d8d3c2cdc25aa9835448f83b0c84cd2a0 Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Fri, 7 Mar 2025 11:26:34 -0800 Subject: [PATCH 27/58] Update README: Update ALOHA VRAM usage to 18 GB (Previous commit was supposed to do this) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e8ab5b2e..63e9db326 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Inference: * 1 GPU with ~16 GB VRAM for LIBERO sim benchmark tasks -* 1 GPU with ~25 GB VRAM for ALOHA robot tasks +* 1 GPU with ~18 GB VRAM for ALOHA robot tasks Training: * Between 1-8 GPUs with 27-80 GB, depending on the desired training setup (with default bfloat16 data type). See [this FAQ on our project website](https://openvla-oft.github.io/#train-compute) for details. From d8bf4996d6a7a05f1320a2a6f1e7a3e90e1cd38d Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Sat, 8 Mar 2025 09:41:27 -0800 Subject: [PATCH 28/58] Update aloha/constants.py: 25 Hz control, updated task configs --- experiments/robot/aloha/constants.py | 62 ++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/experiments/robot/aloha/constants.py b/experiments/robot/aloha/constants.py index 20cf90099..5599e3590 100644 --- a/experiments/robot/aloha/constants.py +++ b/experiments/robot/aloha/constants.py @@ -1,17 +1,63 @@ ### Task parameters -DATA_DIR = '' +DATA_DIR = '/scr2/moojink/data/aloha1/' TASK_CONFIGS = { - 'aloha_wear_shoe':{ - 'dataset_dir': DATA_DIR + '/aloha_wear_shoe', - 'num_episodes': 50, + # fold shorts + 'fold_shorts':{ + 'dataset_dir': DATA_DIR + '/fold_shorts', + 'num_episodes': 20, 'episode_len': 1000, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + # fold shirt + 'fold_shirt':{ + 'dataset_dir': DATA_DIR + '/fold_shirt', + 'num_episodes': 30, + 'episode_len': 1250, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + # scoop X into bowl + 'scoop_raisins_into_bowl':{ + 'dataset_dir': DATA_DIR + '/scoop_raisins_into_bowl', + 'num_episodes': 15, + 'episode_len': 900, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + 'scoop_almonds_and_green_M&Ms_into_bowl':{ + 'dataset_dir': DATA_DIR + '/scoop_almonds_and_green_M&Ms_into_bowl', + 'num_episodes': 15, + 'episode_len': 900, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + 'scoop_pretzels_into_bowl':{ + 'dataset_dir': DATA_DIR + '/scoop_pretzels_into_bowl', + 'num_episodes': 15, + 'episode_len': 900, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + # put X into pot + 'put_red_pepper_into_pot':{ + 'dataset_dir': DATA_DIR + '/put_red_pepper_into_pot', + 'num_episodes': 100, + 'episode_len': 400, + 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + }, + 'put_yellow_corn_into_pot':{ + 'dataset_dir': DATA_DIR + '/put_yellow_corn_into_pot', + 'num_episodes': 100, + 'episode_len': 400, + 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + }, + 'put_green_pepper_into_pot':{ + 'dataset_dir': DATA_DIR + '/put_green_pepper_into_pot', + 'num_episodes': 100, + 'episode_len': 400, 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] }, } ### ALOHA fixed constants -DT = 0.02 +DT = 0.04 # 1 / 0.04 -> 25 Hz JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] START_ARM_POSE = [0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239, 0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239] @@ -22,8 +68,10 @@ PUPPET_GRIPPER_POSITION_CLOSE = 0.01844 # Gripper joint limits (qpos[6]) -MASTER_GRIPPER_JOINT_OPEN = 0.3083 -MASTER_GRIPPER_JOINT_CLOSE = -0.6842 +MASTER_GRIPPER_JOINT_OPEN = 0.3083 # For ALOHA 1 +MASTER_GRIPPER_JOINT_CLOSE = -0.6842 # For ALOHA 1 +# MASTER_GRIPPER_JOINT_OPEN = -0.8 # For ALOHA 2 +# MASTER_GRIPPER_JOINT_CLOSE = -1.65 # For ALOHA 2 PUPPET_GRIPPER_JOINT_OPEN = 1.4910 PUPPET_GRIPPER_JOINT_CLOSE = -0.6213 From b9d7702dbe801b5334a6a0ebac94c2a52fa0fb46 Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Sat, 8 Mar 2025 09:41:52 -0800 Subject: [PATCH 29/58] Update ALOHA.md: Remove comment about Python 3.8.10 (Python 3.10 actually works for the client conda environment) --- ALOHA.md | 1 - 1 file changed, 1 deletion(-) diff --git a/ALOHA.md b/ALOHA.md index c4f5631a8..bc508fcfe 100644 --- a/ALOHA.md +++ b/ALOHA.md @@ -105,7 +105,6 @@ On the machine that you will use to command the robot, set up a second conda env ```bash # Create and activate client conda environment -# NOTE: We set `python=3.8.10` (different from server conda env) to be compatible with ROS Noetic! conda create -n openvla-oft-aloha python=3.10 -y conda activate openvla-oft-aloha From edc1f9efd56ab821dc7e231cbda302f4f0b01d18 Mon Sep 17 00:00:00 2001 From: Moo Jin Kim Date: Sun, 9 Mar 2025 11:59:37 -0700 Subject: [PATCH 30/58] Update BibTeX in README --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 63e9db326..2ee12646a 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,10 @@ If you run into any issues, please open a new GitHub issue. If you do not receiv If you use our code in your work, please cite [our paper](https://arxiv.org/abs/2502.19645): ```bibtex -@article{kim25finetuning, -title={Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success}, -author={{Moo Jin} Kim and Chelsea Finn and Percy Liang}, -journal = {arXiv preprint arXiv:2502.19645}, -year={2025},} +@article{kim2025fine, + title={Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success}, + author={Kim, Moo Jin and Finn, Chelsea and Liang, Percy}, + journal={arXiv preprint arXiv:2502.19645}, + year={2025} +} ``` From 408568d091f73147eb031d5ef380faae7afa8a9f Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 13 Mar 2025 14:02:42 +0000 Subject: [PATCH 31/58] moved lerobot submodule to third_party subdir --- .gitmodules | 3 +++ third_party/lerobot | 1 + 2 files changed, 4 insertions(+) create mode 160000 third_party/lerobot diff --git a/.gitmodules b/.gitmodules index 2d9697f66..77c2cdd50 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "lerobot"] path = lerobot url = https://github.com/huggingface/lerobot +[submodule "third_party/lerobot"] + path = third_party/lerobot + url = https://github.com/huggingface/lerobot.git diff --git a/third_party/lerobot b/third_party/lerobot new file mode 160000 index 000000000..c37b1d45b --- /dev/null +++ b/third_party/lerobot @@ -0,0 +1 @@ +Subproject commit c37b1d45b6bf5b26bf9e507aead77fb4839fa11a From a90df8e6948b5c023ee4ea9bd2fcd4fb88f1023e Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 13 Mar 2025 15:34:38 +0000 Subject: [PATCH 32/58] impl devcontainer --- .devcontainer/devcontainer.json | 36 +++++++++++++++++++ .devcontainer/docker-compose.yml | 28 +++++++++++++++ .gitmodules | 3 -- Dockerfile | 25 ++++++++----- README.md | 61 ++++++++++++++++++++------------ docker-compose.yml | 17 --------- lerobot | 1 - 7 files changed, 119 insertions(+), 52 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml delete mode 100644 docker-compose.yml delete mode 160000 lerobot diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..a8eb7a295 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,36 @@ +{ + "name": "OpenVLA Development", + "dockerComposeFile": "docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspace/openvla", + "containerEnv": { + "PYTHONPATH": "${containerWorkspaceFolder}" + }, + "remoteEnv": { + // Environment variables will be loaded from .env + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter", + "github.copilot" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.provider": "black", + "editor.formatOnSave": true, + "editor.rulers": [121] + } + } + }, + "remoteUser": "root", + "postCreateCommand": "pip install -e .", + // Load environment variables from .env file + "features": { + "ghcr.io/devcontainers/features/dotnet:1": {} + } + } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..453872554 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,28 @@ +services: + devcontainer: + build: + context: .. + dockerfile: ./Dockerfile + working_dir: /workspace/openvla + command: sleep infinity + volumes: + - ..:/workspace/openvla:cached + - finetuner-cache:/root/.cache + - ${HOME}/.cache/huggingface:/root/.cache/huggingface # use cached models/datasets from host + environment: + - WANDB_API_KEY=${WANDB_API_KEY} + - WANDB_MODE=online + - HF_HOME=/root/.cache/huggingface + - HF_TOKEN=${HF_TOKEN} + shm_size: 16gb + network_mode: host + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +volumes: + finetuner-cache: \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 77c2cdd50..1327453ea 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "aloha_sim_insertion_scripted_image"] path = aloha_sim_insertion_scripted_image url = https://huggingface.co/datasets/lerobot/aloha_sim_insertion_scripted_image -[submodule "lerobot"] - path = lerobot - url = https://github.com/huggingface/lerobot [submodule "third_party/lerobot"] path = third_party/lerobot url = https://github.com/huggingface/lerobot.git diff --git a/Dockerfile b/Dockerfile index 93f1b63ba..82e569c00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,29 @@ FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-devel -WORKDIR /home/marvin/alan/openvla_finetuner - -# Install system dependencies for flash-attn +# Install system dependencies for flash-attn. RUN apt-get update && apt-get install -y \ git \ ninja-build \ && rm -rf /var/lib/apt/lists/* -# Install openvla, lerobot, and flash-attn; download openvla-7b -COPY . /workspaces/openvla_finetuner -WORKDIR /workspaces/openvla_finetuner +WORKDIR /workspace +RUN git clone -b merge-finetuner-changes https://github.com/nomagiclab/openvla.git && \ + cd openvla && \ + git submodule init third_party/lerobot && \ + git submodule update --recursive --init third_party/lerobot + +WORKDIR /workspace/openvla + +RUN ls -la third_party/lerobot + +# Editable install of openvla, then lerobot submodule, then reinstall newly +# missing openvla dependencies to negotiate dependency incompatibility. +# Then install flash-attn separately (per OpenVLA instructions) +# and download the openvla-7b model. RUN pip install -e . && \ - cd lerobot && \ + cd third_party/lerobot/ && \ pip install -e . && \ - cd .. && \ + cd ../../ && \ pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' | xargs pip install && \ pip install packaging ninja && \ pip install "flash-attn==2.5.5" --no-build-isolation && \ diff --git a/README.md b/README.md index ef62188ed..4f980aead 100644 --- a/README.md +++ b/README.md @@ -104,44 +104,59 @@ above models are derived from Llama-2, and as such are subject to the ## Installation -> **Note**: These installation instructions are for full-scale pretraining (and distributed fine-tuning); if looking to - just run inference with OpenVLA models (or perform lightweight fine-tuning), see instructions above! +See [original instructions](https://github.com/openvla/openvla?tab=readme-ov-file#installation) for OpenVLA. -This repository was built using Python 3.10, but should be backwards compatible with any Python >= 3.8. We require -PyTorch 2.2.* -- installation instructions [can be found here](https://pytorch.org/get-started/locally/). The latest -version of this repository was developed and thoroughly tested with: - - PyTorch 2.2.0, torchvision 0.17.0, transformers 4.40.1, tokenizers 0.19.1, timm 0.9.10, and flash-attn 2.5.5 +#### Devcontainer -**[5/21/24] Note**: Following reported regressions and breaking changes in later versions of `transformers`, `timm`, and -`tokenizers` we explicitly pin the above versions of the dependencies. We are working on implementing thorough tests, -and plan on relaxing these constraints as soon as we can. +We provide a VS Code devcontainer configuration. -Use the setup commands below to get started: +1. Clone the repository with submodules: + ```bash + git clone --recurse-submodules https://github.com/openvla/openvla.git + cd openvla + ``` -```bash -# Create and activate conda environment -conda create -n openvla python=3.10 -y -conda activate openvla +2. Open the project in VS Code. + +3. When prompted by VS Code to "Reopen in Container", click "Reopen in Container". + +4. Building the container might take up to 30 minutes at first. + +To configure secrets, get a fresh `.env` file from template: + +``` +cp .env.template .env +``` +and populate it. -# Install PyTorch. Below is a sample command to do this, but you should check the following link -# to find installation instructions that are specific to your compute platform: -# https://pytorch.org/get-started/locally/ -conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch -c nvidia -y # UPDATE ME! +#### Manually managed virtualenv -# Clone and install the openvla repo -git clone https://github.com/openvla/openvla.git +Otherwise, for manual installation, something like this might work (soon to be replaced by a `uv`-managed configuration) + +```bash +# Create and activate virtualenv +virtualenv -p 3.10 openvla_env +source openvla_env/bin/activate + +# Clone the repository with submodules +git clone --recurse-submodules https://github.com/openvla/openvla.git cd openvla + +# Install OpenVLA pip install -e . +# Install lerobot (included as a submodule) +cd third_party/lerobot/ +pip install -e . +cd ../../ + # Install Flash Attention 2 for training (https://github.com/Dao-AILab/flash-attention) -# =>> If you run into difficulty, try `pip cache remove flash_attn` first +# =>> If you run into difficulty, try `pip uninstall flash_attn -y` first pip install packaging ninja ninja --version; echo $? # Verify Ninja --> should return exit code "0" pip install "flash-attn==2.5.5" --no-build-isolation ``` -If you run into any problems during the installation process, please file a GitHub Issue. - **Note:** See `vla-scripts/` for full training and verification scripts for OpenVLA models. Note that `scripts/` is mostly a holdover from the original (base) `prismatic-vlms` repository, with support for training and evaluating visually-conditioned language models; while you can use this repo to train VLMs AND VLAs, note that trying to generate diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 55fefc208..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ - -services: - openvla-finetuner: - image: openvla-finetuner:latest - build: . - runtime: nvidia - environment: - - WANDB_API_KEY=${WANDB_API_KEY} - - WANDB_MODE=online - - HF_HOME=/root/.cache/huggingface - volumes: - - ./:/workspaces/openvla_finetuner - - finetuner-cache:/root/.cache - command: sleep infinity - -volumes: - finetuner-cache: \ No newline at end of file diff --git a/lerobot b/lerobot deleted file mode 160000 index 44f9b21e7..000000000 --- a/lerobot +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 44f9b21e74936c366b55609d1847b843bd04f3ab From 7a14a1b84f690a7de83375c7f21fa657da387124 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 13 Mar 2025 15:35:05 +0000 Subject: [PATCH 33/58] rm unused submodule --- .gitmodules | 3 --- aloha_sim_insertion_scripted_image | 1 - 2 files changed, 4 deletions(-) delete mode 160000 aloha_sim_insertion_scripted_image diff --git a/.gitmodules b/.gitmodules index 1327453ea..bce919cf1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "aloha_sim_insertion_scripted_image"] - path = aloha_sim_insertion_scripted_image - url = https://huggingface.co/datasets/lerobot/aloha_sim_insertion_scripted_image [submodule "third_party/lerobot"] path = third_party/lerobot url = https://github.com/huggingface/lerobot.git diff --git a/aloha_sim_insertion_scripted_image b/aloha_sim_insertion_scripted_image deleted file mode 160000 index f32ed18e5..000000000 --- a/aloha_sim_insertion_scripted_image +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f32ed18e53003d5fb3d9727f35a1be5dcf27c431 From 54a215e2916ca7ef9fc04f19139a66bb4218f418 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 13 Mar 2025 15:37:36 +0000 Subject: [PATCH 34/58] rm old lerobot conv script --- .../additional-datasets/nomagic_ur5e_raw.py | 669 ------------------ 1 file changed, 669 deletions(-) delete mode 100644 scripts/additional-datasets/nomagic_ur5e_raw.py diff --git a/scripts/additional-datasets/nomagic_ur5e_raw.py b/scripts/additional-datasets/nomagic_ur5e_raw.py deleted file mode 100644 index 22626f4f2..000000000 --- a/scripts/additional-datasets/nomagic_ur5e_raw.py +++ /dev/null @@ -1,669 +0,0 @@ -""" -convert_nomagic_ur5e_raw.py - -Convert raw CSV/MP4 data from Nomagic's UR5e robot arm into a LeRobotDataset. - -The data should be in a format similar to the following: - -raw/trajectories/ -urXPose_20241219_133722.csv -urXPose_20241219_133814.csv - ... - -raw/videos/ - 2024-12-19-12:37:26:897865_d5fb919d-2b3a-4d4b-b885-1f890a255b66.mp4 - 2024-12-19-12:38:19:058352_99a53149-eb87-4a0d-979f-01b1283c5804.mp4 - ... - -The final directory will look like: - -data/my_lerobot_dataset/ - data/ - chunk-000/ - episode_000000.parquet - episode_000001.parquet - ... - meta/ - info.json - stats.json - episodes.jsonl - tasks.jsonl - videos/ - chunk-000/ - observation.images.side/ - episode_000000.mp4 - episode_000001.mp4 - ... - -Notes: - - This script assumes that the CSV files have a matching MP4 file - by a shared timestamp in the filename, e.g."urXPose_20241219_133722.csv" - ↔ "2024-12-19-12:37:26:897865_d5fb919d-2b3a-4d4b-b885-1f890a255b66.mp4" -""" - -import exiftool -import os -import re -import json -import shutil -import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -import cv2 -from typing import Optional - -from pathlib import Path -from typing import List, Dict -import torch -import numpy as np -from dataclasses import dataclass -from scipy.spatial.transform import Rotation as R - -from datasets import Dataset -from lerobot.common.datasets.utils import ( - check_timestamps_sync, - calculate_episode_data_index -) - -import logging - -# Set up logging -logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler() # Add a stream handler to show messages in console - ] -) - -@dataclass -class LeRobotTrajectoryStep: - """ - Represents a single entry (frame) in the final LeRobot parquet data. - For example: - timestamp (sec), - episode_index, - next.done (bool), - task_index, - index, # global index in the episode - frame_index, # might be same as index, or you can offset if needed - action # a list of [dx, dy, dz, dox, doy, doz, grip] - """ - timestamp: float - episode_index: int - next_done: bool - task_index: int - index: int - frame_index: int - action: List[float] - - -def pair_trajectories_and_videos( - csv_trajectories_dir: Path, - mp4_videos_dir: Path -) -> list[tuple[Path, Path]]: - """ - Pair up CSV trajectories and MP4 videos. - Uses alphanumeric order on filenames. - """ - - csv_trajectories = sorted(csv_trajectories_dir.glob("*.csv")) - mp4_videos = sorted(mp4_videos_dir.glob("*.mp4")) - - logging.debug(f"Found CSV files: {[f.name for f in csv_trajectories]}") - logging.debug(f"Found MP4 files: {[f.name for f in mp4_videos]}") - - # We pair them up in order since they're already sorted chronologically - # TODO: compare actual timestamps in filename, not alphanumeric order - - # We pair them up alphanumerically - trajectory_video_pairs = list(zip(csv_trajectories, mp4_videos)) - - for csv_trajectory, mp4_video in trajectory_video_pairs: - logging.debug(f"Paired {csv_trajectory.name} with {mp4_video.name}") - - return trajectory_video_pairs - -def compute_actions_from_rows(rowA: pd.Series, rowB: pd.Series): - """ - Convert two consecutive CSV rows into a delta action (dx, dy, dz, dox, doy, doz, grip). - Calculate Euler angle difference (dox, doy, doz) from quaternion pair using scipy.Rotation. - """ - # Position deltas - dx = float(rowB["PositionX"] - rowA["PositionX"]) - dy = float(rowB["PositionY"] - rowA["PositionY"]) - dz = float(rowB["PositionZ"] - rowA["PositionZ"]) - - # Get quaternions in [x,y,z,w] format for scipy.Rotation - q1 = [rowA["OrientationX"], rowA["OrientationY"], - rowA["OrientationZ"], rowA["OrientationW"]] - q2 = [rowB["OrientationX"], rowB["OrientationY"], - rowB["OrientationZ"], rowB["OrientationW"]] - - # Calculate orientation difference - r1 = R.from_rotvec( - [ - rowA["OrientationX"], - rowA["OrientationY"], - rowA["OrientationZ"] - ] - ) - r2 = R.from_rotvec( - [ - rowB["OrientationX"], - rowB["OrientationY"], - rowB["OrientationZ"] - ] - ) - r_diff = r2 * r1.inv() - euler_diff = r_diff.as_euler("xyz") - dox, doy, doz = euler_diff - - # Gripper - map Gripper::Action values to float - grip_val = rowA["GripperAction"] - - return [dx, dy, dz, dox, doy, doz, grip_val] - -def read_frame_list_from_path( - video_path: Path | str -) -> list[cv2.typing.MatLike]: - """ - Given a path to an MP4 file, read all frames from the file into a list. - """ - cap = cv2.VideoCapture(str(video_path)) - frame_list = list() - while (ret := cap.read())[0]: # ret[0] is success flag, ret[1] is the frame - frame_list.append(ret[1]) - return frame_list - -def make_video_timestamps( - mp4_filepath: Path | str, - frame_list: list[cv2.typing.MatLike], -) -> list[int]: - """ - Given a path to an MP4 file and a list of frames from that file, - use ExifTool to read a timestamp of as many frames as possible from the - video metadata, then interpolate evenly to remaining frames. - """ - # Read timestamps from metadata. Result is given in Unix nanoseconds. - with exiftool.ExifToolHelper() as et: - mp4_metadata = et.get_metadata(str(mp4_filepath)) - timestamp_list: list[int] = [ - int(t) # From str - for t in mp4_metadata[0]["XMP:Timestamps"] - ] - - # Emit a warning if not all frames are retrieved from the metadata. - if len(timestamp_list) < len(frame_list): - logging.warning( - f"ExifTool found {len(timestamp_list)} timestamps " - f"for {len(frame_list)} frames of video at {mp4_filepath}. " - "Will append interpolated timestamps to match frame count." - ) - - # Interpolate timestamps. - fps = mp4_metadata[0]["QuickTime:VideoFrameRate"] - delta_t = int((1 / fps) * 1e9) # In nanoseconds - while len(timestamp_list) < len(frame_list): - timestamp_list.append(timestamp_list[-1] + delta_t) - - # Convert timestamps to Unix miliseconds. - timestamp_list = [ - int(t / 1e6) # In miliseconds - for t in timestamp_list - ] - - # Check max deviation of frame timestamp deltas from period. - timestamp_array = np.array(timestamp_list, dtype=np.float64) / 1e3 - diffs = np.diff(timestamp_array) - logging.debug( - "Max deviation of frame timestamp deltas from period " - f"set by {fps=} is {np.max(np.abs(diffs - (1/fps))):.6f}s" - ) - - return timestamp_list - -def csv_to_lerobot_trajectory( - csv_trajectory_filepath: Path, - mp4_filepath: Path, - maybe_lerobot_episode_index: int, - tolerance_s: float = 1e-5, -) -> tuple[pa.Table, np.ndarray]: - """ - Convert one CSV + MP4 into a single "episode_{:06d}.parquet" and - copy the MP4 to "episode_{:06d}.mp4" in observation.images.side subdir. - - Also save a video containing only those frames that were matched. - """ - - # Load CSV data. - csv_trajectory_df = pd.read_csv(csv_trajectory_filepath) - - # --- Preprocess the trajectory data. --- - # Drop non-synchronized rows entirely. "Synchronized" is a boolean column - # intended to show if ViperLink was connected to UR5e at timestamp. - sync_mask = csv_trajectory_df["IsSynchronized"] == 1 - csv_trajectory_df = csv_trajectory_df[sync_mask] - - # Append binary grip action to each row. - # In ViperLink, `Gripper::Action::NONE` has the effect of taking previous - # action (or `RELEASE` at trajectory start). We preserve this behavior. - current_grip_action = -1.0 # Initial action is -1.0 (RELEASE) - for idx, row in csv_trajectory_df.iterrows(): - if row["GripperAction"] == "Gripper::Action::RELEASE": - current_grip_action = -1.0 - elif row["GripperAction"] == "Gripper::Action::GRAB": - current_grip_action = 1.0 - csv_trajectory_df.at[idx, "GripperAction"] = current_grip_action - - # Append timestamp in Unix miliseconds to each CSV row. - csv_trajectory_df["miliseconds"] = ( - pd - .to_datetime(csv_trajectory_df["Timestamp"]) - .add(pd.Timedelta(hours=-1)) # UTC-1 - .apply(lambda x: x.timestamp() * 1e3) # Miliseconds - .astype(int) - ) - - # --- Preprocess the video. --- - # Read video timestamp. - mp4_frame_list = read_frame_list_from_path(mp4_filepath) - mp4_timestamp_list: list[int] = make_video_timestamps( - mp4_filepath=mp4_filepath, # Should be seconds - frame_list=mp4_frame_list - ) - - # --- Match trajectory steps to video frames. --- - row_ts_begin = csv_trajectory_df["miliseconds"].min() - row_ts_end = csv_trajectory_df["miliseconds"].max() - matched_rows_by_frame: list[Optional[pd.Series]] \ - = [None] * len(mp4_timestamp_list) - tolerance_unix_ms = tolerance_s * 1e3 - for frame_idx, frame_ts in enumerate(mp4_timestamp_list): - if ( - frame_ts < row_ts_begin - tolerance_unix_ms or - frame_ts > row_ts_end + tolerance_unix_ms - ): - continue # Unmatchable within tolerance - - # Find a match by binary search on the trajectory timestamps. - try: - matching_row = csv_trajectory_df.iloc[ - csv_trajectory_df["miliseconds"].searchsorted(frame_ts) - ] - except IndexError: # Likely means frame is outside trajectory, - continue # but within tolerance. Ignore it for now. - - # If the match is further than the tolerance, don't include it. - if ( - frame_ts < matching_row["miliseconds"] - tolerance_unix_ms or - frame_ts > matching_row["miliseconds"] + tolerance_unix_ms - ): - logging.warning( - f"Matching candidate timestep further " - f"({np.abs(frame_ts - matching_row['miliseconds']):.8f}ms) from frame than " - f"allowed by tolerance of {tolerance_unix_ms}ms. Won't match this frame." - ) - continue # Unmatchable within tolerance - - # Save the match. - matched_rows_by_frame[frame_idx] = matching_row - - # Verify that matched rows form a single, continuous trajectory. - switch_count = int(matched_rows_by_frame[0] != None) - for row_idx, row in enumerate(matched_rows_by_frame[:-1]): - next_row = matched_rows_by_frame[row_idx + 1] - if type(row) != type(next_row): - switch_count += 1 - if switch_count > 2: - raise ValueError( - "Trajectory data forms multiple trajectories when " - "matched against video." - ) - - # --- Build the trajectory. --- - trajectory = [ - (row_frame_pair_idx, row) - for row_frame_pair_idx, row - in enumerate(matched_rows_by_frame) - if row is not None # If frame[frame_idx] matches some row - ] - # This is the timestamp of the initial frame in the trajectory. - # It is (brittly) guaranteed to be >=0, since we match steps to - # frames by binsearch on step timestamps, and skip index errors. - # Since the timestep of the first frame is zeroed out by LeRobotDataset - # initializer, we take it as the zero-point relative to step timestamps, - # i.e. saved timestamp is `timestamp_isn_unix_ms - trajectory_ts_begin`. - trajectory_init_frame_idx = trajectory[0][0] - trajectory_ts_begin = mp4_timestamp_list[trajectory_init_frame_idx] - lerobot_trajectory: List[LeRobotTrajectoryStep] = [ - LeRobotTrajectoryStep( - timestamp=float(row["miliseconds"] - trajectory_ts_begin) / 1e3, - episode_index=maybe_lerobot_episode_index, # ^ seconds - next_done=next_row is None or next_row_idx == len(trajectory) - 1, # TODO: Not ideal - task_index=0, # TODO: Assumes (incorrectly) only single task in data - index=step_idx, - frame_index=step_idx, - action=compute_actions_from_rows(row, next_row) - ) - for step_idx, ((_, row), (next_row_idx, next_row)) - in enumerate(zip(trajectory[:-1], trajectory[1:])) - ] - matched_video = np.array([ - mp4_frame_list[frame_idx] - for frame_idx, _ in trajectory[:-1] - ], dtype=np.uint8) - - # Write trajectory out as a parquet. - pa_trajectory = pa.Table.from_pydict({ - "timestamp": [f.timestamp for f in lerobot_trajectory], - "episode_index": [f.episode_index for f in lerobot_trajectory], - "next.done": [f.next_done for f in lerobot_trajectory], - "task_index": [f.task_index for f in lerobot_trajectory], - "index": [f.index for f in lerobot_trajectory], - "frame_index": [f.frame_index for f in lerobot_trajectory], - "action": pa.array([f.action for f in lerobot_trajectory], type=pa.list_(pa.float32())) - }) - - return pa_trajectory, matched_video - -def save_single_trajectory( - trajectory: pa.Table, - out_dir: Path, - episode_index: int, -) -> None: - """ - Save a single trajectory to a parquet file at - out_dir / data / chunk-000 / f"episode_{episode_index:06d}.parquet" - """ - trajectory_outdir = out_dir \ - / "data" \ - / "chunk-000" \ - / f"episode_{episode_index:06d}.parquet" - pq.write_table(trajectory, trajectory_outdir) - logging.debug( - f"[Episode {episode_index}] Saved parquet => {trajectory_outdir}" - ) - -def save_single_video( - video: np.ndarray, - out_dir: Path, - episode_index: int, - fps: float, -) -> Path: - """ - Save a single video, given a numpy array of frames (dtype=uint8), - to the expected output location at - out_dir / "videos" / "chunk-000" / "observation.images.side" - / f"episode_{episode_index:06d}.mp4" - """ - video_outdir = out_dir \ - / "videos" \ - / "chunk-000" \ - / "observation.images.side" - video_outdir.mkdir(parents=True, exist_ok=True) - episode_mp4 = video_outdir / f"episode_{episode_index:06d}.mp4" - - if len(video) == 0: - logging.debug(f"[Episode {episode_index}] No frames to save for MP4 => {episode_mp4}") - return - - # Assume video shape is (num_frames, height, width, channels) - height, width, channels = video[0].shape - fourcc = cv2.VideoWriter_fourcc(*'mp4v') - writer = cv2.VideoWriter(str(episode_mp4), fourcc, fps, (width, height)) - - for frame in video: - writer.write(frame) - - writer.release() - logging.debug(f"[Episode {episode_index}] Saved MP4 => {episode_mp4}") - return episode_mp4 - -def build_meta_files(out_root: Path, total_episodes: int, episode_lengths: List[int]): - meta_dir = out_root / "meta" - meta_dir.mkdir(exist_ok=True) - - # Get total number of video frames - video_dir = out_root / "videos" / "chunk-000" / "observation.images.side" - video_frame_counts = [] - for episode_idx in range(total_episodes): - video_path = video_dir / f"episode_{episode_idx:06d}.mp4" - cap = cv2.VideoCapture(str(video_path)) - frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - video_frame_counts.append(frame_count) - cap.release() - total_video_frames = sum(video_frame_counts) - - # Read video metadata from first video file - first_video = next((out_root / "videos" / "chunk-000" / "observation.images.side").glob("*.mp4")) - cap = cv2.VideoCapture(str(first_video)) - - # Get basic video properties - video_fps = cap.get(cv2.CAP_PROP_FPS) - width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - # Read the first frame to get number of channels - ret, frame = cap.read() - channels = frame.shape[2] if ret else 3 - - # Get codec information - fourcc = int(cap.get(cv2.CAP_PROP_FOURCC)) - codec = "".join([chr((fourcc >> 8 * i) & 0xFF) for i in range(4)]) - - - cap.release() - - # info.json - info_data = { - "codebase_version": "v2.0", - "robot_type": "UR5e", - "total_episodes": total_episodes, - "total_frames": total_video_frames, - "total_tasks": 1, - "total_videos": total_episodes, - "total_chunks": 1, - "chunks_size": total_episodes, - "fps": video_fps, - "splits": {"train": f"0:{total_episodes}"}, - "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet", - "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4", - "features": { - "observation.images.side": { - "dtype": "video", - "shape": [height, width, channels], - "names": ["height", "width", "channel"], - "video_info": { - "video.fps": video_fps, - "video.codec": codec, - "pix_fmt": "yuv420p", # This is still hardcoded as it's not easily accessible via OpenCV - "has_audio": False # OpenCV doesn't expose audio info, but these are known to be video-only - } - }, - "action": { - "dtype": "float32", - "shape": [7], - "names": [ - "PositionX", "PositionY", "PositionZ", - "OrientationX", "OrientationY", "OrientationZ", - "GripperAction" - ] - } - } - } - with open(meta_dir / "info.json", "w") as f: - json.dump(info_data, f, indent=2) - - # stats.json - # Collect all actions across episodes into a list - all_actions = [] - for episode_idx in range(total_episodes): - episode_path = out_root \ - / "data" \ - / "chunk-000" \ - / f"episode_{episode_idx:06d}.parquet" - table = pq.read_table(episode_path) - actions = table["action"].to_numpy() - all_actions.extend(actions) - - # Convert to numpy array and compute quantiles along first axis - all_actions = np.array(all_actions) - q01 = np.quantile(all_actions, 0.01, axis=0).tolist() - q99 = np.quantile(all_actions, 0.99, axis=0).tolist() - - stats_data = { - "action": { - "q01": q01, - "q99": q99 - } - } - with open(meta_dir / "stats.json", "w") as f: - json.dump(stats_data, f, indent=2) - - # episodes.jsonl - with open(meta_dir / "episodes.jsonl", "w") as f: - for eidx, length in enumerate(episode_lengths): - row = { - "episode_index": eidx, - "tasks": ["Pick up the object"], - "length": length - } - f.write(json.dumps(row) + "\n") - - # tasks.jsonl - with open(meta_dir / "tasks.jsonl", "w") as f: - row = { - "task_index": 0, - "task": "Demonstration from raw robot data" - } - f.write(json.dumps(row) + "\n") - -def main( - raw_data_prefix: Path = None, - out_root: Path = None, - tolerance_s: float = 1e-5 -): - - # Where is your raw data? - if raw_data_prefix is None: - raw_data_prefix = Path(".") - raw_traj_dir = raw_data_prefix / "trajectories" - print(f"Looking for data in:\n {raw_traj_dir}") - - # Where do you want the new dataset to live? - if out_root is None: - raise ValueError("give a outfile location") - data_out_dir = out_root / "data" / "chunk-000" - data_out_dir.mkdir(parents=True, exist_ok=True) - - # Pair up CSV + MP4 - pairs: list[tuple[Path, Path]] = [] - for ep_idx in os.listdir(raw_traj_dir): - ep_files = os.listdir(raw_traj_dir / ep_idx) - csv_file = [ - ep_file - for ep_file in ep_files - if ep_file.endswith(".csv") - ][0] - mp4_file = [ - ep_file - for ep_file in ep_files - if ("side_view" in ep_file) and ep_file.endswith(".mp4") - ][0] - pairs.append((raw_traj_dir / ep_idx / csv_file, raw_traj_dir / ep_idx / mp4_file)) - - # Convert each episode - lerobot_episode_lengths = [] - lerobot_episode_index = 0 - for episode_index, (csv_f, mp4_f) in enumerate(pairs): - print("\n") - logging.debug(( - f"Processing episode {episode_index} " - f"with CSV: {csv_f.name} and MP4: {mp4_f.name}" - )) - lerobot_trajectory, matched_video = csv_to_lerobot_trajectory( - csv_trajectory_filepath=csv_f, - mp4_filepath=mp4_f, - maybe_lerobot_episode_index=lerobot_episode_index, - tolerance_s=tolerance_s, - ) - - # Check that the trajectory satisfies the tolerance. - timestamps = lerobot_trajectory["timestamp"].to_numpy() - diffs = np.diff(timestamps) - fps = cv2.VideoCapture(mp4_f).get(cv2.CAP_PROP_FPS) - within_tolerance = torch.tensor( - np.abs(diffs - 1/fps) <= tolerance_s - ) - if not torch.all(within_tolerance): - # Find indices where tolerance check failed - failed_indices = torch.where(~within_tolerance)[0] - failed_diffs = diffs[failed_indices] - expected_interval = 1/fps - - logging.debug( - f"Episode {episode_index} failed tolerance check and will not be included in the LeRobot dataset.\n" - f"Found {len(failed_indices)} timestamp intervals outside tolerance of {tolerance_s}s:\n" - f"- Expected interval between frames: {expected_interval:.6f}s\n" - f"- Number of failed indices: {failed_indices.shape[0]}\n" - f"- Maximum deviation from expected: {np.max(np.abs(failed_diffs - expected_interval)):.6f}s\n" - f"- Maximum allowed deviation: ±{tolerance_s:.6f}s" - ) - continue - - # Save the trajectory and video - save_single_trajectory( - lerobot_trajectory, - out_root, - lerobot_episode_index - ) - video_path = save_single_video( - matched_video, - out_root, - lerobot_episode_index, - fps=cv2.VideoCapture(mp4_f).get(cv2.CAP_PROP_FPS) - ) - - # Save the length of the trajectory. - lerobot_episode_length = len(lerobot_trajectory) - lerobot_episode_lengths.append(lerobot_episode_length) - lerobot_frame_count = cv2.VideoCapture(video_path).get(cv2.CAP_PROP_FRAME_COUNT) - logging.debug(( - f"Episode {episode_index} " - f"will be saved in LeRobot dataset as episode {lerobot_episode_index}.\n " - f"trajectory length: {lerobot_episode_length} " - f"number of frames: {int(lerobot_frame_count)}" - )) - - lerobot_episode_index += 1 - - lerobot_episodes_total = len(lerobot_episode_lengths) - - # Build meta files - build_meta_files( - out_root=out_root, - total_episodes=lerobot_episodes_total, - episode_lengths=lerobot_episode_lengths - ) - print("\nDone creating LeRobot-style dataset at:", out_root) - -if __name__ == "__main__": - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--raw_data_prefix", type=Path, default=None, - help="Path prefix to raw data directory") - parser.add_argument("--out_root", type=Path, default=None, - help="Output directory for the dataset") - parser.add_argument("--tolerance_s", type=float, default=1e-5, - help=("Maximum allowed deviation from expected " - "frame interval (in seconds)")) - args = parser.parse_args() - print("Running with args:", args) - - main( - raw_data_prefix=args.raw_data_prefix, - out_root=args.out_root, - tolerance_s=args.tolerance_s - ) \ No newline at end of file From b9307f63a14c601c468914ffba0d52c16bfbf8c2 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 13 Mar 2025 16:09:15 +0000 Subject: [PATCH 35/58] merged manual create env script to new fork --- .env.template | 3 + README.md | 9 ++- manual_create_env.sh | 85 +++++++++++++++++++++++ prismatic/vla/datasets/datasets.py | 105 +++++------------------------ vla-scripts/finetune.py | 4 +- vla-scripts/finetune.sub | 68 +++++++++++++++++++ 6 files changed, 182 insertions(+), 92 deletions(-) create mode 100644 .env.template create mode 100755 manual_create_env.sh create mode 100644 vla-scripts/finetune.sub diff --git a/.env.template b/.env.template new file mode 100644 index 000000000..30f2e5d00 --- /dev/null +++ b/.env.template @@ -0,0 +1,3 @@ +HF_TOKEN= +WANDB_API_KEY= +WANDB_PROJECT="ur5e" \ No newline at end of file diff --git a/README.md b/README.md index 4f980aead..335b4f053 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,14 @@ cp .env.template .env ``` and populate it. -#### Manually managed virtualenv +#### Manual -Otherwise, for manual installation, something like this might work (soon to be replaced by a `uv`-managed configuration) +Otherwise, there is a script, `manual_create_env.sh` which will make a Python +virtualenv for you. + +``` +./manual_create_env.sh +``` ```bash # Create and activate virtualenv diff --git a/manual_create_env.sh b/manual_create_env.sh new file mode 100755 index 000000000..1f276992d --- /dev/null +++ b/manual_create_env.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# +# Creates a Python 3.10 virtual environment, installs dependencies, +# and downloads the OpenVLA model. Intended for environments +# without Docker support where GPU is Slurm-managed (e.g. entropy). +# Designed to be used in conjunction with vla-scripts/finetune.sub. +# +# Usage: ./manual_create_env.sh +# +# Assumptions: +# - This script is run from the root directory of the openvla_finetuner project. +# - The user has a working Python 3.10 and `virtualenv` installation. +# - The user has internet access to download dependencies and the model. +# - The user has Slurm installed and configured. + +set -e +set -o pipefail + +function check_virtualenv { + if ! command -v virtualenv &> /dev/null; then + echo "Error: virtualenv is not installed" >&2 + exit 1 + fi +} + +function verify_directory { + if [[ ! -f "$(basename "$0")" ]]; then + echo "Error: This script must be run from the openvla/ directory" >&2 + exit 1 + fi +} + +function setup_virtualenv { + if [[ ! -d ".venv" ]]; then + virtualenv -p 3.10 .venv || exit 1 + fi + source .venv/bin/activate || exit 1 + pip install --upgrade pip || exit 1 + pip install "setuptools<60" || exit 1 # Or else build dlimp_openvla will fail. +} + +function install_dependencies { + # Note: We intentionally don't exit on pip check error below due to known + # dependency conflicts between OpenVLA (which needs torch==2.2.0) and LeRobot + # (which needs torch>=2.2.1). These conflicts are expected and the + # installation will still work for our purposes. + + # Install OpenVLA. + pip install -e . || exit 1 + + # Install LeRobot. + pushd third_party/lerobot || exit 1 + pip install -e . || exit 1 + popd || exit 1 + + # Reinstall OpenVLA dependencies that LeRobot may have overwritten. + set +e # Temporarily disable exit on error. + pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' | \ + xargs pip install + set -e # Re-enable exit on error. + + # Install Flash Attention. + pip install packaging ninja || exit 1 + # We will install flash-attn inside the Slurm job, after loading CUDA. + # This avoids errors when CUDA_HOME is not set during environment creation. + # pip install \ + # "flash-attn==2.5.5" \ + # --no-build-isolation || exit 1 +} + +function download_model { + pip install huggingface-hub || exit 1 + huggingface-cli download openvla/openvla-7b || exit 1 +} + +function main { + check_virtualenv + verify_directory + setup_virtualenv + install_dependencies + download_model +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" \ No newline at end of file diff --git a/prismatic/vla/datasets/datasets.py b/prismatic/vla/datasets/datasets.py index f1dc66e36..9166b5f7e 100644 --- a/prismatic/vla/datasets/datasets.py +++ b/prismatic/vla/datasets/datasets.py @@ -184,9 +184,8 @@ def __iter__(self) -> Dict[str, Any]: LeRobotDatasetMetadata, ) -# TODO: zrob dataset ktory by jednoczesnie byl LeRobotDatasetem i implementoeal to czego tam potrzebuje openvla -- te prompt buildery, itp. -class OpenVLALeRobotDataset(LeRobotDataset): +class RLDSLeRobotDataset(LeRobotDataset): def __init__( self, @@ -223,28 +222,20 @@ def __init__( self.image_transform = image_transform self.prompt_builder_fn = prompt_builder_fn - # Note =>> We expect the dataset to store statistics for action de-normalization. + # NOTE: We expect the dataset to store statistics for action de-normalization: + # 1/100st quantile of each action under "q01" and 99/100th quantile under "q99". self.dataset_statistics = { - "openvla_lerobot_dataset": { + "rlds_lerobot_dataset": { "action": { "q01": np.array(self.meta.stats["action"]["q01"]), "q99": np.array(self.meta.stats["action"]["q99"]), } } } - - # Retrieve the name of image observations within metadata. - metadata_feature_dict = self.meta.info['features'] - obs_image_keys = [ - k for k in metadata_feature_dict.keys() - if isinstance(metadata_feature_dict[k], dict) - and metadata_feature_dict[k].get("dtype") == "video" - ] - if len(obs_image_keys) == 0: - raise ValueError(f"Provided data contains no videos") - if len(obs_image_keys) > 1: - raise ValueError(f"Provided data contains >1 video per episode") - self.obs_image_key = obs_image_keys[0] + + # NOTE: This is hardcoded as a social contract. + # This is the only key to image observations that will be used. + self.obs_image_key = "observation.images.side" def __len__(self): @@ -269,9 +260,13 @@ def __getitem__(self, idx): instruction = self.meta.tasks[task_idx] # Retrieve action. - action: torch.Tensor = hf_item["action"] - q01 = np.array(self.dataset_statistics["openvla_lerobot_dataset"]["action"]["q01"]) - q99 = np.array(self.dataset_statistics["openvla_lerobot_dataset"]["action"]["q99"]) + action: torch.Tensor = torch.cat([ + hf_item["action.pose"], + hf_item["action.gripper"].unsqueeze(0) + ]) + + qs = self.dataset_statistics["rlds_lerobot_dataset"]["action"] + q01, q99 = np.array(qs["q01"]), np.array(qs["q99"]) action = (2*action - q01 - q99) / (q99 - q01) # normalize to [-1, 1] action: str = self.action_tokenizer(action) @@ -280,7 +275,7 @@ def __getitem__(self, idx): conversation = [ { "from": "human", - "value": f"What action should the robot take to {instruction}?" + "value": f"Hey I need the robot to do this: {instruction}. We upgraded from the old pincer gripper to this new suction cup end effector - it's that blue circular cup with the yellow ring at the end of the silver arm. Big difference is we can't tell if it's got a good seal just by looking at it (unlike before where we could see the gripper fingers close). Also the suction cup needs a flat surface to grip well, and we need enough vacuum pressure for different weights. Sometimes we need to wiggle it a bit to break the seal when releasing too. What's the best way to handle this with the new setup?" }, { "from": "gpt", @@ -307,70 +302,4 @@ def __getitem__(self, idx): labels[: -(len(action) + 1)] = IGNORE_INDEX return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels) - - - -class DummyDataset(Dataset): - def __init__( - self, - action_tokenizer: ActionTokenizer, - base_tokenizer: PreTrainedTokenizerBase, - image_transform: ImageTransform, - prompt_builder_fn: Type[PromptBuilder], - ) -> None: - self.action_tokenizer = action_tokenizer - self.base_tokenizer = base_tokenizer - self.image_transform = image_transform - self.prompt_builder_fn = prompt_builder_fn - - # Note =>> We expect the dataset to store statistics for action de-normalization. Specifically, we store the - # per-dimension 1st and 99th action quantile. The values below correspond to "no normalization" for simplicity. - self.dataset_statistics = { - "dummy_dataset": { - "action": {"q01": np.zeros((7,), dtype=np.float32), "q99": np.ones((7,), dtype=np.float32)} - } - } - - def __len__(self): - # TODO =>> Replace with number of elements in your dataset! - return 10000 - - def __getitem__(self, idx): - """Get a single training example.""" - # Generate random image, action and instruction - image = Image.fromarray( - np.asarray(np.random.rand(224, 224, 3) * 255.0, dtype=np.uint8) - ) - action = np.asarray(np.random.rand(7), dtype=np.float32) - instruction = "do something spectacular" - - # Build conversation prompt - prompt_builder = self.prompt_builder_fn("openvla") - conversation = [ - { - "from": "human", - "value": f"What action should the robot take to {instruction}?" - }, - { - "from": "gpt", - "value": self.action_tokenizer(action) - } - ] - - # Add conversation turns to prompt builder - for turn in conversation: - prompt_builder.add_turn(turn["from"], turn["value"]) - - # Tokenize (w/ `base_tokenizer`) - input_ids = self.base_tokenizer(prompt_builder.get_prompt(), add_special_tokens=True).input_ids - labels = list(input_ids) - - # Tensorize =>> Run Image Transform to get `pixel_values` =>> Return - # =>> IMPORTANT :: IF WE'RE USING HF .forward(..., labels=labels), SHIFTING HAPPENS _INSIDE_ MODEL! - input_ids, labels = torch.tensor(input_ids), torch.tensor(labels) - pixel_values = self.image_transform(image) - - # [CRITICAL] We do not want to take the loss for anything but the predicted action tokens! - labels[: -(len(action) + 1)] = IGNORE_INDEX - - return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels) \ No newline at end of file + \ No newline at end of file diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 7e16110e6..8746dd7bd 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -208,9 +208,9 @@ def finetune(cfg: FinetuneConfig) -> None: # your own Dataset, make sure to add the appropriate logic to the training loop! # # TODO: Figure this out # --- - from prismatic.vla.datasets.datasets import OpenVLALeRobotDataset + from prismatic.vla.datasets.datasets import RLDSLeRobotDataset - vla_dataset = OpenVLALeRobotDataset( + vla_dataset = RLDSLeRobotDataset( repo_id="NotRequired", action_tokenizer=action_tokenizer, base_tokenizer=processor.tokenizer, diff --git a/vla-scripts/finetune.sub b/vla-scripts/finetune.sub new file mode 100644 index 000000000..47416d6af --- /dev/null +++ b/vla-scripts/finetune.sub @@ -0,0 +1,68 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -n 1 +#SBATCH -c 8 +#SBATCH --gres=gpu:1 +#SBATCH -t 6:00:00 +#SBATCH -p a100 +#SBATCH --mem=50G +#SBATCH -o .slurmlog/slurm-%j.out +#SBATCH -e .slurmlog/slurm-%j.err + +# --- Environment Setup --- + +module purge # Start with a clean environment. +module load cuda/11.8 # REPLACE with the CORRECT CUDA version! + +# Activate the virtual environment (created by create_env.sh). +source .venv/bin/activate + +# Install flash-attn *after* loading CUDA. +pip install "flash-attn==2.5.5" --no-build-isolation + +# Set environment variables for Weights & Biases logging +if [[ -z "${WANDB_API_KEY}" ]]; then + echo "Warning: WANDB_API_KEY not set in environment, reading from .env file" + if [ -f .env ]; then + export WANDB_API_KEY=$(grep WANDB_API_KEY .env | cut -d '=' -f2) + else + echo "Error: .env file not found" + exit 1 + fi +fi + +if [[ -z "${WANDB_MODE}" ]]; then + export WANDB_MODE="online" +fi + +# --- Run the Finetuning Script --- + +# Set PYTHONPATH and run the finetuning script using torchrun. +# Note: We use the absolute path to finetune.py to avoid issues with +# relative paths within the Slurm job. +PYTHONPATH="$PYTHONPATH:$(pwd)/lerobot" \ +torchrun \ + --standalone \ + --nnodes 1 \ + --nproc-per-node 1 \ + "$(pwd)"/vla-scripts/finetune.py \ + --vla_path "openvla/openvla-7b" \ + --data_root_dir "data" \ + --dataset_name "nomagic-simple-box" \ + --run_root_dir ".runs/" \ + --adapter_tmp_dir ".adapter/" \ + --lora_rank 32 \ + --batch_size 16 \ + --grad_accumulation_steps 1 \ + --learning_rate 5e-4 \ + --image_aug True \ + --max_steps 25000 \ + --save_steps 1000 \ + --save_latest_checkpoint_only False \ + --tolerance_s 0.01 \ + --wandb_entity robotgeneralist \ + --wandb_project openvla + +# To run this script, edit the options above, and then +# execute the following command from the root repository directory: +# sbatch vla-scripts/finetune.sub \ No newline at end of file From 9c34d079d1269d68006073ae67317263c0f57a6c Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 13 Mar 2025 17:56:08 +0000 Subject: [PATCH 36/58] debugging finetuning --- finetune_lerobot.sh | 6 +-- prismatic/vla/datasets/datasets.py | 68 ++++++++++++++++++++++++++++++ vla-scripts/finetune.py | 2 +- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/finetune_lerobot.sh b/finetune_lerobot.sh index 91aafa1bf..e5c082b8a 100755 --- a/finetune_lerobot.sh +++ b/finetune_lerobot.sh @@ -2,12 +2,12 @@ PYTHONPATH="$PYTHONPATH:$(pwd)/lerobot" && \ torchrun --standalone --nnodes 1 --nproc-per-node 1 vla-scripts/finetune.py \ --vla_path "openvla/openvla-7b" \ - --data_root_dir "data/simple_task_lerobot" \ - --dataset_name "main" \ + --data_root_dir "data/robotgeneralist" \ + --dataset_name "nomagic-simple-box" \ --run_root_dir ".runs/" \ --adapter_tmp_dir ".adapter/" \ --lora_rank 32 \ - --batch_size 2 \ + --batch_size 1 \ --grad_accumulation_steps 8 \ --learning_rate 5e-4 \ --image_aug True \ diff --git a/prismatic/vla/datasets/datasets.py b/prismatic/vla/datasets/datasets.py index 9166b5f7e..8d25bfdf8 100644 --- a/prismatic/vla/datasets/datasets.py +++ b/prismatic/vla/datasets/datasets.py @@ -177,6 +177,73 @@ def __iter__(self) -> Dict[str, Any]: yield out ### + +class DummyDataset(Dataset): + def __init__( + self, + action_tokenizer: ActionTokenizer, + base_tokenizer: PreTrainedTokenizerBase, + image_transform: ImageTransform, + prompt_builder_fn: Type[PromptBuilder], + ) -> None: + self.action_tokenizer = action_tokenizer + self.base_tokenizer = base_tokenizer + self.image_transform = image_transform + self.prompt_builder_fn = prompt_builder_fn + + # Note =>> We expect the dataset to store statistics for action de-normalization. Specifically, we store the + # per-dimension 1st and 99th action quantile. The values below correspond to "no normalization" for simplicity. + self.dataset_statistics = { + "dummy_dataset": { + "action": {"q01": np.zeros((7,), dtype=np.float32), "q99": np.ones((7,), dtype=np.float32)} + } + } + + def __len__(self): + # TODO =>> Replace with number of elements in your dataset! + return 10000 + + def __getitem__(self, idx): + """Get a single training example.""" + # Generate random image, action and instruction + image = Image.fromarray( + np.asarray(np.random.rand(224, 224, 3) * 255.0, dtype=np.uint8) + ) + action = np.asarray(np.random.rand(7), dtype=np.float32) + instruction = "do something spectacular" + + # Build conversation prompt + prompt_builder = self.prompt_builder_fn("openvla") + conversation = [ + { + "from": "human", + "value": f"What action should the robot take to {instruction}?" + }, + { + "from": "gpt", + "value": self.action_tokenizer(action) + } + ] + + # Add conversation turns to prompt builder + for turn in conversation: + prompt_builder.add_turn(turn["from"], turn["value"]) + + # Tokenize (w/ `base_tokenizer`) + input_ids = self.base_tokenizer(prompt_builder.get_prompt(), add_special_tokens=True).input_ids + labels = list(input_ids) + + # Tensorize =>> Run Image Transform to get `pixel_values` =>> Return + # =>> IMPORTANT :: IF WE'RE USING HF .forward(..., labels=labels), SHIFTING HAPPENS _INSIDE_ MODEL! + input_ids, labels = torch.tensor(input_ids), torch.tensor(labels) + pixel_values = self.image_transform(image) + + # [CRITICAL] We do not want to take the loss for anything but the predicted action tokens! + labels[: -(len(action) + 1)] = IGNORE_INDEX + + return dict(pixel_values=pixel_values, input_ids=input_ids, labels=labels) + + from typing import Callable from lerobot.common.datasets.lerobot_dataset import ( @@ -224,6 +291,7 @@ def __init__( # NOTE: We expect the dataset to store statistics for action de-normalization: # 1/100st quantile of each action under "q01" and 99/100th quantile under "q99". + print(self.meta.stats) self.dataset_statistics = { "rlds_lerobot_dataset": { "action": { diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 8746dd7bd..051122fb5 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -224,7 +224,7 @@ def finetune(cfg: FinetuneConfig) -> None: tolerance_s=cfg.tolerance_s, image_transforms=None, download_videos=False, - local_files_only=True, + # local_files_only=True, ) From 55f2713f9263349d3b85a0d60816d560cec90059 Mon Sep 17 00:00:00 2001 From: Maciej Mehl Date: Tue, 18 Mar 2025 18:19:07 +0100 Subject: [PATCH 37/58] add ur5e constants --- prismatic/vla/constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/prismatic/vla/constants.py b/prismatic/vla/constants.py index 229ae944a..73b174144 100644 --- a/prismatic/vla/constants.py +++ b/prismatic/vla/constants.py @@ -44,6 +44,12 @@ class NormalizationType(str, Enum): "ACTION_PROPRIO_NORMALIZATION_TYPE": NormalizationType.BOUNDS_Q99, } +UR5E_CONSTANTS = { + "NUM_ACTIONS_CHUNK": 8, + "ACTION_DIM": 7, + "PROPRIO_DIM": 0, + "ACTION_PROPRIO_NORMALIZATION_TYPE": NormalizationType.BOUNDS_Q99, +} # Function to detect robot platform from command line arguments def detect_robot_platform(): @@ -55,6 +61,8 @@ def detect_robot_platform(): return "ALOHA" elif "bridge" in cmd_args: return "BRIDGE" + elif "ur5e" in cmd_args: + return "UR5E" else: # Default to LIBERO if unclear return "LIBERO" @@ -70,6 +78,8 @@ def detect_robot_platform(): constants = ALOHA_CONSTANTS elif ROBOT_PLATFORM == "BRIDGE": constants = BRIDGE_CONSTANTS +elif ROBOT_PLATFORM == "UR5E": + constants = UR5E_CONSTANTS # Assign constants to global variables NUM_ACTIONS_CHUNK = constants["NUM_ACTIONS_CHUNK"] From 7d1b76ce14cf5394dfe62dc5d968e5d64dd297e4 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Tue, 18 Mar 2025 17:57:26 +0000 Subject: [PATCH 38/58] properly stack wrist imgs onto side img --- prismatic/util/data_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/prismatic/util/data_utils.py b/prismatic/util/data_utils.py index 046ded92d..7faca4afa 100644 --- a/prismatic/util/data_utils.py +++ b/prismatic/util/data_utils.py @@ -305,7 +305,21 @@ def __call__(self, instances: Sequence[Dict[str, torch.Tensor]]) -> Dict[str, to attention_mask = input_ids.ne(self.pad_token_id) # 8. Collate images and actions + # Stack main images pixel_values = torch.stack([item["pixel_values"] for item in processed_items]) + + # If wrist images are available, combine them with the main images + if self.use_wrist_image and all("pixel_values_wrist" in item for item in processed_items): + pixel_values_wrist = torch.stack([item["pixel_values_wrist"] for item in processed_items]) + + # Reshape wrist images if needed (from [B, num_wrist, C, H, W] to [B, num_wrist*C, H, W]) + if pixel_values_wrist.dim() == 5: # [B, num_wrist, C, H, W] + B, num_wrist, C, H, W = pixel_values_wrist.shape + pixel_values_wrist = pixel_values_wrist.view(B, num_wrist * C, H, W) + + # Concatenate main and wrist images along the channel dimension + pixel_values = torch.cat([pixel_values, pixel_values_wrist], dim=1) + actions = torch.stack([item["actions"] for item in processed_items]) # 9. Build final batch From 1f3b5fb0cb50818d95ecf20720b6fa64c098d410 Mon Sep 17 00:00:00 2001 From: Maciej Mehl Date: Tue, 18 Mar 2025 19:19:53 +0100 Subject: [PATCH 39/58] log to wandb using total step idx --- prismatic/models/action_heads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prismatic/models/action_heads.py b/prismatic/models/action_heads.py index b3043c078..c892f5a2c 100644 --- a/prismatic/models/action_heads.py +++ b/prismatic/models/action_heads.py @@ -102,7 +102,7 @@ def predict_action(self, actions_hidden_states): # - shape: (batch_size, chunk_len, action_dim) batch_size = actions_hidden_states.shape[0] device = actions_hidden_states.device - rearranged_actions_hidden_states = actions_hidden_states.reshape(batch_size, NUM_ACTIONS_CHUNK, -1) + rearranged_actions_hidden_states = actions_hidden_states.reshape(batch_size, -1) action = self.model(rearranged_actions_hidden_states) return action From b6d5a3b002ca247f39fc28b931892b33bd555ebe Mon Sep 17 00:00:00 2001 From: mehhl Date: Tue, 18 Mar 2025 19:21:50 +0100 Subject: [PATCH 40/58] Revert "log to wandb using total step idx" This reverts commit 1f3b5fb0cb50818d95ecf20720b6fa64c098d410. --- prismatic/models/action_heads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prismatic/models/action_heads.py b/prismatic/models/action_heads.py index c892f5a2c..b3043c078 100644 --- a/prismatic/models/action_heads.py +++ b/prismatic/models/action_heads.py @@ -102,7 +102,7 @@ def predict_action(self, actions_hidden_states): # - shape: (batch_size, chunk_len, action_dim) batch_size = actions_hidden_states.shape[0] device = actions_hidden_states.device - rearranged_actions_hidden_states = actions_hidden_states.reshape(batch_size, -1) + rearranged_actions_hidden_states = actions_hidden_states.reshape(batch_size, NUM_ACTIONS_CHUNK, -1) action = self.model(rearranged_actions_hidden_states) return action From fa91eb381cfa6f297af72a8173d0ca44648e520c Mon Sep 17 00:00:00 2001 From: mehhl Date: Tue, 18 Mar 2025 19:24:41 +0100 Subject: [PATCH 41/58] actually log to wandb using total step idx --- vla-scripts/finetune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 25bb29c47..a01807084 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -1170,7 +1170,7 @@ def finetune(cfg: FinetuneConfig) -> None: smoothened_metrics = compute_smoothened_metrics(recent_metrics) # Push Metrics to W&B (every wandb_log_freq gradient steps) - log_step = epoch_gradient_step_idx if not cfg.resume else cfg.resume_step + epoch_gradient_step_idx + log_step = total_gradient_step_idx if not cfg.resume else cfg.resume_step + total_gradient_step_idx if distributed_state.is_main_process and log_step % cfg.wandb_log_freq == 0: log_metrics_to_wandb(smoothened_metrics, "VLA Train", log_step, wandb) From 19c4960eb8107d6a9f5962aa2a147d75da0144fd Mon Sep 17 00:00:00 2001 From: mehhl Date: Wed, 19 Mar 2025 12:35:41 +0100 Subject: [PATCH 42/58] more explicit wrapping of image transforms --- prismatic/util/data_utils.py | 15 +++++++++------ vla-scripts/finetune.py | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/prismatic/util/data_utils.py b/prismatic/util/data_utils.py index 7faca4afa..65e39c200 100644 --- a/prismatic/util/data_utils.py +++ b/prismatic/util/data_utils.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import ( + Any, Callable, Dict, Sequence, @@ -40,26 +41,28 @@ def tree_map_with_key(fn: Callable, tree: dict, keys: Sequence = ()) -> dict: } -def create_tensor_compatible_image_transform(transform_fn): +def greyscale_float_tensor_preprocessing_wrapper( + transform_fn: Callable[[Image.Image | np.ndarray], Any] +) -> Callable[[torch.Tensor | Image.Image | np.ndarray], Any]: """ - Wraps a transform function that expects PIL Images to work with tensors coming from LeRobotDataset. + Wraps a transform function that expects PIL Images to work with greyscale float tensors. Args: - transform_fn: A function that takes a PIL Image and transforms it + transform_fn: A function that takes a PIL Image / numpy arrayand transforms it Returns: - A function that can handle tensors from LeRobotDataset + A function that can handle greyscale float tensors, PIL Images, and numpy arrays """ def wrapper(tensor_image): if isinstance(tensor_image, torch.Tensor): # Convert tensor to PIL Image - # The tensor is typically [C, H, W] with values in [0, 1] + # The tensor is expected to be [C, H, W] with values in [0, 1] img_array = (tensor_image.permute(1, 2, 0).numpy() * 255).astype(np.uint8) pil_image = Image.fromarray(img_array) # Apply the transform return transform_fn(pil_image) else: - # If it's already a PIL Image or numpy array, apply transform directly + # If it's something else, let the transform function handle it return transform_fn(tensor_image) return wrapper diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index a01807084..ab57c0d32 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -73,7 +73,7 @@ ) from prismatic.vla.datasets.rlds.utils.data_utils import save_dataset_statistics from prismatic.util.data_utils import ( - create_tensor_compatible_image_transform, + greyscale_float_tensor_preprocessing_wrapper, VLACollatorForLeRobotDataset, ) @@ -988,7 +988,7 @@ def finetune(cfg: FinetuneConfig) -> None: repo_dir = Path(cfg.lerobot_dataset_root_dir) / cfg.lerobot_dataset_name # Wrap the image transform function to handle tensors - wrapped_transform = create_tensor_compatible_image_transform( + wrapped_transform = greyscale_float_tensor_preprocessing_wrapper( processor.image_processor.apply_transform ) From 3e4bd3476b83d765c2d436cdcfa18e38086cf4c3 Mon Sep 17 00:00:00 2001 From: mehhl Date: Wed, 19 Mar 2025 13:31:29 +0100 Subject: [PATCH 43/58] separate utils location for lerobotdataset --- prismatic/util/data_utils.py | 160 +------------------- prismatic/util/extern/__init__.py | 0 prismatic/util/extern/hf/__init__.py | 0 prismatic/util/extern/hf/lerobot_utils.py | 176 ++++++++++++++++++++++ 4 files changed, 177 insertions(+), 159 deletions(-) create mode 100644 prismatic/util/extern/__init__.py create mode 100644 prismatic/util/extern/hf/__init__.py create mode 100644 prismatic/util/extern/hf/lerobot_utils.py diff --git a/prismatic/util/data_utils.py b/prismatic/util/data_utils.py index 65e39c200..83fd82f19 100644 --- a/prismatic/util/data_utils.py +++ b/prismatic/util/data_utils.py @@ -10,19 +10,13 @@ Callable, Dict, Sequence, - Optional, Tuple, - Type, ) import numpy as np import torch from PIL import Image from torch.nn.utils.rnn import pad_sequence -from transformers import PreTrainedTokenizerBase - -from prismatic.vla.action_tokenizer import ActionTokenizer -from prismatic.models.backbones.llm.prompting import PurePromptBuilder # HuggingFace Default / LLaMa-2 IGNORE_INDEX (for labels) @@ -194,156 +188,4 @@ def __call__(self, instances: Sequence[Dict[str, torch.Tensor]]) -> Dict[str, to ) if dataset_names is not None: output["dataset_names"] = dataset_names - return output - -@dataclass -class VLACollatorForLeRobotDataset: - """ - Collator for LeRobotDataset instances specifically for VLA training. - - This collator handles: - 1. Action tokenization - 2. Prompt construction - 3. Input/label tokenization and masking - 4. Proper batching with padding - """ - action_tokenizer: ActionTokenizer - base_tokenizer: PreTrainedTokenizerBase - prompt_builder_fn: Type[PurePromptBuilder] - pad_token_id: int - model_max_length: int - predict_stop_token: bool = True - use_wrist_image: bool = False - use_proprio: bool = False - action_norm_stats: Optional[Dict[str, np.ndarray]] = None - - def __call__(self, instances: Sequence[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: - """Process and collate a batch of instances from LeRobotDataset.""" - batch_size = len(instances) - - # 1. Extract data from instances - processed_items = [] - for item in instances: - # Extract task/instruction - task = item.get("task", "") - - # Extract and normalize actions - action = torch.cat([ - item.get("action.pose", torch.zeros(6)), - item.get("action.gripper", torch.zeros(1)).unsqueeze(0) - ]) - - # Normalize actions if stats are provided - if self.action_norm_stats is not None: - q01, q99 = self.action_norm_stats.get("q01"), self.action_norm_stats.get("q99") - if q01 is not None and q99 is not None: - action = (2*action - torch.tensor(q01) - torch.tensor(q99)) / (torch.tensor(q99) - torch.tensor(q01)) - - # Tokenize action - action_tokens = self.action_tokenizer(action) - - # 2. Build prompt - prompt_builder = self.prompt_builder_fn("openvla") - conversation = [ - {"from": "human", "value": f"What action should the robot take to {task}?"}, - {"from": "gpt", "value": action_tokens}, - ] - for turn in conversation: - prompt_builder.add_turn(turn["from"], turn["value"]) - - # 3. Tokenize - tokenized = self.base_tokenizer( - prompt_builder.get_prompt(), - add_special_tokens=True, - return_tensors="pt" - ) - input_ids = tokenized.input_ids.squeeze(0) - - # 4. Create labels (copy input_ids) - labels = input_ids.clone() - - # 5. Mask labels (only keep action tokens for loss) - action_tokens_len = len(action_tokens) - labels[:-action_tokens_len-1] = IGNORE_INDEX - if not self.predict_stop_token: - labels[-1] = IGNORE_INDEX - - # 6. Add to processed items - processed_item = { - "input_ids": input_ids, - "labels": labels, - "pixel_values": item.get("pixel_values") if "pixel_values" in item else item.get(next(k for k in item if "image" in k.lower())), - "actions": action - } - - # Add dataset name if available - if "dataset_name" in item: - processed_item["dataset_name"] = item["dataset_name"] - - # Add wrist camera if used - if self.use_wrist_image and any("wrist" in k.lower() for k in item): - wrist_keys = [k for k in item if "wrist" in k.lower()] - if wrist_keys: - processed_item["pixel_values_wrist"] = torch.stack([item[k] for k in wrist_keys]) - - # Add proprioceptive data if used - if self.use_proprio and "proprio" in item: - processed_item["proprio"] = item["proprio"] - - processed_items.append(processed_item) - - # 7. Collate inputs with padding - input_ids = [item["input_ids"] for item in processed_items] - labels = [item["labels"] for item in processed_items] - - # Pad sequences - input_ids = pad_sequence(input_ids, batch_first=True, padding_value=self.pad_token_id) - labels = pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX) - - # Truncate if necessary - input_ids = input_ids[:, :self.model_max_length] - labels = labels[:, :self.model_max_length] - - # Create attention mask based on padding - attention_mask = input_ids.ne(self.pad_token_id) - - # 8. Collate images and actions - # Stack main images - pixel_values = torch.stack([item["pixel_values"] for item in processed_items]) - - # If wrist images are available, combine them with the main images - if self.use_wrist_image and all("pixel_values_wrist" in item for item in processed_items): - pixel_values_wrist = torch.stack([item["pixel_values_wrist"] for item in processed_items]) - - # Reshape wrist images if needed (from [B, num_wrist, C, H, W] to [B, num_wrist*C, H, W]) - if pixel_values_wrist.dim() == 5: # [B, num_wrist, C, H, W] - B, num_wrist, C, H, W = pixel_values_wrist.shape - pixel_values_wrist = pixel_values_wrist.view(B, num_wrist * C, H, W) - - # Concatenate main and wrist images along the channel dimension - pixel_values = torch.cat([pixel_values, pixel_values_wrist], dim=1) - - actions = torch.stack([item["actions"] for item in processed_items]) - - # 9. Build final batch - batch = { - "pixel_values": pixel_values, - "input_ids": input_ids, - "attention_mask": attention_mask, - "labels": labels, - "actions": actions, - } - - # Add wrist images if available - if self.use_wrist_image and all("pixel_values_wrist" in item for item in processed_items): - batch["pixel_values_wrist"] = torch.cat([item["pixel_values_wrist"] for item in processed_items], dim=1) - - # Add proprioceptive data if available - if self.use_proprio and all("proprio" in item for item in processed_items): - batch["proprio"] = torch.stack([item["proprio"] for item in processed_items]) - - # Add dataset names if available - if all("dataset_name" in item for item in processed_items): - batch["dataset_names"] = [item["dataset_name"] for item in processed_items] - - return batch \ No newline at end of file + return output \ No newline at end of file diff --git a/prismatic/util/extern/__init__.py b/prismatic/util/extern/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/prismatic/util/extern/hf/__init__.py b/prismatic/util/extern/hf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py new file mode 100644 index 000000000..f7e9fc069 --- /dev/null +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -0,0 +1,176 @@ +""" +lerobot_utils.py + +Utilities for working with LeRobotDataset v2.0. +""" + +from dataclasses import dataclass +from typing import ( + Optional, + Sequence, + Type, +) + +import numpy as np +import torch +from torch.nn.utils.rnn import pad_sequence +from transformers import PreTrainedTokenizerBase + +from prismatic.vla.constants import ( + IGNORE_INDEX, +) +from prismatic.vla.action_tokenizer import ActionTokenizer +from prismatic.models.backbones.llm.prompting import PurePromptBuilder + + +@dataclass +class VLACollatorForLeRobotDataset: + """ + Collator for LeRobotDataset instances specifically for VLA training. + + This collator handles: + 1. Action tokenization + 2. Prompt construction + 3. Input/label tokenization and masking + 4. Proper batching with padding + """ + action_tokenizer: ActionTokenizer + base_tokenizer: PreTrainedTokenizerBase + prompt_builder_fn: Type[PurePromptBuilder] + pad_token_id: int + model_max_length: int + predict_stop_token: bool = True + use_wrist_image: bool = False + use_proprio: bool = False + action_norm_stats: Optional[dict[str, np.ndarray]] = None + + def __call__(self, instances: Sequence[dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]: + """Process and collate a batch of instances from LeRobotDataset.""" + batch_size = len(instances) + + # 1. Extract data from instances + processed_items = [] + for item in instances: + # Extract task/instruction + task = item.get("task", "") + + # Extract and normalize actions + action = torch.cat([ + item.get("action.pose", torch.zeros(6)), + item.get("action.gripper", torch.zeros(1)).unsqueeze(0) + ]) + + # Normalize actions if stats are provided + if self.action_norm_stats is not None: + q01, q99 = self.action_norm_stats.get("q01"), self.action_norm_stats.get("q99") + if q01 is not None and q99 is not None: + action = (2*action - torch.tensor(q01) - torch.tensor(q99)) / (torch.tensor(q99) - torch.tensor(q01)) + + # Tokenize action + action_tokens = self.action_tokenizer(action) + + # 2. Build prompt + prompt_builder = self.prompt_builder_fn("openvla") + conversation = [ + {"from": "human", "value": f"What action should the robot take to {task}?"}, + {"from": "gpt", "value": action_tokens}, + ] + for turn in conversation: + prompt_builder.add_turn(turn["from"], turn["value"]) + + # 3. Tokenize + tokenized = self.base_tokenizer( + prompt_builder.get_prompt(), + add_special_tokens=True, + return_tensors="pt" + ) + input_ids = tokenized.input_ids.squeeze(0) + + # 4. Create labels (copy input_ids) + labels = input_ids.clone() + + # 5. Mask labels (only keep action tokens for loss) + action_tokens_len = len(action_tokens) + labels[:-action_tokens_len-1] = IGNORE_INDEX + if not self.predict_stop_token: + labels[-1] = IGNORE_INDEX + + # 6. Add to processed items + processed_item = { + "input_ids": input_ids, + "labels": labels, + "pixel_values": item.get("pixel_values") if "pixel_values" in item else item.get(next(k for k in item if "image" in k.lower())), + "actions": action + } + + # Add dataset name if available + if "dataset_name" in item: + processed_item["dataset_name"] = item["dataset_name"] + + # Add wrist camera if used + if self.use_wrist_image and any("wrist" in k.lower() for k in item): + wrist_keys = [k for k in item if "wrist" in k.lower()] + if wrist_keys: + processed_item["pixel_values_wrist"] = torch.stack([item[k] for k in wrist_keys]) + + # Add proprioceptive data if used + if self.use_proprio and "proprio" in item: + processed_item["proprio"] = item["proprio"] + + processed_items.append(processed_item) + + # 7. Collate inputs with padding + input_ids = [item["input_ids"] for item in processed_items] + labels = [item["labels"] for item in processed_items] + + # Pad sequences + input_ids = pad_sequence(input_ids, batch_first=True, padding_value=self.pad_token_id) + labels = pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX) + + # Truncate if necessary + input_ids = input_ids[:, :self.model_max_length] + labels = labels[:, :self.model_max_length] + + # Create attention mask based on padding + attention_mask = input_ids.ne(self.pad_token_id) + + # 8. Collate images and actions + # Stack main images + pixel_values = torch.stack([item["pixel_values"] for item in processed_items]) + + # If wrist images are available, combine them with the main images + if self.use_wrist_image and all("pixel_values_wrist" in item for item in processed_items): + pixel_values_wrist = torch.stack([item["pixel_values_wrist"] for item in processed_items]) + + # Reshape wrist images if needed (from [B, num_wrist, C, H, W] to [B, num_wrist*C, H, W]) + if pixel_values_wrist.dim() == 5: # [B, num_wrist, C, H, W] + B, num_wrist, C, H, W = pixel_values_wrist.shape + pixel_values_wrist = pixel_values_wrist.view(B, num_wrist * C, H, W) + + # Concatenate main and wrist images along the channel dimension + pixel_values = torch.cat([pixel_values, pixel_values_wrist], dim=1) + + actions = torch.stack([item["actions"] for item in processed_items]) + + # 9. Build final batch + batch = { + "pixel_values": pixel_values, + "input_ids": input_ids, + "attention_mask": attention_mask, + "labels": labels, + "actions": actions, + } + + # Add wrist images if available + if self.use_wrist_image and all("pixel_values_wrist" in item for item in processed_items): + batch["pixel_values_wrist"] = torch.cat([item["pixel_values_wrist"] for item in processed_items], dim=1) + + # Add proprioceptive data if available + if self.use_proprio and all("proprio" in item for item in processed_items): + batch["proprio"] = torch.stack([item["proprio"] for item in processed_items]) + + # Add dataset names if available + if all("dataset_name" in item for item in processed_items): + batch["dataset_names"] = [item["dataset_name"] for item in processed_items] + + return batch \ No newline at end of file From b3b256db5a657b6aa948419f5a871e6f9b4b9bda Mon Sep 17 00:00:00 2001 From: mehhl Date: Wed, 19 Mar 2025 13:33:33 +0100 Subject: [PATCH 44/58] utils to get dataset stats from lerobotd --- prismatic/util/extern/hf/lerobot_utils.py | 69 +++++++++++++++++++++++ vla-scripts/finetune.py | 38 ++++--------- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index f7e9fc069..9ccee119b 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -16,13 +16,82 @@ from torch.nn.utils.rnn import pad_sequence from transformers import PreTrainedTokenizerBase +from lerobot.common.datasets.lerobot_dataset import LeRobotDataset + from prismatic.vla.constants import ( + ACTION_DIM, + ACTION_PROPRIO_NORMALIZATION_TYPE, IGNORE_INDEX, ) from prismatic.vla.action_tokenizer import ActionTokenizer from prismatic.models.backbones.llm.prompting import PurePromptBuilder +def create_action_norm_stats_dict_from_lerobot_dataset( + dataset: LeRobotDataset, +) -> dict[str, dict[str, list[float]]]: + """ + Get statistics for unnormalizing actions from a v2.0 LeRobotDataset. + """ + if ACTION_PROPRIO_NORMALIZATION_TYPE != "bounds_q99": + raise NotImplementedError( + "For now, only q01/q99 normalization is supported " + "for OpenVLA-OFT with LeRobotDataset v2.0" + ) + assert ( + "action" in dataset.meta.stats + and "q01" in dataset.meta.stats["action"] + and "q99" in dataset.meta.stats["action"] + and len(dataset.meta.stats["action"]["q01"]) == ACTION_DIM + and len(dataset.meta.stats["action"]["q99"]) == ACTION_DIM + ), "Dataset must have q01 and q99 stored for each action dimension" + + action_norm_stats = { + "q01": dataset.meta.stats["action"]["q01"].tolist(), + "q99": dataset.meta.stats["action"]["q99"].tolist(), + } + return action_norm_stats + + +def create_rlds_dataset_stats_dict_from_lerobot_dataset( + dataset: LeRobotDataset, +) -> dict[str, dict[str, float | list[float] | dict]]: + """ + Create a dictionary of statistics from a v2.0 LeRobotDataset that stores + action normalization statistics. + """ + + try: + action_norm_stats = \ + create_action_norm_stats_dict_from_lerobot_dataset(dataset) + except Exception as e: + raise ValueError( + f"Couldn't retrieve action norm stats from dataset: {e}" + ) + + dataset_stats = { + dataset.name: { + # Copy all action statistics + "action": action_norm_stats, + # Add trajectory/transition counts + "num_trajectories": dataset.num_episodes, + "num_transitions": dataset.num_frames + } + } + + # Add proprioceptive statistics if available + if "proprio" in dataset.meta.stats: + dataset_stats[dataset.name]["proprio"] \ + = dataset.meta.stats["proprio"] + + # Add any other available statistics + for key, value in dataset.meta.stats.items(): + if key not in ["action", "proprio"]: + dataset_stats[dataset.name][key] = value + + return dataset_stats + + @dataclass class VLACollatorForLeRobotDataset: """ diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index ab57c0d32..183fa161e 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -59,7 +59,11 @@ get_current_action_mask, get_next_actions_mask, ) -from prismatic.util.data_utils import PaddedCollatorForActionPrediction +from prismatic.util.extern.hf.lerobot_utils import ( + create_action_norm_stats_dict_from_lerobot_dataset, + create_rlds_dataset_stats_dict_from_lerobot_dataset, + VLACollatorForLeRobotDataset, +) from prismatic.vla.action_tokenizer import ActionTokenizer from prismatic.vla.constants import ( ACTION_DIM, @@ -639,7 +643,9 @@ def save_training_checkpoint( if distributed_state.is_main_process: os.makedirs(checkpoint_dir, exist_ok=True) os.makedirs(adapter_dir, exist_ok=True) - save_dataset_statistics(train_dataset.dataset_statistics, checkpoint_dir) + dataset_stats \ + = create_rlds_dataset_stats_dict_from_lerobot_dataset(train_dataset) + save_dataset_statistics(dataset_stats, checkpoint_dir) print(f"Saving Model Checkpoint for Step {log_step}") # Wait for directories to be created @@ -1030,31 +1036,11 @@ def finetune(cfg: FinetuneConfig) -> None: if cfg.use_lerobot_dataset: dataset_name = cfg.lerobot_dataset_name.split("/")[-1] - # Build action stats for unnorm and dataset stats for dataloader - action_norm_stats = { - "q01": train_dataset.meta.stats["action"]["q01"].tolist(), - "q99": train_dataset.meta.stats["action"]["q99"].tolist(), - } - dataset_stats = { - dataset_name: { - # Copy all action statistics - "action": action_norm_stats, - # Add trajectory/transition counts - "num_trajectories": train_dataset.num_episodes, - "num_transitions": train_dataset.num_frames - } - } - - # Add proprioceptive statistics if available - if "proprio" in train_dataset.meta.stats: - dataset_stats[dataset_name]["proprio"] \ - = train_dataset.meta.stats["proprio"] + action_norm_stats \ + = create_action_norm_stats_dict_from_lerobot_dataset(train_dataset) + dataset_stats \ + = create_rlds_dataset_stats_dict_from_lerobot_dataset(train_dataset) - # Add any other available statistics - for key, value in train_dataset.meta.stats.items(): - if key not in ["action", "proprio"]: - dataset_stats[dataset_name][key] = value - # Save dataset statistics for unnorming actions during inference if distributed_state.is_main_process: save_dataset_statistics(dataset_stats, run_dir) From 52a8faf00f6218bcf03b400cf61659a3f1c4e8b0 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 20 Mar 2025 10:32:51 +0000 Subject: [PATCH 45/58] rm dangling import --- vla-scripts/finetune.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 183fa161e..bfd454196 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -76,10 +76,7 @@ RLDSDataset, ) from prismatic.vla.datasets.rlds.utils.data_utils import save_dataset_statistics -from prismatic.util.data_utils import ( - greyscale_float_tensor_preprocessing_wrapper, - VLACollatorForLeRobotDataset, -) +from prismatic.util.data_utils import greyscale_float_tensor_preprocessing_wrapper # Sane Defaults From febdc9f3c97913d717ae24b28e1e33de85512cc8 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 20 Mar 2025 10:35:41 +0000 Subject: [PATCH 46/58] better traceback from no-norm-stats val error --- prismatic/util/extern/hf/lerobot_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index 9ccee119b..9f73197ab 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -67,7 +67,7 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( except Exception as e: raise ValueError( f"Couldn't retrieve action norm stats from dataset: {e}" - ) + ) from e dataset_stats = { dataset.name: { From c47a20fccb5cc40f8c4890bdf3f207bd4f1b783d Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 20 Mar 2025 10:53:51 +0000 Subject: [PATCH 47/58] minor fixes to new finetune utils --- prismatic/util/extern/hf/lerobot_utils.py | 7 ++++--- vla-scripts/finetune.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index 9f73197ab..59aa75b6c 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -55,6 +55,7 @@ def create_action_norm_stats_dict_from_lerobot_dataset( def create_rlds_dataset_stats_dict_from_lerobot_dataset( dataset: LeRobotDataset, + dataset_name: str, ) -> dict[str, dict[str, float | list[float] | dict]]: """ Create a dictionary of statistics from a v2.0 LeRobotDataset that stores @@ -70,7 +71,7 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( ) from e dataset_stats = { - dataset.name: { + dataset_name: { # Copy all action statistics "action": action_norm_stats, # Add trajectory/transition counts @@ -81,13 +82,13 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( # Add proprioceptive statistics if available if "proprio" in dataset.meta.stats: - dataset_stats[dataset.name]["proprio"] \ + dataset_stats[dataset_name]["proprio"] \ = dataset.meta.stats["proprio"] # Add any other available statistics for key, value in dataset.meta.stats.items(): if key not in ["action", "proprio"]: - dataset_stats[dataset.name][key] = value + dataset_stats[dataset_name][key] = value return dataset_stats diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index bfd454196..f36ac89cf 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -641,7 +641,10 @@ def save_training_checkpoint( os.makedirs(checkpoint_dir, exist_ok=True) os.makedirs(adapter_dir, exist_ok=True) dataset_stats \ - = create_rlds_dataset_stats_dict_from_lerobot_dataset(train_dataset) + = create_rlds_dataset_stats_dict_from_lerobot_dataset( + train_dataset, + dataset_name="train" + ) save_dataset_statistics(dataset_stats, checkpoint_dir) print(f"Saving Model Checkpoint for Step {log_step}") @@ -1036,7 +1039,10 @@ def finetune(cfg: FinetuneConfig) -> None: action_norm_stats \ = create_action_norm_stats_dict_from_lerobot_dataset(train_dataset) dataset_stats \ - = create_rlds_dataset_stats_dict_from_lerobot_dataset(train_dataset) + = create_rlds_dataset_stats_dict_from_lerobot_dataset( + train_dataset, + dataset_name="train", + ) # Save dataset statistics for unnorming actions during inference if distributed_state.is_main_process: From b217a5b4fef2d795886ae7a76e7d5183a89fcc30 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 20 Mar 2025 12:01:41 +0000 Subject: [PATCH 48/58] undo syntax error in finetune.py --- vla-scripts/finetune.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index f36ac89cf..5fc03761f 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -695,9 +695,9 @@ def save_training_checkpoint( dist.barrier() - action_head, def run_validation( vla, + action_head, noisy_action_projector, proprio_projector, val_dataloader, @@ -1042,7 +1042,7 @@ def finetune(cfg: FinetuneConfig) -> None: = create_rlds_dataset_stats_dict_from_lerobot_dataset( train_dataset, dataset_name="train", - ) + ) # Save dataset statistics for unnorming actions during inference if distributed_state.is_main_process: From 4ed62864cd8ef0c50b61dccf8d3bed9dc81b66d3 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Thu, 20 Mar 2025 17:25:49 +0000 Subject: [PATCH 49/58] rm unused code in finetune.py --- vla-scripts/finetune.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 5fc03761f..9546a52a8 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -1009,12 +1009,6 @@ def finetune(cfg: FinetuneConfig) -> None: local_files_only=True, video_backend=None, ) - if cfg.use_val_set: - # TODO: create a separate validation set? - indices = list(range(len(train_dataset))) - np.random.shuffle(indices) - split = int(np.floor(0.2 * len(train_dataset))) - train_indices, val_indices = indices[split:], indices[:split] # batch_transform = RLDSBatchTransform( From 705e2928378e9726aaa992a98a6224b15ab02d3e Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 24 Mar 2025 12:06:08 +0100 Subject: [PATCH 50/58] trajectory-based train/val split --- prismatic/util/extern/hf/lerobot_utils.py | 27 +++++++++++++++++++++++ vla-scripts/finetune.py | 22 +++++------------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index 59aa75b6c..4f474e2f7 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -14,6 +14,7 @@ import numpy as np import torch from torch.nn.utils.rnn import pad_sequence +from torch.utils.data import Subset from transformers import PreTrainedTokenizerBase from lerobot.common.datasets.lerobot_dataset import LeRobotDataset @@ -93,6 +94,32 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( return dataset_stats +def create_train_val_split_from_lerobot_dataset( + dataset: LeRobotDataset, + split: float = 0.1, +) -> tuple[LeRobotDataset, LeRobotDataset]: + """ + Create a trajectory-based train/val split from a LeRobotDataset. + """ + + episode_indices = list(range(dataset.num_episodes)) + np.random.shuffle(episode_indices) + split = int(np.floor(split * len(episode_indices))) + step_indices_by_episode = [ + np.arange( + start=dataset.episode_data_index['from'][ep_idx], + stop=dataset.episode_data_index['to'][ep_idx], + ) + for ep_idx in episode_indices + ] + train_indices = np.concatenate(step_indices_by_episode[split:]) + val_indices = np.concatenate(step_indices_by_episode[:split]) + + train_subset = Subset(dataset, train_indices) + val_subset = Subset(dataset, val_indices) + return train_subset, val_subset + + @dataclass class VLACollatorForLeRobotDataset: """ diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 5fc03761f..e6ed4645f 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -62,6 +62,7 @@ from prismatic.util.extern.hf.lerobot_utils import ( create_action_norm_stats_dict_from_lerobot_dataset, create_rlds_dataset_stats_dict_from_lerobot_dataset, + create_train_val_split_from_lerobot_dataset, VLACollatorForLeRobotDataset, ) from prismatic.vla.action_tokenizer import ActionTokenizer @@ -1009,13 +1010,6 @@ def finetune(cfg: FinetuneConfig) -> None: local_files_only=True, video_backend=None, ) - if cfg.use_val_set: - # TODO: create a separate validation set? - indices = list(range(len(train_dataset))) - np.random.shuffle(indices) - split = int(np.floor(0.2 * len(train_dataset))) - train_indices, val_indices = indices[split:], indices[:split] - # batch_transform = RLDSBatchTransform( # action_tokenizer, @@ -1061,15 +1055,11 @@ def finetune(cfg: FinetuneConfig) -> None: ) if cfg.use_val_set: - from torch.utils.data import Subset - - indices = list(range(len(train_dataset))) - np.random.shuffle(indices) - split = int(np.floor(0.2 * len(train_dataset))) - train_indices, val_indices = indices[split:], indices[:split] - - train_subset = Subset(train_dataset, train_indices) - val_subset = Subset(train_dataset, val_indices) + train_subset, val_subset \ + = create_train_val_split_from_lerobot_dataset( + train_dataset, + split=0.1, + ) train_sampler = RandomSampler(train_subset) dataloader = DataLoader( From f8507ebce0382a9ee85f378b2de6f4e2365fdca0 Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 24 Mar 2025 12:07:52 +0100 Subject: [PATCH 51/58] add param to pick constants.py cfg --- vla-scripts/finetune.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index e6ed4645f..716c9625c 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -142,6 +142,8 @@ class FinetuneConfig: lerobot_dataset_name: str = "robotgeneralist/nomagic-simple-box" lerobot_tolerance_s: float = 0.01 + # Environment + constants_config: str = "ur5e" # Which set of constants (from constants.py) to use # fmt: on From 82e2aa92935a9b8b61d1e599f6812807fea59c97 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Sat, 29 Mar 2025 13:04:29 +0100 Subject: [PATCH 52/58] adding robot_interface as dependency --- .devcontainer/devcontainer.json | 67 ++++++++++++++++---------------- .devcontainer/docker-compose.yml | 9 ++--- .devcontainer/setup_host.sh | 5 +++ Dockerfile | 32 +++++++++++---- pyproject.toml | 1 + 5 files changed, 69 insertions(+), 45 deletions(-) create mode 100644 .devcontainer/setup_host.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a8eb7a295..9dad21981 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,36 +1,37 @@ { - "name": "OpenVLA Development", - "dockerComposeFile": "docker-compose.yml", - "service": "devcontainer", - "workspaceFolder": "/workspace/openvla", - "containerEnv": { - "PYTHONPATH": "${containerWorkspaceFolder}" - }, - "remoteEnv": { - // Environment variables will be loaded from .env - }, - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-toolsai.jupyter", - "github.copilot" - ], - "settings": { - "python.defaultInterpreterPath": "/usr/local/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.provider": "black", - "editor.formatOnSave": true, - "editor.rulers": [121] - } + "name": "OpenVLA Development", + "dockerComposeFile": "docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspace/openvla", + "containerEnv": { + "PYTHONPATH": "${containerWorkspaceFolder}" + }, + "remoteEnv": { + // Environment variables will be loaded from .env + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter", + "github.copilot" + ], + "settings": { + "python.defaultInterpreterPath": "/.venv/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.provider": "black", + "editor.formatOnSave": true, + "editor.rulers": [ + 121 + ] } - }, - "remoteUser": "root", - "postCreateCommand": "pip install -e .", - // Load environment variables from .env file - "features": { - "ghcr.io/devcontainers/features/dotnet:1": {} } - } \ No newline at end of file + }, + "remoteUser": "root", + // Load environment variables from .env file + "features": { + // "ghcr.io/devcontainers/features/dotnet:1": {} + } +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 453872554..a275e44b8 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,3 +1,5 @@ +version: '3.8' # Or your specific version + services: devcontainer: build: @@ -7,7 +9,7 @@ services: command: sleep infinity volumes: - ..:/workspace/openvla:cached - - finetuner-cache:/root/.cache + - /data:/data # Mount /data to be used for large stuff - ${HOME}/.cache/huggingface:/root/.cache/huggingface # use cached models/datasets from host environment: - WANDB_API_KEY=${WANDB_API_KEY} @@ -22,7 +24,4 @@ services: devices: - driver: nvidia count: all - capabilities: [gpu] - -volumes: - finetuner-cache: \ No newline at end of file + capabilities: [gpu] \ No newline at end of file diff --git a/.devcontainer/setup_host.sh b/.devcontainer/setup_host.sh new file mode 100644 index 000000000..8e1f3bb4f --- /dev/null +++ b/.devcontainer/setup_host.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Ensure that what we are mounting exists on host +mkdir -p ${HOME}/.cache/huggingface +mkdir -p /data \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 82e569c00..0731333c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,24 @@ -FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-devel +FROM pzal1/robot_interface_system_deps + +RUN git clone https://github.com/nomagiclab/robot-interface.git /workspace/robot-interface +RUN cd /workspace/robot-interface && git checkout 32-rename-everything-according-to-the-new-robot-interface-name + +# Add this line to check for setup files +RUN echo "--- Contents of /workspace/robot-interface ---" && ls -la /workspace/robot-interface && echo "--------------------------------------------" # Install system dependencies for flash-attn. RUN apt-get update && apt-get install -y \ git \ ninja-build \ + cuda-nvcc-12-1 \ + cuda-cudart-dev-12-1 \ && rm -rf /var/lib/apt/lists/* +# Set CUDA environment variables +ENV CUDA_HOME=/usr/local/cuda-12.1 +# Ensure the venv and CUDA bin dirs are in the PATH +ENV PATH="/.venv/bin:${CUDA_HOME}/bin:$PATH" + WORKDIR /workspace RUN git clone -b merge-finetuner-changes https://github.com/nomagiclab/openvla.git && \ cd openvla && \ @@ -16,16 +29,21 @@ WORKDIR /workspace/openvla RUN ls -la third_party/lerobot +# Ensure pip is functional and clear cache in a separate step +RUN /.venv/bin/python -m ensurepip --upgrade && \ + /.venv/bin/python -m pip cache purge + # Editable install of openvla, then lerobot submodule, then reinstall newly # missing openvla dependencies to negotiate dependency incompatibility. # Then install flash-attn separately (per OpenVLA instructions) # and download the openvla-7b model. -RUN pip install -e . && \ +RUN /.venv/bin/python -m pip install -vvv -e . && \ cd third_party/lerobot/ && \ - pip install -e . && \ + /.venv/bin/python -m pip install -e . && \ cd ../../ && \ - pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' | xargs pip install && \ - pip install packaging ninja && \ - pip install "flash-attn==2.5.5" --no-build-isolation && \ - pip install huggingface-hub && \ + MISSING_DEPS=$(/.venv/bin/python -m pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' || true) && \ + if [ -n "$MISSING_DEPS" ]; then /.venv/bin/python -m pip install $MISSING_DEPS; fi && \ + /.venv/bin/python -m pip install packaging ninja && \ + /.venv/bin/python -m pip install "flash-attn==2.5.5" --no-build-isolation && \ + /.venv/bin/python -m pip install huggingface-hub && \ huggingface-cli download openvla/openvla-7b diff --git a/pyproject.toml b/pyproject.toml index 562e9ba27..559dca6a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "uvicorn", "fastapi", "json-numpy", + "robot-interface @ file:../robot-interface", ] [project.optional-dependencies] From 2a3251037f468d8adc93afd30fcab2ad432e3f0e Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 24 Mar 2025 12:06:08 +0100 Subject: [PATCH 53/58] trajectory-based train/val split --- prismatic/util/extern/hf/lerobot_utils.py | 27 +++++++++++++++++++++++ vla-scripts/finetune.py | 15 +++++-------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index 59aa75b6c..4f474e2f7 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -14,6 +14,7 @@ import numpy as np import torch from torch.nn.utils.rnn import pad_sequence +from torch.utils.data import Subset from transformers import PreTrainedTokenizerBase from lerobot.common.datasets.lerobot_dataset import LeRobotDataset @@ -93,6 +94,32 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( return dataset_stats +def create_train_val_split_from_lerobot_dataset( + dataset: LeRobotDataset, + split: float = 0.1, +) -> tuple[LeRobotDataset, LeRobotDataset]: + """ + Create a trajectory-based train/val split from a LeRobotDataset. + """ + + episode_indices = list(range(dataset.num_episodes)) + np.random.shuffle(episode_indices) + split = int(np.floor(split * len(episode_indices))) + step_indices_by_episode = [ + np.arange( + start=dataset.episode_data_index['from'][ep_idx], + stop=dataset.episode_data_index['to'][ep_idx], + ) + for ep_idx in episode_indices + ] + train_indices = np.concatenate(step_indices_by_episode[split:]) + val_indices = np.concatenate(step_indices_by_episode[:split]) + + train_subset = Subset(dataset, train_indices) + val_subset = Subset(dataset, val_indices) + return train_subset, val_subset + + @dataclass class VLACollatorForLeRobotDataset: """ diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 9546a52a8..c784f57b9 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -62,6 +62,7 @@ from prismatic.util.extern.hf.lerobot_utils import ( create_action_norm_stats_dict_from_lerobot_dataset, create_rlds_dataset_stats_dict_from_lerobot_dataset, + create_train_val_split_from_lerobot_dataset, VLACollatorForLeRobotDataset, ) from prismatic.vla.action_tokenizer import ActionTokenizer @@ -1055,15 +1056,11 @@ def finetune(cfg: FinetuneConfig) -> None: ) if cfg.use_val_set: - from torch.utils.data import Subset - - indices = list(range(len(train_dataset))) - np.random.shuffle(indices) - split = int(np.floor(0.2 * len(train_dataset))) - train_indices, val_indices = indices[split:], indices[:split] - - train_subset = Subset(train_dataset, train_indices) - val_subset = Subset(train_dataset, val_indices) + train_subset, val_subset \ + = create_train_val_split_from_lerobot_dataset( + train_dataset, + split=0.1, + ) train_sampler = RandomSampler(train_subset) dataloader = DataLoader( From 090b6dde5836ee4a989c48f8728c749f9fb8b2f8 Mon Sep 17 00:00:00 2001 From: mehhl Date: Mon, 24 Mar 2025 12:07:52 +0100 Subject: [PATCH 54/58] add param to pick constants.py cfg --- vla-scripts/finetune.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index c784f57b9..6513e43a2 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -142,6 +142,8 @@ class FinetuneConfig: lerobot_dataset_name: str = "robotgeneralist/nomagic-simple-box" lerobot_tolerance_s: float = 0.01 + # Environment + constants_config: str = "ur5e" # Which set of constants (from constants.py) to use # fmt: on From 167dfcbcc37717ed0eb9da218a70d20139b1c8c8 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Sat, 29 Mar 2025 13:17:26 +0100 Subject: [PATCH 55/58] minor fixes to dockerfile --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0731333c5..95a743750 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,6 @@ FROM pzal1/robot_interface_system_deps RUN git clone https://github.com/nomagiclab/robot-interface.git /workspace/robot-interface RUN cd /workspace/robot-interface && git checkout 32-rename-everything-according-to-the-new-robot-interface-name -# Add this line to check for setup files -RUN echo "--- Contents of /workspace/robot-interface ---" && ls -la /workspace/robot-interface && echo "--------------------------------------------" - # Install system dependencies for flash-attn. RUN apt-get update && apt-get install -y \ git \ @@ -24,7 +21,7 @@ RUN git clone -b merge-finetuner-changes https://github.com/nomagiclab/openvla.g cd openvla && \ git submodule init third_party/lerobot && \ git submodule update --recursive --init third_party/lerobot - + WORKDIR /workspace/openvla RUN ls -la third_party/lerobot @@ -37,7 +34,7 @@ RUN /.venv/bin/python -m ensurepip --upgrade && \ # missing openvla dependencies to negotiate dependency incompatibility. # Then install flash-attn separately (per OpenVLA instructions) # and download the openvla-7b model. -RUN /.venv/bin/python -m pip install -vvv -e . && \ +RUN /.venv/bin/python -m pip install -e . && \ cd third_party/lerobot/ && \ /.venv/bin/python -m pip install -e . && \ cd ../../ && \ From af330799f4590c2bd9dd64b6ffa5857119e7d254 Mon Sep 17 00:00:00 2001 From: Eddie Margaret Date: Sat, 29 Mar 2025 13:18:02 +0100 Subject: [PATCH 56/58] change base branch in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95a743750..b25d02b18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ ENV CUDA_HOME=/usr/local/cuda-12.1 ENV PATH="/.venv/bin:${CUDA_HOME}/bin:$PATH" WORKDIR /workspace -RUN git clone -b merge-finetuner-changes https://github.com/nomagiclab/openvla.git && \ +RUN git clone -b 4-investigate-and-likely-switch-to-openvla-oft https://github.com/nomagiclab/openvla.git && \ cd openvla && \ git submodule init third_party/lerobot && \ git submodule update --recursive --init third_party/lerobot From 3dbaff836df7ef2cc7df908cab2143efa7304930 Mon Sep 17 00:00:00 2001 From: mehhl Date: Wed, 9 Apr 2025 14:38:59 +0000 Subject: [PATCH 57/58] revert to 2598b4f Revert to state before direct integration of robot-interface into openvla environment. We will use a different approach: serve openvla in one Python environment (using `vla-scripts/finetune.py`), and query server from robot-interface client in another Python environment. modified: .devcontainer/devcontainer.json modified: .devcontainer/docker-compose.yml deleted: .devcontainer/setup_host.sh modified: Dockerfile modified: prismatic/util/extern/hf/lerobot_utils.py modified: pyproject.toml modified: vla-scripts/finetune.py --- .devcontainer/devcontainer.json | 67 +++++++++++------------ .devcontainer/docker-compose.yml | 9 +-- .devcontainer/setup_host.sh | 5 -- Dockerfile | 33 +++-------- prismatic/util/extern/hf/lerobot_utils.py | 27 --------- pyproject.toml | 1 - vla-scripts/finetune.py | 18 +++--- 7 files changed, 57 insertions(+), 103 deletions(-) delete mode 100644 .devcontainer/setup_host.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9dad21981..a8eb7a295 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,37 +1,36 @@ { - "name": "OpenVLA Development", - "dockerComposeFile": "docker-compose.yml", - "service": "devcontainer", - "workspaceFolder": "/workspace/openvla", - "containerEnv": { - "PYTHONPATH": "${containerWorkspaceFolder}" - }, - "remoteEnv": { - // Environment variables will be loaded from .env - }, - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-toolsai.jupyter", - "github.copilot" - ], - "settings": { - "python.defaultInterpreterPath": "/.venv/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.provider": "black", - "editor.formatOnSave": true, - "editor.rulers": [ - 121 - ] + "name": "OpenVLA Development", + "dockerComposeFile": "docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspace/openvla", + "containerEnv": { + "PYTHONPATH": "${containerWorkspaceFolder}" + }, + "remoteEnv": { + // Environment variables will be loaded from .env + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter", + "github.copilot" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.provider": "black", + "editor.formatOnSave": true, + "editor.rulers": [121] + } } + }, + "remoteUser": "root", + "postCreateCommand": "pip install -e .", + // Load environment variables from .env file + "features": { + "ghcr.io/devcontainers/features/dotnet:1": {} } - }, - "remoteUser": "root", - // Load environment variables from .env file - "features": { - // "ghcr.io/devcontainers/features/dotnet:1": {} - } -} \ No newline at end of file + } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index a275e44b8..453872554 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' # Or your specific version - services: devcontainer: build: @@ -9,7 +7,7 @@ services: command: sleep infinity volumes: - ..:/workspace/openvla:cached - - /data:/data # Mount /data to be used for large stuff + - finetuner-cache:/root/.cache - ${HOME}/.cache/huggingface:/root/.cache/huggingface # use cached models/datasets from host environment: - WANDB_API_KEY=${WANDB_API_KEY} @@ -24,4 +22,7 @@ services: devices: - driver: nvidia count: all - capabilities: [gpu] \ No newline at end of file + capabilities: [gpu] + +volumes: + finetuner-cache: \ No newline at end of file diff --git a/.devcontainer/setup_host.sh b/.devcontainer/setup_host.sh deleted file mode 100644 index 8e1f3bb4f..000000000 --- a/.devcontainer/setup_host.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# Ensure that what we are mounting exists on host -mkdir -p ${HOME}/.cache/huggingface -mkdir -p /data \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b25d02b18..82e569c00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,46 +1,31 @@ -FROM pzal1/robot_interface_system_deps - -RUN git clone https://github.com/nomagiclab/robot-interface.git /workspace/robot-interface -RUN cd /workspace/robot-interface && git checkout 32-rename-everything-according-to-the-new-robot-interface-name +FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-devel # Install system dependencies for flash-attn. RUN apt-get update && apt-get install -y \ git \ ninja-build \ - cuda-nvcc-12-1 \ - cuda-cudart-dev-12-1 \ && rm -rf /var/lib/apt/lists/* -# Set CUDA environment variables -ENV CUDA_HOME=/usr/local/cuda-12.1 -# Ensure the venv and CUDA bin dirs are in the PATH -ENV PATH="/.venv/bin:${CUDA_HOME}/bin:$PATH" - WORKDIR /workspace -RUN git clone -b 4-investigate-and-likely-switch-to-openvla-oft https://github.com/nomagiclab/openvla.git && \ +RUN git clone -b merge-finetuner-changes https://github.com/nomagiclab/openvla.git && \ cd openvla && \ git submodule init third_party/lerobot && \ git submodule update --recursive --init third_party/lerobot - + WORKDIR /workspace/openvla RUN ls -la third_party/lerobot -# Ensure pip is functional and clear cache in a separate step -RUN /.venv/bin/python -m ensurepip --upgrade && \ - /.venv/bin/python -m pip cache purge - # Editable install of openvla, then lerobot submodule, then reinstall newly # missing openvla dependencies to negotiate dependency incompatibility. # Then install flash-attn separately (per OpenVLA instructions) # and download the openvla-7b model. -RUN /.venv/bin/python -m pip install -e . && \ +RUN pip install -e . && \ cd third_party/lerobot/ && \ - /.venv/bin/python -m pip install -e . && \ + pip install -e . && \ cd ../../ && \ - MISSING_DEPS=$(/.venv/bin/python -m pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' || true) && \ - if [ -n "$MISSING_DEPS" ]; then /.venv/bin/python -m pip install $MISSING_DEPS; fi && \ - /.venv/bin/python -m pip install packaging ninja && \ - /.venv/bin/python -m pip install "flash-attn==2.5.5" --no-build-isolation && \ - /.venv/bin/python -m pip install huggingface-hub && \ + pip check | awk '$1 ~ /openvla/ {gsub(/,/,"",$5); print $5}' | xargs pip install && \ + pip install packaging ninja && \ + pip install "flash-attn==2.5.5" --no-build-isolation && \ + pip install huggingface-hub && \ huggingface-cli download openvla/openvla-7b diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index 4f474e2f7..59aa75b6c 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -14,7 +14,6 @@ import numpy as np import torch from torch.nn.utils.rnn import pad_sequence -from torch.utils.data import Subset from transformers import PreTrainedTokenizerBase from lerobot.common.datasets.lerobot_dataset import LeRobotDataset @@ -94,32 +93,6 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( return dataset_stats -def create_train_val_split_from_lerobot_dataset( - dataset: LeRobotDataset, - split: float = 0.1, -) -> tuple[LeRobotDataset, LeRobotDataset]: - """ - Create a trajectory-based train/val split from a LeRobotDataset. - """ - - episode_indices = list(range(dataset.num_episodes)) - np.random.shuffle(episode_indices) - split = int(np.floor(split * len(episode_indices))) - step_indices_by_episode = [ - np.arange( - start=dataset.episode_data_index['from'][ep_idx], - stop=dataset.episode_data_index['to'][ep_idx], - ) - for ep_idx in episode_indices - ] - train_indices = np.concatenate(step_indices_by_episode[split:]) - val_indices = np.concatenate(step_indices_by_episode[:split]) - - train_subset = Subset(dataset, train_indices) - val_subset = Subset(dataset, val_indices) - return train_subset, val_subset - - @dataclass class VLACollatorForLeRobotDataset: """ diff --git a/pyproject.toml b/pyproject.toml index 559dca6a5..562e9ba27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,6 @@ dependencies = [ "uvicorn", "fastapi", "json-numpy", - "robot-interface @ file:../robot-interface", ] [project.optional-dependencies] diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 716c9625c..9546a52a8 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -62,7 +62,6 @@ from prismatic.util.extern.hf.lerobot_utils import ( create_action_norm_stats_dict_from_lerobot_dataset, create_rlds_dataset_stats_dict_from_lerobot_dataset, - create_train_val_split_from_lerobot_dataset, VLACollatorForLeRobotDataset, ) from prismatic.vla.action_tokenizer import ActionTokenizer @@ -142,8 +141,6 @@ class FinetuneConfig: lerobot_dataset_name: str = "robotgeneralist/nomagic-simple-box" lerobot_tolerance_s: float = 0.01 - # Environment - constants_config: str = "ur5e" # Which set of constants (from constants.py) to use # fmt: on @@ -1013,6 +1010,7 @@ def finetune(cfg: FinetuneConfig) -> None: video_backend=None, ) + # batch_transform = RLDSBatchTransform( # action_tokenizer, # processor.tokenizer, @@ -1057,11 +1055,15 @@ def finetune(cfg: FinetuneConfig) -> None: ) if cfg.use_val_set: - train_subset, val_subset \ - = create_train_val_split_from_lerobot_dataset( - train_dataset, - split=0.1, - ) + from torch.utils.data import Subset + + indices = list(range(len(train_dataset))) + np.random.shuffle(indices) + split = int(np.floor(0.2 * len(train_dataset))) + train_indices, val_indices = indices[split:], indices[:split] + + train_subset = Subset(train_dataset, train_indices) + val_subset = Subset(train_dataset, val_indices) train_sampler = RandomSampler(train_subset) dataloader = DataLoader( From dd70fe445517614b51078ddf8c3121791444163c Mon Sep 17 00:00:00 2001 From: Maciej Mehl <106269097+mehhl@users.noreply.github.com> Date: Wed, 9 Apr 2025 16:55:31 +0200 Subject: [PATCH 58/58] 6 debug data format differences between finetuning and evaluation (#7) * fix to val set split * read action chunks from prepared lerobotdatasets temporary fix to a pressing issue, implement action chunking properly by reading actions from 'action_chunk' col of step instead of 'action.pose' and 'action.gripper'. 'action_chunk' col must be prepared offline * updated slurm jobconf for action chunked lerbdset * fixes to manual create env * create data/ dir when manually creating env * fix double-saving weights when finetuning --- manual_create_env.sh | 11 ++- prismatic/util/extern/hf/lerobot_utils.py | 98 +++++++++++++++++++---- vla-scripts/finetune.py | 43 ++++++---- vla-scripts/finetune.sub | 52 ++++++++---- 4 files changed, 158 insertions(+), 46 deletions(-) diff --git a/manual_create_env.sh b/manual_create_env.sh index 1f276992d..333f06396 100755 --- a/manual_create_env.sh +++ b/manual_create_env.sh @@ -73,13 +73,22 @@ function download_model { huggingface-cli download openvla/openvla-7b || exit 1 } +function create_save_dirs { + verify_directory + mkdir -p data + mkdir -p .runs + mkdir -p .slurmlog +} + function main { check_virtualenv verify_directory setup_virtualenv install_dependencies download_model + create_save_dirs } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main "$@" \ No newline at end of file + main "$@" +fi \ No newline at end of file diff --git a/prismatic/util/extern/hf/lerobot_utils.py b/prismatic/util/extern/hf/lerobot_utils.py index 59aa75b6c..e1434bdac 100644 --- a/prismatic/util/extern/hf/lerobot_utils.py +++ b/prismatic/util/extern/hf/lerobot_utils.py @@ -93,6 +93,38 @@ def create_rlds_dataset_stats_dict_from_lerobot_dataset( return dataset_stats +def create_train_val_split_from_lerobot_dataset( + dataset: LeRobotDataset, + split: float = 0.1, +) -> tuple[LeRobotDataset, LeRobotDataset]: + """ + Create a trajectory-based train/val split from a LeRobotDataset. + """ + + episode_indices = list(range(dataset.num_episodes)) + np.random.shuffle(episode_indices) + split = int(np.floor(split * len(episode_indices))) + step_indices_by_episode = [ + np.arange( + start=dataset.episode_data_index['from'][ep_idx], + stop=dataset.episode_data_index['to'][ep_idx], + ) + for ep_idx in episode_indices + ] + train_indices = [ + int(idx) + for idx in np.concatenate(step_indices_by_episode[split:]) + ] + val_indices = [ + int(idx) + for idx in np.concatenate(step_indices_by_episode[:split]) + ] + + train_subset = Subset(dataset, train_indices) + val_subset = Subset(dataset, val_indices) + return train_subset, val_subset + + @dataclass class VLACollatorForLeRobotDataset: """ @@ -124,21 +156,57 @@ def __call__(self, instances: Sequence[dict[str, torch.Tensor]]) -> dict[str, to # Extract task/instruction task = item.get("task", "") - # Extract and normalize actions - action = torch.cat([ - item.get("action.pose", torch.zeros(6)), - item.get("action.gripper", torch.zeros(1)).unsqueeze(0) - ]) - + # === MODIFIED: Extract action_chunk === + # TODO(alan): Remove this once we have a way to get the action chunks from the dataset + # Retrieve the pre-computed action chunk (which is already a tensor from the Dataset subclass) + action_chunk_tensor = item.get("action_chunk") + if action_chunk_tensor is None: + raise NotImplementedError( + "Item missing 'action_chunk'. This collator requires the dataset to be pre-processed " + "by a script (e.g., create_dataset_with_action_chunking.py) to add this column. " + "Standard LeRobotDataset does not provide chunks directly in this format." + ) + + # Ensure the tensor has the correct dtype (float32) + action_chunk_tensor = action_chunk_tensor.to(dtype=torch.float32) + + # === Original Sanity Check (can keep) === + if action_chunk_tensor.shape[1] != ACTION_DIM: + raise ValueError(f"Action chunk dimension {action_chunk_tensor.shape[1]} does not match ACTION_DIM {ACTION_DIM}") + # ======================================== + + # === MODIFIED: Normalize the whole chunk === # Normalize actions if stats are provided if self.action_norm_stats is not None: - q01, q99 = self.action_norm_stats.get("q01"), self.action_norm_stats.get("q99") - if q01 is not None and q99 is not None: - action = (2*action - torch.tensor(q01) - torch.tensor(q99)) / (torch.tensor(q99) - torch.tensor(q01)) - - # Tokenize action - action_tokens = self.action_tokenizer(action) - + q01 = self.action_norm_stats.get("q01") + q99 = self.action_norm_stats.get("q99") + # Raise error if normalization stats are expected but incomplete + if q01 is None or q99 is None: + raise ValueError( + "'action_norm_stats' was provided, but missing " + "'q01' or 'q99' keys. Cannot normalize actions." + ) + + # Proceed with normalization only if stats are valid + q01 = torch.tensor(q01, dtype=action_chunk_tensor.dtype) + q99 = torch.tensor(q99, dtype=action_chunk_tensor.dtype) + # Apply normalization across the whole chunk tensor + normalized_action_chunk \ + = (2 * action_chunk_tensor - q01 - q99) / (q99 - q01) + else: + # Keep actions unnormalized if no stats were provided at all + raise ValueError( + "Action normalization stats (`action_norm_stats`) " + "were not provided to the collator, but normalization " + "is expected." + ) + # ========================================= + + # === MODIFIED: Tokenize the flattened chunk === + # Tokenize the flattened action chunk sequence + action_tokens = self.action_tokenizer(normalized_action_chunk.view(-1)) + # ============================================ + # 2. Build prompt prompt_builder = self.prompt_builder_fn("openvla") conversation = [ @@ -160,7 +228,7 @@ def __call__(self, instances: Sequence[dict[str, torch.Tensor]]) -> dict[str, to labels = input_ids.clone() # 5. Mask labels (only keep action tokens for loss) - action_tokens_len = len(action_tokens) + action_tokens_len = len(action_tokens) # Length based on action chunk labels[:-action_tokens_len-1] = IGNORE_INDEX if not self.predict_stop_token: labels[-1] = IGNORE_INDEX @@ -170,7 +238,7 @@ def __call__(self, instances: Sequence[dict[str, torch.Tensor]]) -> dict[str, to "input_ids": input_ids, "labels": labels, "pixel_values": item.get("pixel_values") if "pixel_values" in item else item.get(next(k for k in item if "image" in k.lower())), - "actions": action + "actions": normalized_action_chunk # Store the potentially normalized action chunk tensor } # Add dataset name if available diff --git a/vla-scripts/finetune.py b/vla-scripts/finetune.py index 9546a52a8..f5c1146c6 100644 --- a/vla-scripts/finetune.py +++ b/vla-scripts/finetune.py @@ -1055,15 +1055,17 @@ def finetune(cfg: FinetuneConfig) -> None: ) if cfg.use_val_set: - from torch.utils.data import Subset - - indices = list(range(len(train_dataset))) - np.random.shuffle(indices) - split = int(np.floor(0.2 * len(train_dataset))) - train_indices, val_indices = indices[split:], indices[:split] - - train_subset = Subset(train_dataset, train_indices) - val_subset = Subset(train_dataset, val_indices) + train_subset, val_subset \ + = create_train_val_split_from_lerobot_dataset( + train_dataset, + split=0.1, + ) + + train_indices_list = [int(idx) for idx in train_subset.indices] + val_indices_list = [int(idx) for idx in val_subset.indices] + + print(f"All train indices: {train_indices_list}") + print(f"All val indices: {val_indices_list}") train_sampler = RandomSampler(train_subset) dataloader = DataLoader( @@ -1175,10 +1177,12 @@ def finetune(cfg: FinetuneConfig) -> None: ) # Optimizer Step - if ( - (batch_idx + 1) % cfg.grad_accumulation_steps == 0 - or batch_idx == len(dataloader) - 1 - ): + max_num_grad_acc_steps_done: bool \ + = (batch_idx + 1) % cfg.grad_accumulation_steps == 0 + all_batches_done: bool \ + = batch_idx == len(dataloader) - 1 + time_to_step: bool = max_num_grad_acc_steps_done or all_batches_done + if time_to_step: optimizer.step() scheduler.step() optimizer.zero_grad() @@ -1186,7 +1190,11 @@ def finetune(cfg: FinetuneConfig) -> None: total_gradient_step_idx += 1 # Save model checkpoint:o either keep latest checkpoint only or all checkpoints - if epoch_gradient_step_idx > 0 and log_step % cfg.save_freq == 0: + if ( + time_to_step + and log_step > 0 + and log_step % cfg.save_freq == 0 + ): save_training_checkpoint( cfg=cfg, run_dir=run_dir, @@ -1201,7 +1209,12 @@ def finetune(cfg: FinetuneConfig) -> None: ) # Test model on validation set - if cfg.use_val_set and log_step > 0 and log_step % cfg.val_freq == 0: + if ( + time_to_step + and cfg.use_val_set + and log_step > 0 + and log_step % cfg.val_freq == 0 + ): run_validation( vla=vla, action_head=action_head, diff --git a/vla-scripts/finetune.sub b/vla-scripts/finetune.sub index 47416d6af..0d5178765 100644 --- a/vla-scripts/finetune.sub +++ b/vla-scripts/finetune.sub @@ -1,11 +1,11 @@ #!/bin/bash #SBATCH -N 1 #SBATCH -n 1 -#SBATCH -c 8 -#SBATCH --gres=gpu:1 -#SBATCH -t 6:00:00 +#SBATCH -c 16 +#SBATCH --gres=gpu:2 +#SBATCH -t 48:00:00 #SBATCH -p a100 -#SBATCH --mem=50G +#SBATCH --mem=100G #SBATCH -o .slurmlog/slurm-%j.out #SBATCH -e .slurmlog/slurm-%j.err @@ -41,27 +41,49 @@ fi # Note: We use the absolute path to finetune.py to avoid issues with # relative paths within the Slurm job. PYTHONPATH="$PYTHONPATH:$(pwd)/lerobot" \ +mkdir -p "$(pwd)/.runs" && \ +mkdir -p "$(pwd)/.slurmlog" && \ torchrun \ --standalone \ --nnodes 1 \ - --nproc-per-node 1 \ + --nproc-per-node 2 \ "$(pwd)"/vla-scripts/finetune.py \ --vla_path "openvla/openvla-7b" \ --data_root_dir "data" \ --dataset_name "nomagic-simple-box" \ --run_root_dir ".runs/" \ - --adapter_tmp_dir ".adapter/" \ - --lora_rank 32 \ - --batch_size 16 \ - --grad_accumulation_steps 1 \ + --shuffle_buffer_size 100000 \ + --use_l1_regression true \ + --use_diffusion false \ + --num_diffusion_steps 50 \ + --use_film true \ + --num_images_in_input 3 \ + --use_proprio false \ + --batch_size 4 \ --learning_rate 5e-4 \ - --image_aug True \ - --max_steps 25000 \ - --save_steps 1000 \ - --save_latest_checkpoint_only False \ - --tolerance_s 0.01 \ + --lr_warmup_steps 0 \ + --num_steps_before_decay 100000 \ + --grad_accumulation_steps 2 \ + --max_steps 200000 \ + --use_val_set true \ + --val_freq 1000 \ + --val_time_limit 180 \ + --save_freq 1000 \ + --save_latest_checkpoint_only true \ + --resume false \ + --image_aug true \ + --use_lora true \ + --lora_rank 32 \ + --lora_dropout 0.1 \ + --merge_lora_during_training false \ --wandb_entity robotgeneralist \ - --wandb_project openvla + --wandb_project openvla \ + --wandb_log_freq 100 \ + --use_lerobot_dataset true \ + --lerobot_dataset_root_dir "data" \ + --lerobot_dataset_name "robotgeneralist/nomagic-simple-box-action-chunk-size-8" \ + --lerobot_tolerance_s 0.01 \ + --constants_config ur5e # To run this script, edit the options above, and then # execute the following command from the root repository directory: