From 4511aa3ccc1410fbbd721338d1af760481db0802 Mon Sep 17 00:00:00 2001 From: mio Date: Fri, 10 Apr 2026 17:06:29 +0800 Subject: [PATCH 1/5] =?UTF-8?q?refactor:=20retire=20motrix=5Flegacy=20?= =?UTF-8?q?=E2=80=94=20owner-layer=20decomposition=20(#177)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decompose the `motrix_legacy` compatibility bag into three owner-layer mechanisms, following the existing `reward_motrix` sidecar pattern: **Reward** — route all backend-specific rewards through `reward@reward_motrix` Hydra defaults. Motrix reward YAML files already existed; task YAMLs now reference them via defaults list instead of embedding reward_config in the bag. `extract_reward_config()` remains the single reward resolution path. **Algo** — new `algo_motrix` sidecar config group (4 files under `conf/{ppo,offpolicy}/algo_motrix/`). `BackendAdapter.resolve_algo_dict()` merges the sidecar over `cfg.algo` and returns a new dict — cfg.algo is never mutated. Training scripts read from the resolved dict. Supports `applies_to.algo` filtering (e.g., SAC-only overrides for Go1 offpolicy) and CLI explicit-key precedence. **Env** — backend-specific env overrides in `backend_profile` section of task YAML, using flat fields that match env cfg structure (e.g., `commands.vel_limit`, `control_config.action_scale`, `gait_phase_init_mode`). Applied via `registry.make` env_cfg_override — no `apply_*()` methods. Removed: - `motrix_legacy` config key from all 5 task YAMLs - `LegacyMotrixProfile` dataclass + `apply_legacy_motrix_profile()` from Go1/Go2 - `backend_overrides: dict` + `apply_backend_overrides()` from G1 - `_use_legacy_motrix_baseline()` — replaced by direct `backend_type` check - `adapt_legacy_go1_motrix_obs_if_needed()` from off-policy worker (dead code) - Runtime mutation of `cfg.algo` by BackendAdapter Updated: - Commands.vel_limit → proper dataclass field (enables registry.make override) - Training scripts use resolved algo dict instead of cfg.algo - Tests assert resolved config values, not bag structure - Architecture docs updated --- AGENTS.md | 2 +- conf/offpolicy/algo_motrix/g1_sac.yaml | 1 + conf/offpolicy/algo_motrix/go1.yaml | 4 + conf/offpolicy/task/g1_sac.yaml | 5 +- conf/offpolicy/task/go1_joystick.yaml | 31 +--- conf/ppo/algo_motrix/g1.yaml | 9 + conf/ppo/algo_motrix/go1.yaml | 6 + conf/ppo/task/g1_joystick.yaml | 58 ++---- conf/ppo/task/go1_joystick.yaml | 35 +--- conf/ppo/task/go2_joystick.yaml | 19 +- docs/zh_CN/00-development-architecture.md | 2 +- scripts/train_offpolicy.py | 170 ++++++++++-------- scripts/train_rsl_rl.py | 22 ++- src/unilab/algos/torch/offpolicy/worker.py | 61 +------ src/unilab/envs/locomotion/common/commands.py | 12 +- src/unilab/envs/locomotion/g1/joystick.py | 40 +---- src/unilab/envs/locomotion/go1/joystick.py | 15 -- src/unilab/envs/locomotion/go2/joystick.py | 15 -- src/unilab/training/backend_adapter.py | 111 +++++++++--- tests/scripts/test_train_scripts.py | 149 ++++++++------- tests/training/test_training_helpers.py | 21 ++- 21 files changed, 355 insertions(+), 433 deletions(-) create mode 100644 conf/offpolicy/algo_motrix/g1_sac.yaml create mode 100644 conf/offpolicy/algo_motrix/go1.yaml create mode 100644 conf/ppo/algo_motrix/g1.yaml create mode 100644 conf/ppo/algo_motrix/go1.yaml diff --git a/AGENTS.md b/AGENTS.md index 01d69a443..663746779 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ UniLab 是一个 **高性能、模块化、contract 驱动** 的 RL infrastructu | 区域 | 不可破坏的不变量 | |------|----------------| | Env | `NpEnvState.obs` 必须是 dict；`reset()` 返回 `(obs_dict, info_dict)`；`obs_groups_spec` 影响 wrapper 和 learner 维度。 | -| Config / Reward | reward 通过 Hydra 注入；`training.sim_backend` 和 `motrix_legacy` 必须尊重显式 override。 | +| Config / Reward | reward 通过 Hydra 注入；`training.sim_backend`、`algo_motrix`、`backend_profile` 必须尊重显式 override。 | | Backend | backend-specific 逻辑留在 backend / env 适配层，不向训练脚本扩散。 | | Async | 不绕开 runner lifecycle，也不另起 collector / learner 同步协议。 | diff --git a/conf/offpolicy/algo_motrix/g1_sac.yaml b/conf/offpolicy/algo_motrix/g1_sac.yaml new file mode 100644 index 000000000..a7fb3d245 --- /dev/null +++ b/conf/offpolicy/algo_motrix/g1_sac.yaml @@ -0,0 +1 @@ +use_symmetry: false diff --git a/conf/offpolicy/algo_motrix/go1.yaml b/conf/offpolicy/algo_motrix/go1.yaml new file mode 100644 index 000000000..24bad6ca7 --- /dev/null +++ b/conf/offpolicy/algo_motrix/go1.yaml @@ -0,0 +1,4 @@ +applies_to: + algo: sac +num_envs: 4096 +max_iterations: 2000 diff --git a/conf/offpolicy/task/g1_sac.yaml b/conf/offpolicy/task/g1_sac.yaml index 577330410..439ec3b63 100644 --- a/conf/offpolicy/task/g1_sac.yaml +++ b/conf/offpolicy/task/g1_sac.yaml @@ -1,5 +1,6 @@ # @package _global_ defaults: + - /algo_motrix@algo_motrix: g1_sac - override /reward: g1_sac_mujoco training: @@ -13,7 +14,3 @@ algo: algo_params: alpha_init: 0.001 target_entropy_ratio: 0.0 -motrix_legacy: - enabled: true - algo_overrides: - use_symmetry: false diff --git a/conf/offpolicy/task/go1_joystick.yaml b/conf/offpolicy/task/go1_joystick.yaml index ca9422c75..df48cc482 100644 --- a/conf/offpolicy/task/go1_joystick.yaml +++ b/conf/offpolicy/task/go1_joystick.yaml @@ -1,5 +1,7 @@ # @package _global_ defaults: + - /reward@reward_motrix: go1_sac_motrix + - /algo_motrix@algo_motrix: go1 - override /reward: go1_sac_mujoco training: @@ -7,27 +9,8 @@ training: algo: num_envs: 2048 max_iterations: 3000 -motrix_legacy: - enabled: true - applies_to: - algo: sac - algo_overrides: - num_envs: 4096 - max_iterations: 2000 - env_cfg_override: - legacy_motrix_profile: - enabled: true - command_vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] - reward_config: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - tracking_sigma: 0.25 - base_height_target: 0.3 +backend_profile: + commands: + vel_limit: + - [0.5, 0.0, 0.0] + - [0.5, 0.0, 0.0] diff --git a/conf/ppo/algo_motrix/g1.yaml b/conf/ppo/algo_motrix/g1.yaml new file mode 100644 index 000000000..e3d75021f --- /dev/null +++ b/conf/ppo/algo_motrix/g1.yaml @@ -0,0 +1,9 @@ +empirical_normalization: true +obs_groups: + actor: + - policy +policy: + init_noise_std: 0.5 +algorithm: + learning_rate: 3.0e-4 + entropy_coef: 5.0e-3 diff --git a/conf/ppo/algo_motrix/go1.yaml b/conf/ppo/algo_motrix/go1.yaml new file mode 100644 index 000000000..2e8587a7c --- /dev/null +++ b/conf/ppo/algo_motrix/go1.yaml @@ -0,0 +1,6 @@ +empirical_normalization: true +policy: + init_noise_std: 0.5 +algorithm: + learning_rate: 3.0e-4 + entropy_coef: 1.0e-3 diff --git a/conf/ppo/task/g1_joystick.yaml b/conf/ppo/task/g1_joystick.yaml index db3f8ede4..3683ebb10 100644 --- a/conf/ppo/task/g1_joystick.yaml +++ b/conf/ppo/task/g1_joystick.yaml @@ -1,5 +1,7 @@ # @package _global_ defaults: + - /reward@reward_motrix: g1_ppo_motrix + - /algo_motrix@algo_motrix: g1 - override /reward: g1_ppo_mujoco training: @@ -12,50 +14,12 @@ algo: - policy actor: - actor -motrix_legacy: - enabled: true - algo_overrides: - max_iterations: 151 - empirical_normalization: true - obs_groups: - actor: - - policy - policy: - init_noise_std: 0.5 - algorithm: - learning_rate: 3.0e-4 - entropy_coef: 5.0e-3 - env_cfg_override: - reward_config: - scales: - tracking_lin_vel: 3.0 - tracking_ang_vel: 0.25 - forward_progress: 1.5 - under_speed: -1.0 - upper_body_pose: -0.05 - feet_phase: 0.5 - lin_vel_z: -1.0 - ang_vel_xy: -0.2 - base_height: -120.0 - orientation: -2.5 - action_rate: -0.005 - pose: -0.05 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.5 - max_tilt_deg: 35.0 - backend_overrides: - enabled: true - control_action_scale: 0.5 - command_vel_limit: - - [0.45, 0.0, 0.0] - - [0.55, 0.0, 0.0] - gait_phase_init_mode: independent - reset_base_qvel_limit: 0.05 - reward_scale_overrides: - under_speed: -1.0 - upper_body_pose: -0.05 - tracking_ang_vel: 0.25 +backend_profile: + control_config: + action_scale: 0.5 + commands: + vel_limit: + - [0.45, 0.0, 0.0] + - [0.55, 0.0, 0.0] + gait_phase_init_mode: independent + reset_base_qvel_limit: 0.05 diff --git a/conf/ppo/task/go1_joystick.yaml b/conf/ppo/task/go1_joystick.yaml index 0c9dea4c2..b6e585511 100644 --- a/conf/ppo/task/go1_joystick.yaml +++ b/conf/ppo/task/go1_joystick.yaml @@ -1,5 +1,7 @@ # @package _global_ defaults: + - /reward@reward_motrix: go1_ppo_motrix + - /algo_motrix@algo_motrix: go1 - override /reward: go1_ppo_mujoco training: @@ -11,31 +13,8 @@ algo: - policy actor: - actor -motrix_legacy: - enabled: true - algo_overrides: - max_iterations: 151 - empirical_normalization: true - policy: - init_noise_std: 0.5 - algorithm: - learning_rate: 3.0e-4 - entropy_coef: 1.0e-3 - env_cfg_override: - legacy_motrix_profile: - enabled: true - command_vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] - reward_config: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 +backend_profile: + commands: + vel_limit: + - [0.5, 0.0, 0.0] + - [0.5, 0.0, 0.0] diff --git a/conf/ppo/task/go2_joystick.yaml b/conf/ppo/task/go2_joystick.yaml index 914ea10bb..1659aa119 100644 --- a/conf/ppo/task/go2_joystick.yaml +++ b/conf/ppo/task/go2_joystick.yaml @@ -19,14 +19,11 @@ algo: algorithm: learning_rate: 3.0e-4 entropy_coef: 1.0e-3 -motrix_legacy: - enabled: true - env_cfg_override: - legacy_motrix_profile: - enabled: true - command_vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] - domain_rand: - randomize_kp: false - randomize_kd: false +backend_profile: + commands: + vel_limit: + - [0.5, 0.0, 0.0] + - [0.5, 0.0, 0.0] + domain_rand: + randomize_kp: false + randomize_kd: false diff --git a/docs/zh_CN/00-development-architecture.md b/docs/zh_CN/00-development-architecture.md index 19e9a3949..50c368a3e 100644 --- a/docs/zh_CN/00-development-architecture.md +++ b/docs/zh_CN/00-development-architecture.md @@ -68,7 +68,7 @@ CPU Physics Sim ──shm──► Collector / IPC ──shm──► GPU Learne UniLab 使用 dataclass + Hydra。schema 位于 `src/unilab/config/structured_configs.py`，运行时配置位于 `conf/{ppo,appo,offpolicy}/`。 -合成顺序: `{algo}/config*.yaml` -> `task=...` -> `reward[_{backend}]` -> CLI override -> 必要时再叠加 `motrix_legacy`。 +合成顺序: `{algo}/config*.yaml` -> `task=...` -> `reward[_{backend}]` -> `algo_motrix` (backend sidecar) -> CLI override -> `backend_profile` (env overrides)。 - reward 必须显式注入 - 如果 backend 选择会影响 task 或 reward 行为，就必须通过 config 表达 diff --git a/scripts/train_offpolicy.py b/scripts/train_offpolicy.py index 7784c1311..4f5622c7d 100644 --- a/scripts/train_offpolicy.py +++ b/scripts/train_offpolicy.py @@ -74,18 +74,29 @@ def extract_play_obs(obs_dict): return obs_out -def build_task_motrix_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict | None: - return BackendAdapter(cfg, root_dir=ROOT_DIR, algo_name=algo_name).build_task_env_cfg_override() +def _backend_adapter(algo_name: str, cfg: DictConfig) -> BackendAdapter: + return BackendAdapter(cfg, root_dir=ROOT_DIR, algo_name=algo_name) -def resolve_sac_use_symmetry(cfg: DictConfig) -> bool: +def build_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict | None: + return _backend_adapter(algo_name, cfg).build_task_env_cfg_override() + + +def resolve_offpolicy_algo(algo_name: str, cfg: DictConfig) -> dict: + return _backend_adapter(algo_name, cfg).resolve_algo_dict() + + +def resolve_sac_use_symmetry(resolved_algo: dict, cfg: DictConfig) -> bool: """Symmetry augmentation currently depends on MuJoCo model APIs.""" - return bool(cfg.algo.use_symmetry and cfg.training.sim_backend == "mujoco") + return bool(resolved_algo.get("use_symmetry", False) and cfg.training.sim_backend == "mujoco") def build_runner(algo_name: str, cfg: DictConfig): """Build algorithm runner from unified Hydra config.""" - env_cfg_override = build_task_motrix_offpolicy_env_cfg_override(algo_name, cfg) + adapter = _backend_adapter(algo_name, cfg) + env_cfg_override = adapter.build_task_env_cfg_override() + algo = adapter.resolve_algo_dict() + algo_params = algo.get("algo_params", {}) if algo_name == "sac": from unilab.algos.torch.fast_sac.learner import FastSACLearner @@ -113,18 +124,18 @@ def build_runner(algo_name: str, cfg: DictConfig): learner_kwargs = { "obs_dim": obs_dim, "action_dim": action_dim, - "gamma": cfg.algo.gamma, - "tau": cfg.algo.tau, - "actor_lr": cfg.algo.actor_lr, - "critic_lr": cfg.algo.critic_lr, - "alpha_lr": cfg.algo.algo_params.alpha_lr, - "alpha_init": cfg.algo.algo_params.alpha_init, - "target_entropy_ratio": cfg.algo.algo_params.target_entropy_ratio, - "actor_hidden_dim": cfg.algo.actor_hidden_dim, - "critic_hidden_dim": cfg.algo.critic_hidden_dim, - "num_atoms": cfg.algo.num_atoms, - "use_layer_norm": cfg.algo.use_layer_norm, - "max_grad_norm": cfg.algo.algo_params.max_grad_norm, + "gamma": algo["gamma"], + "tau": algo["tau"], + "actor_lr": algo["actor_lr"], + "critic_lr": algo["critic_lr"], + "alpha_lr": algo_params["alpha_lr"], + "alpha_init": algo_params["alpha_init"], + "target_entropy_ratio": algo_params["target_entropy_ratio"], + "actor_hidden_dim": algo["actor_hidden_dim"], + "critic_hidden_dim": algo["critic_hidden_dim"], + "num_atoms": algo["num_atoms"], + "use_layer_norm": algo["use_layer_norm"], + "max_grad_norm": algo_params["max_grad_norm"], "use_amp": cfg.training.use_amp, "privileged_dim": privileged_dim, # symmetry not supported in multi-GPU mode (mujoco_model not picklable) @@ -138,17 +149,17 @@ def build_runner(algo_name: str, cfg: DictConfig): algo_type="sac", learner_kwargs=learner_kwargs, num_gpus=cfg.training.num_gpus, - num_envs=cfg.algo.num_envs, - replay_buffer_n=cfg.algo.replay_buffer_n, - batch_size=cfg.algo.batch_size, - warmup_steps=cfg.algo.warmup_steps, - updates_per_step=cfg.algo.updates_per_step, - policy_frequency=cfg.algo.policy_frequency, + num_envs=algo["num_envs"], + replay_buffer_n=algo["replay_buffer_n"], + batch_size=algo["batch_size"], + warmup_steps=algo["warmup_steps"], + updates_per_step=algo["updates_per_step"], + policy_frequency=algo["policy_frequency"], sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, device=device, - actor_hidden_dim=cfg.algo.actor_hidden_dim, - use_layer_norm=cfg.algo.use_layer_norm, + actor_hidden_dim=algo["actor_hidden_dim"], + use_layer_norm=algo["use_layer_norm"], obs_normalization=False, sim_backend=cfg.training.sim_backend, env_cfg_override=env_cfg_override, @@ -158,30 +169,30 @@ def build_runner(algo_name: str, cfg: DictConfig): env_name=cfg.training.task_name, env_cfg_override=env_cfg_override, device=cfg.training.device, - num_envs=cfg.algo.num_envs, - replay_buffer_n=cfg.algo.replay_buffer_n, - batch_size=cfg.algo.batch_size, - warmup_steps=cfg.algo.warmup_steps, - updates_per_step=cfg.algo.updates_per_step, - policy_frequency=cfg.algo.policy_frequency, - gamma=cfg.algo.gamma, - tau=cfg.algo.tau, - actor_lr=cfg.algo.actor_lr, - critic_lr=cfg.algo.critic_lr, - alpha_lr=cfg.algo.algo_params.alpha_lr, - alpha_init=cfg.algo.algo_params.alpha_init, - target_entropy_ratio=cfg.algo.algo_params.target_entropy_ratio, - obs_normalization=cfg.algo.obs_normalization, - actor_hidden_dim=cfg.algo.actor_hidden_dim, - critic_hidden_dim=cfg.algo.critic_hidden_dim, - num_atoms=cfg.algo.num_atoms, - use_layer_norm=cfg.algo.use_layer_norm, - max_grad_norm=cfg.algo.algo_params.max_grad_norm, + num_envs=algo["num_envs"], + replay_buffer_n=algo["replay_buffer_n"], + batch_size=algo["batch_size"], + warmup_steps=algo["warmup_steps"], + updates_per_step=algo["updates_per_step"], + policy_frequency=algo["policy_frequency"], + gamma=algo["gamma"], + tau=algo["tau"], + actor_lr=algo["actor_lr"], + critic_lr=algo["critic_lr"], + alpha_lr=algo_params["alpha_lr"], + alpha_init=algo_params["alpha_init"], + target_entropy_ratio=algo_params["target_entropy_ratio"], + obs_normalization=algo["obs_normalization"], + actor_hidden_dim=algo["actor_hidden_dim"], + critic_hidden_dim=algo["critic_hidden_dim"], + num_atoms=algo["num_atoms"], + use_layer_norm=algo["use_layer_norm"], + max_grad_norm=algo_params["max_grad_norm"], use_amp=cfg.training.use_amp, sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, sim_backend=cfg.training.sim_backend, - use_symmetry=resolve_sac_use_symmetry(cfg), + use_symmetry=resolve_sac_use_symmetry(algo, cfg), ) if algo_name == "td3": @@ -191,31 +202,31 @@ def build_runner(algo_name: str, cfg: DictConfig): env_name=cfg.training.task_name, env_cfg_override=env_cfg_override, device=cfg.training.device, - num_envs=cfg.algo.num_envs, - replay_buffer_n=cfg.algo.replay_buffer_n, - batch_size=cfg.algo.batch_size, - warmup_steps=cfg.algo.warmup_steps, - num_updates=cfg.algo.updates_per_step, - policy_frequency=cfg.algo.policy_frequency, + num_envs=algo["num_envs"], + replay_buffer_n=algo["replay_buffer_n"], + batch_size=algo["batch_size"], + warmup_steps=algo["warmup_steps"], + num_updates=algo["updates_per_step"], + policy_frequency=algo["policy_frequency"], sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, - gamma=cfg.algo.gamma, - tau=cfg.algo.tau, - actor_lr=cfg.algo.actor_lr, - critic_lr=cfg.algo.critic_lr, - actor_hidden_dim=cfg.algo.actor_hidden_dim, - critic_hidden_dim=cfg.algo.critic_hidden_dim, - num_atoms=cfg.algo.num_atoms, - v_min=cfg.algo.algo_params.v_min, - v_max=cfg.algo.algo_params.v_max, - init_scale=cfg.algo.algo_params.init_scale, - log_std_min=cfg.algo.algo_params.log_std_min, - log_std_max=cfg.algo.algo_params.log_std_max, - policy_noise=cfg.algo.algo_params.policy_noise, - noise_clip=cfg.algo.algo_params.noise_clip, - weight_decay=cfg.algo.algo_params.weight_decay, - use_cdq=cfg.algo.algo_params.use_cdq, - obs_normalization=cfg.algo.obs_normalization, + gamma=algo["gamma"], + tau=algo["tau"], + actor_lr=algo["actor_lr"], + critic_lr=algo["critic_lr"], + actor_hidden_dim=algo["actor_hidden_dim"], + critic_hidden_dim=algo["critic_hidden_dim"], + num_atoms=algo["num_atoms"], + v_min=algo_params["v_min"], + v_max=algo_params["v_max"], + init_scale=algo_params["init_scale"], + log_std_min=algo_params["log_std_min"], + log_std_max=algo_params["log_std_max"], + policy_noise=algo_params["policy_noise"], + noise_clip=algo_params["noise_clip"], + weight_decay=algo_params["weight_decay"], + use_cdq=algo_params["use_cdq"], + obs_normalization=algo["obs_normalization"], sim_backend=cfg.training.sim_backend, ) @@ -229,7 +240,9 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: from unilab.utils.algo_utils import build_actor - env_cfg_override = build_task_motrix_offpolicy_env_cfg_override(algo_name, cfg) + adapter = _backend_adapter(algo_name, cfg) + env_cfg_override = adapter.build_task_env_cfg_override() + algo = adapter.resolve_algo_dict() device = default_device(torch, cfg.training.device) print(f"Using device for play: {device}") @@ -248,24 +261,25 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: "sac", obs_dim, action_dim, - cfg.algo.actor_hidden_dim, - cfg.algo.use_layer_norm, + algo["actor_hidden_dim"], + algo["use_layer_norm"], device, ) elif algo_name == "td3": from unilab.algos.torch.fast_td3.learner import EmpiricalNormalization, TD3Actor + algo_params = algo.get("algo_params", {}) actor = TD3Actor( obs_dim, action_dim, cfg.training.play_env_num, - cfg.algo.algo_params.init_scale, - cfg.algo.actor_hidden_dim, - cfg.algo.algo_params.log_std_min, - cfg.algo.algo_params.log_std_max, + algo_params["init_scale"], + algo["actor_hidden_dim"], + algo_params["log_std_min"], + algo_params["log_std_max"], device, ) - if cfg.algo.obs_normalization: + if algo["obs_normalization"]: normalizer = EmpiricalNormalization(shape=obs_dim, device=device) else: raise ValueError(f"Unsupported algo: {algo_name}") @@ -274,9 +288,9 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: load_path, load_path_dir = resolve_checkpoint_path( ROOT_DIR, - cfg.algo.algo_log_name, + algo["algo_log_name"], cfg.training.task_name, - cfg.algo.load_run, + algo["load_run"], ) if not load_path or not os.path.exists(load_path): print(f"Could not find checkpoint. load_path={load_path}") diff --git a/scripts/train_rsl_rl.py b/scripts/train_rsl_rl.py index 781721574..0f8f51a30 100644 --- a/scripts/train_rsl_rl.py +++ b/scripts/train_rsl_rl.py @@ -45,14 +45,18 @@ def _backend_adapter(cfg: DictConfig) -> BackendAdapter: ) -def build_task_motrix_ppo_env_cfg_override(cfg: DictConfig) -> dict: +def build_ppo_env_cfg_override(cfg: DictConfig) -> dict: return _backend_adapter(cfg).build_task_env_cfg_override() -def build_motrix_play_ppo_env_cfg_override(cfg: DictConfig) -> dict: +def build_ppo_play_env_cfg_override(cfg: DictConfig) -> dict: return _backend_adapter(cfg).build_play_env_cfg_override() +def resolve_ppo_algo(cfg: DictConfig) -> dict: + return _backend_adapter(cfg).resolve_algo_dict() + + def run_motrix_rsl_play_loop( wrapped_env, policy, @@ -80,7 +84,7 @@ def _get_log_root(cfg: DictConfig) -> str: def play_rsl_rl(cfg: DictConfig, device: str) -> str | None: """Play mode for RSL-RL.""" - env_cfg_override = build_motrix_play_ppo_env_cfg_override(cfg) + env_cfg_override = build_ppo_play_env_cfg_override(cfg) env = create_env( cfg, @@ -158,7 +162,9 @@ def play_rsl_rl(cfg: DictConfig, device: str) -> str | None: def main(cfg: DictConfig) -> None: ensure_registries() - env_cfg_override = build_task_motrix_ppo_env_cfg_override(cfg) + adapter = _backend_adapter(cfg) + env_cfg_override = adapter.build_task_env_cfg_override() + resolved_algo = adapter.resolve_algo_dict() if torch.cuda.is_available(): device = "cuda" @@ -169,9 +175,9 @@ def main(cfg: DictConfig) -> None: print(f"Using device: {device}") # Compute effective max_iterations (supports num_timesteps override) - max_iterations = cfg.algo.max_iterations + max_iterations = resolved_algo["max_iterations"] if cfg.training.num_timesteps: - n_steps_per_iter = cfg.algo.num_steps_per_env * cfg.algo.num_envs + n_steps_per_iter = resolved_algo["num_steps_per_env"] * resolved_algo["num_envs"] max_iterations = max(1, int(cfg.training.num_timesteps / n_steps_per_iter)) print( f"Overriding max_iterations to {max_iterations} based on " @@ -205,12 +211,12 @@ def main(cfg: DictConfig) -> None: if not cfg.training.play_only: env = create_env( cfg, - num_envs=cfg.algo.num_envs, + num_envs=resolved_algo["num_envs"], env_cfg_override=env_cfg_override, ) wrapped_env = RslRlVecEnvWrapper(env, device=device) - train_cfg = OmegaConf.to_container(cfg.algo, resolve=True) + train_cfg = dict(resolved_algo) if "runner" not in train_cfg: train_cfg["runner"] = {} diff --git a/src/unilab/algos/torch/offpolicy/worker.py b/src/unilab/algos/torch/offpolicy/worker.py index a056d7cd0..baa8f50cb 100644 --- a/src/unilab/algos/torch/offpolicy/worker.py +++ b/src/unilab/algos/torch/offpolicy/worker.py @@ -22,7 +22,7 @@ def resolve_collector_actor_dims( """Resolve actor dims for the collector. Prefer explicit dims from the parent process so learner and collector - build identical actor shapes on legacy / override-heavy env paths. + build identical actor shapes on override-heavy env paths. """ if obs_dim is None: from unilab.utils.obs_utils import get_obs_dims @@ -38,44 +38,6 @@ def resolve_collector_actor_dims( return obs_dim, action_dim -def adapt_legacy_go1_motrix_obs_if_needed( - *, - env_name: str, - env_cfg_override: dict | None, - obs: np.ndarray, - privileged: np.ndarray | None, - expected_obs_dim: int, -) -> tuple[np.ndarray, np.ndarray | None]: - """Adapt legacy Go1 Motrix collector obs when the env still emits the default layout. - - Default Go1 obs is 49D and privileged linvel is 3D: - gyro(3) + gravity(3) + diff(12) + dof_vel(12) + last_actions(12) + command(3) + feet_phase(4) - - Legacy Motrix Go1 obs is 48D: - linvel(3) + gyro(3) + gravity(3) + diff(12) + dof_vel(12) + last_actions(12) + command(3) - - If the collector unexpectedly sees the default 49D obs while the runner - expects the legacy 48D layout, rebuild the exact legacy obs from - privileged linvel plus the first 45 default obs dims. - """ - if obs.shape[1] == expected_obs_dim: - return obs, privileged - - legacy_profile = (env_cfg_override or {}).get("legacy_motrix_profile", {}) - if ( - env_name == "Go1JoystickFlatTerrain" - and legacy_profile.get("enabled", False) - and expected_obs_dim == 48 - and obs.shape[1] == 49 - and privileged is not None - and privileged.shape[1] == 3 - ): - adapted_obs = np.concatenate([privileged, obs[:, :45]], axis=1).astype(np.float32) - return adapted_obs, None - - return obs, privileged - - def off_policy_collector_fn( stop_event, env_name: str, @@ -220,13 +182,6 @@ def _run_collector( obs_np = np.asarray(obs_np, dtype=np.float32) if priv_np is not None: priv_np = np.asarray(priv_np, dtype=np.float32) - obs_np, priv_np = adapt_legacy_go1_motrix_obs_if_needed( - env_name=env_name, - env_cfg_override=env_cfg_override, - obs=obs_np, - privileged=priv_np, - expected_obs_dim=obs_dim, - ) prev_dones_np = np.zeros(num_envs, dtype=np.float32) max_episode_steps = getattr(getattr(env, "cfg", None), "max_episode_steps", None) if max_episode_steps is not None and int(max_episode_steps) > 0: @@ -302,13 +257,6 @@ def _run_collector( next_obs_np = np.asarray(next_obs_np, dtype=np.float32) if next_priv_np is not None: next_priv_np = np.asarray(next_priv_np, dtype=np.float32) - next_obs_np, next_priv_np = adapt_legacy_go1_motrix_obs_if_needed( - env_name=env_name, - env_cfg_override=env_cfg_override, - obs=next_obs_np, - privileged=next_priv_np, - expected_obs_dim=obs_dim, - ) rewards_np = np.asarray(state.reward, dtype=np.float32).ravel() terminated_np = ( @@ -340,13 +288,6 @@ def _run_collector( final_obs_np = np.asarray(final_obs_np, dtype=np.float32) if final_priv_np is not None: final_priv_np = np.asarray(final_priv_np, dtype=np.float32) - final_obs_np, final_priv_np = adapt_legacy_go1_motrix_obs_if_needed( - env_name=env_name, - env_cfg_override=env_cfg_override, - obs=final_obs_np, - privileged=final_priv_np, - expected_obs_dim=obs_dim, - ) next_obs_np[has_final_np] = final_obs_np[has_final_np] if final_priv_np is not None and next_priv_np is not None: next_priv_np[has_final_np] = final_priv_np[has_final_np] diff --git a/src/unilab/envs/locomotion/common/commands.py b/src/unilab/envs/locomotion/common/commands.py index b17504aee..13316bab9 100644 --- a/src/unilab/envs/locomotion/common/commands.py +++ b/src/unilab/envs/locomotion/common/commands.py @@ -1,6 +1,6 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field import numpy as np @@ -9,10 +9,12 @@ @dataclass class Commands: - vel_limit = [ - [-0.6, -0.4, -0.8], # [vx_min, vy_min, vyaw_min] - [1.0, 0.4, 0.8], # [vx_max, vy_max, vyaw_max] - ] + vel_limit: list[list[float]] = field( + default_factory=lambda: [ + [-0.6, -0.4, -0.8], # [vx_min, vy_min, vyaw_min] + [1.0, 0.4, 0.8], # [vx_max, vy_max, vyaw_max] + ] + ) def sample_velocity_commands( diff --git a/src/unilab/envs/locomotion/g1/joystick.py b/src/unilab/envs/locomotion/g1/joystick.py index 5726621c2..037260b80 100644 --- a/src/unilab/envs/locomotion/g1/joystick.py +++ b/src/unilab/envs/locomotion/g1/joystick.py @@ -110,35 +110,6 @@ class G1JoystickPPOCfg(G1BaseCfg): domain_rand: DomainRandConfig = field(default_factory=DomainRandConfig) gait_phase_init_mode: str = "offset_phase" reset_base_qvel_limit: float = 0.5 - backend_overrides: dict | None = None - - def apply_backend_overrides(self, backend_type: str) -> None: - if not self.backend_overrides or not self.backend_overrides.get("enabled", False): - return - if backend_type != "motrix": - return - if self.reward_config is None: - raise ValueError("reward_config must be provided before applying backend overrides") - - action_scale = self.backend_overrides.get("control_action_scale") - if action_scale is not None: - self.control_config.action_scale = float(action_scale) - - command_vel_limit = self.backend_overrides.get("command_vel_limit") - if command_vel_limit is not None: - self.commands.vel_limit = [list(command_vel_limit[0]), list(command_vel_limit[1])] - - reward_scale_overrides = self.backend_overrides.get("reward_scale_overrides") - if reward_scale_overrides: - self.reward_config.scales.update(reward_scale_overrides) - - gait_phase_init_mode = self.backend_overrides.get("gait_phase_init_mode") - if gait_phase_init_mode is not None: - self.gait_phase_init_mode = str(gait_phase_init_mode) - - reset_base_qvel_limit = self.backend_overrides.get("reset_base_qvel_limit") - if reset_base_qvel_limit is not None: - self.reset_base_qvel_limit = float(reset_base_qvel_limit) class G1JoystickDomainRandomizationProvider(LocomotionDRProvider): @@ -181,7 +152,6 @@ class G1JoystickPPO(G1BaseEnv): def __init__(self, cfg: G1JoystickPPOCfg, num_envs=1, backend_type="mujoco"): if cfg.reward_config is None: raise ValueError("reward_config must be provided via Hydra configuration") - cfg.apply_backend_overrides(backend_type) backend = create_backend( backend_type, cfg.model_file, num_envs, cfg.sim_dt, base_name=cfg.asset.base_name ) @@ -221,14 +191,6 @@ def _init_reward_functions(self): "feet_phase": self._reward_feet_phase, } - def _use_legacy_motrix_baseline(self) -> bool: - backend_type = getattr(getattr(self, "_backend", None), "backend_type", None) - return bool( - backend_type == "motrix" - and self._cfg.backend_overrides - and self._cfg.backend_overrides.get("enabled", False) - ) - def update_state(self, state: NpEnvState) -> NpEnvState: linvel = self.get_local_linvel() gyro = self.get_gyro() @@ -262,7 +224,7 @@ def _compute_obs( return {"obs": actor, "privileged": linvel} def get_policy_obs(self, obs: dict[str, np.ndarray]) -> np.ndarray: - if self._use_legacy_motrix_baseline() and "privileged" in obs: + if getattr(self._backend, "backend_type", None) == "motrix" and "privileged" in obs: return np.concatenate([obs["privileged"], obs["obs"]], axis=1) return np.concatenate(list(obs.values()), axis=1) diff --git a/src/unilab/envs/locomotion/go1/joystick.py b/src/unilab/envs/locomotion/go1/joystick.py index df3e83435..675ad5b4e 100644 --- a/src/unilab/envs/locomotion/go1/joystick.py +++ b/src/unilab/envs/locomotion/go1/joystick.py @@ -32,12 +32,6 @@ class RewardConfig: base_height_target: float -@dataclass -class LegacyMotrixProfile: - enabled: bool = False - command_vel_limit: list[list[float]] | None = None - - @dataclass class JoystickSensor: local_linvel = "local_linvel" @@ -54,7 +48,6 @@ class Go1JoystickCfg(Go1BaseCfg): init_state: InitState = field(default_factory=InitState) commands: Commands = field(default_factory=Commands) reward_config: RewardConfig | None = None - legacy_motrix_profile: LegacyMotrixProfile = field(default_factory=LegacyMotrixProfile) sensor: JoystickSensor = field(default_factory=JoystickSensor) # type: ignore[assignment] domain_rand: DomainRandConfig = field( default_factory=lambda: DomainRandConfig( @@ -64,13 +57,6 @@ class Go1JoystickCfg(Go1BaseCfg): ) ) - def apply_legacy_motrix_profile(self) -> None: - profile = self.legacy_motrix_profile - if not profile.enabled: - return - if profile.command_vel_limit is not None: - self.commands.vel_limit = [list(v) for v in profile.command_vel_limit] - class Go1JoystickDomainRandomizationProvider(LocomotionDRProvider): def _compute_reset_obs( @@ -97,7 +83,6 @@ class Go1WalkTask(Go1BaseEnv): def __init__(self, cfg: Go1JoystickCfg, num_envs=1, backend_type="mujoco"): if cfg.reward_config is None: raise ValueError("reward_config must be provided via Hydra configuration") - cfg.apply_legacy_motrix_profile() backend = create_backend( backend_type, cfg.model_file, diff --git a/src/unilab/envs/locomotion/go2/joystick.py b/src/unilab/envs/locomotion/go2/joystick.py index c9a01b816..c53b09df2 100644 --- a/src/unilab/envs/locomotion/go2/joystick.py +++ b/src/unilab/envs/locomotion/go2/joystick.py @@ -41,12 +41,6 @@ class RewardConfig: target_foot_height: float = 0.1 -@dataclass -class LegacyMotrixProfile: - enabled: bool = False - command_vel_limit: list[list[float]] | None = None - - @dataclass class JoystickSensor: local_linvel = "local_linvel" @@ -63,17 +57,9 @@ class Go2JoystickCfg(Go2BaseCfg): init_state: InitState = field(default_factory=InitState) commands: Commands = field(default_factory=Commands) reward_config: RewardConfig | None = None - legacy_motrix_profile: LegacyMotrixProfile = field(default_factory=LegacyMotrixProfile) sensor: JoystickSensor = field(default_factory=JoystickSensor) # type: ignore[assignment] domain_rand: Go2DomainRandConfig = field(default_factory=Go2DomainRandConfig) - def apply_legacy_motrix_profile(self) -> None: - profile = self.legacy_motrix_profile - if not profile.enabled: - return - if profile.command_vel_limit is not None: - self.commands.vel_limit = [list(v) for v in profile.command_vel_limit] - class Go2JoystickDomainRandomizationProvider(LocomotionDRProvider): def _compute_reset_obs( @@ -100,7 +86,6 @@ class Go2WalkTask(Go2BaseEnv): def __init__(self, cfg: Go2JoystickCfg, num_envs=1, backend_type="mujoco"): if cfg.reward_config is None: raise ValueError("reward_config must be provided via Hydra configuration") - cfg.apply_legacy_motrix_profile() backend = create_backend( backend_type, cfg.model_file, diff --git a/src/unilab/training/backend_adapter.py b/src/unilab/training/backend_adapter.py index 0cfd1948d..91669bd1d 100644 --- a/src/unilab/training/backend_adapter.py +++ b/src/unilab/training/backend_adapter.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, cast from omegaconf import DictConfig, OmegaConf @@ -31,33 +31,89 @@ def __init__( arg.split("=", 1)[0].lstrip("+") for arg in sys.argv[1:] if "=" in arg } + def _is_motrix(self) -> bool: + return bool(self.cfg.training.sim_backend == "motrix") + + def resolve_algo_dict(self) -> dict[str, Any]: + """Resolve algo config, merging algo_motrix sidecar when on motrix backend. + + Returns a new dict — cfg.algo is never mutated. + """ + raw = OmegaConf.to_container(self.cfg.algo, resolve=True) + base: dict[str, Any] = cast(dict[str, Any], raw) if isinstance(raw, dict) else {} + + if not self._is_motrix(): + return base + + algo_motrix = getattr(self.cfg, "algo_motrix", None) + if algo_motrix is None: + return base + + if OmegaConf.is_config(algo_motrix): + raw_overrides = OmegaConf.to_container(algo_motrix, resolve=True) + elif isinstance(algo_motrix, dict): + raw_overrides = algo_motrix + else: + return base + + overrides: dict[str, Any] = ( + cast(dict[str, Any], raw_overrides) if isinstance(raw_overrides, dict) else {} + ) + if not overrides: + return base + + # Check applies_to filter (e.g., algo_motrix only for sac, not td3) + applies_to = overrides.pop("applies_to", None) + if applies_to and isinstance(applies_to, dict): + target_algo = applies_to.get("algo") + if ( + self.algo_name is not None + and target_algo is not None + and target_algo != self.algo_name + ): + return base + + # Merge overrides into base, respecting CLI explicit keys + self._merge_overrides(base, overrides, base_path="algo") + return base + def build_task_env_cfg_override(self) -> dict[str, Any]: - """Build env_cfg_override for training/play entrypoints.""" + """Build env_cfg_override for training/play entrypoints. + + Combines reward config (via reward_motrix) with backend_profile env overrides. + Does NOT touch algo config — use resolve_algo_dict() for that. + """ env_cfg_override = extract_reward_config(self.cfg) - motrix_legacy = getattr(self.cfg, "motrix_legacy", None) - if ( - motrix_legacy is None - or not getattr(motrix_legacy, "enabled", False) - or self.cfg.training.sim_backend != "motrix" - ): + + if not self._is_motrix(): return env_cfg_override - applies_to = getattr(motrix_legacy, "applies_to", None) - target_algo = getattr(applies_to, "algo", None) if applies_to is not None else None - if self.algo_name is not None and target_algo is not None and target_algo != self.algo_name: + profile = getattr(self.cfg, "backend_profile", None) + if profile is None: return env_cfg_override - algo_overrides = getattr(motrix_legacy, "algo_overrides", None) - if algo_overrides is not None: - self._apply_nested_overrides( - self.cfg.algo, - algo_overrides, - base_path="algo", - ) + if OmegaConf.is_config(profile): + profile_dict = OmegaConf.to_container(profile, resolve=True) + elif isinstance(profile, dict): + profile_dict = profile + else: + return env_cfg_override - env_profile = getattr(motrix_legacy, "env_cfg_override", None) - if env_profile is not None: - self._apply_env_profile(env_cfg_override, env_profile) + if not isinstance(profile_dict, dict): + return env_cfg_override + + reward_explicit = any( + key == "reward" + or key.startswith("reward.") + or key.startswith("reward@") + or key.startswith("reward_") + for key in self.explicit_keys + ) + for key, value in profile_dict.items(): + key_str = str(key) + if key_str == "reward_config" and reward_explicit: + continue + env_cfg_override[key_str] = value return env_cfg_override @@ -68,7 +124,7 @@ def build_play_env_cfg_override(self) -> dict[str, Any]: if ( play_profile is None or not getattr(play_profile, "enabled", False) - or self.cfg.training.sim_backend != "motrix" + or not self._is_motrix() or not self.cfg.training.play_only ): return env_cfg_override @@ -106,6 +162,17 @@ def build_play_env_cfg_override(self) -> dict[str, Any]: ) return env_cfg_override + def _merge_overrides(self, target: dict, overrides: dict, *, base_path: str) -> None: + """Recursively merge overrides into target dict, skipping CLI-explicit keys.""" + for key, value in overrides.items(): + key_str = str(key) + path = f"{base_path}.{key_str}" + if isinstance(value, dict) and isinstance(target.get(key_str), dict): + if path not in self.explicit_keys: + self._merge_overrides(target[key_str], value, base_path=path) + elif path not in self.explicit_keys: + target[key_str] = value + def _apply_nested_overrides(self, target: Any, overrides: Any, *, base_path: str) -> None: if OmegaConf.is_config(overrides): override_dict = OmegaConf.to_container(overrides, resolve=True) diff --git a/tests/scripts/test_train_scripts.py b/tests/scripts/test_train_scripts.py index d3c4ad76c..cc8bd773c 100644 --- a/tests/scripts/test_train_scripts.py +++ b/tests/scripts/test_train_scripts.py @@ -150,38 +150,42 @@ def test_offpolicy_hydra_algo_td3(): assert cfg.algo.algo == "td3" -def test_offpolicy_task_go1_exposes_motrix_legacy(): +def test_offpolicy_task_go1_exposes_algo_motrix(): cfg = _offpolicy_cfg(["task=go1_joystick"]) - assert cfg.motrix_legacy.enabled is True - assert cfg.motrix_legacy.applies_to.algo == "sac" - assert cfg.motrix_legacy.algo_overrides.num_envs == 4096 - assert cfg.motrix_legacy.algo_overrides.max_iterations == 2000 - assert cfg.motrix_legacy.env_cfg_override.legacy_motrix_profile.enabled is True + assert cfg.algo_motrix.applies_to.algo == "sac" + assert cfg.algo_motrix.num_envs == 4096 + assert cfg.algo_motrix.max_iterations == 2000 -def test_offpolicy_build_task_motrix_env_cfg_override_applies_go1_legacy(): +def test_offpolicy_build_env_cfg_override_and_resolve_algo_go1(): cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) - env_cfg_override = _offpolicy().build_task_motrix_offpolicy_env_cfg_override("sac", cfg) + env_cfg_override = _offpolicy().build_offpolicy_env_cfg_override("sac", cfg) + resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - assert cfg.algo.num_envs == 4096 - assert cfg.algo.max_iterations == 2000 - assert env_cfg_override["legacy_motrix_profile"]["enabled"] is True + # resolved algo merges algo_motrix overrides + assert resolved_algo["num_envs"] == 4096 + assert resolved_algo["max_iterations"] == 2000 + # cfg.algo is NOT mutated + assert cfg.algo.num_envs == 2048 + assert cfg.algo.max_iterations == 3000 + # env_cfg_override has reward + backend_profile fields assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) + assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_offpolicy_build_task_motrix_env_cfg_override_skips_td3(): +def test_offpolicy_resolve_algo_skips_td3(): cfg = _offpolicy_cfg(["algo=td3", "task=go1_joystick", "training.sim_backend=motrix"]) - env_cfg_override = _offpolicy().build_task_motrix_offpolicy_env_cfg_override("td3", cfg) + resolved_algo = _offpolicy().resolve_offpolicy_algo("td3", cfg) - assert cfg.algo.num_envs == 2048 - assert cfg.algo.max_iterations == 3000 - assert "legacy_motrix_profile" not in env_cfg_override + # algo_motrix go1 has applies_to.algo=sac, so td3 should NOT get those overrides + assert resolved_algo["num_envs"] == 2048 + assert resolved_algo["max_iterations"] == 3000 -def test_offpolicy_build_task_motrix_env_cfg_override_respects_cli_algo_override( +def test_offpolicy_resolve_algo_respects_cli_algo_override( monkeypatch: pytest.MonkeyPatch, ): cfg = _offpolicy_cfg( @@ -189,42 +193,44 @@ def test_offpolicy_build_task_motrix_env_cfg_override_respects_cli_algo_override ) monkeypatch.setattr(sys, "argv", ["train_offpolicy.py", "algo.max_iterations=1"]) - _offpolicy().build_task_motrix_offpolicy_env_cfg_override("sac", cfg) + resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - assert cfg.algo.num_envs == 4096 - assert cfg.algo.max_iterations == 1 + assert resolved_algo["num_envs"] == 4096 + assert resolved_algo["max_iterations"] == 1 def test_offpolicy_resolve_sac_use_symmetry_keeps_mujoco_setting(): cfg = _offpolicy_cfg(["task=g1_sac", "training.sim_backend=mujoco"]) + resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - assert _offpolicy().resolve_sac_use_symmetry(cfg) is True + assert _offpolicy().resolve_sac_use_symmetry(resolved_algo, cfg) is True def test_offpolicy_resolve_sac_use_symmetry_disables_motrix(): cfg = _offpolicy_cfg(["task=g1_sac", "training.sim_backend=motrix"]) + resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - assert _offpolicy().resolve_sac_use_symmetry(cfg) is False + assert _offpolicy().resolve_sac_use_symmetry(resolved_algo, cfg) is False -def test_ppo_task_go1_exposes_motrix_legacy(): +def test_ppo_task_go1_exposes_algo_motrix(): cfg = _ppo_cfg(["task=go1_joystick"]) - assert cfg.motrix_legacy.enabled is True - assert cfg.motrix_legacy.algo_overrides.max_iterations == 151 - assert cfg.motrix_legacy.algo_overrides.policy.init_noise_std == pytest.approx(0.5) - assert cfg.motrix_legacy.env_cfg_override.legacy_motrix_profile.enabled is True + assert cfg.algo_motrix.empirical_normalization is True + assert cfg.algo_motrix.policy.init_noise_std == pytest.approx(0.5) + assert cfg.algo_motrix.algorithm.learning_rate == pytest.approx(3.0e-4) + assert cfg.algo_motrix.algorithm.entropy_coef == pytest.approx(1.0e-3) -def test_ppo_task_g1_exposes_motrix_legacy(): +def test_ppo_task_g1_exposes_algo_motrix_and_backend_profile(): cfg = _ppo_cfg(["task=g1_joystick"]) - assert cfg.motrix_legacy.enabled is True - assert cfg.motrix_legacy.algo_overrides.obs_groups.actor == ["policy"] - assert ( - cfg.motrix_legacy.env_cfg_override.backend_overrides.control_action_scale - == pytest.approx(0.5) - ) + assert cfg.algo_motrix.empirical_normalization is True + assert cfg.algo_motrix.obs_groups.actor == ["policy"] + assert cfg.algo_motrix.algorithm.entropy_coef == pytest.approx(5.0e-3) + assert cfg.backend_profile.control_config.action_scale == pytest.approx(0.5) + assert cfg.backend_profile.gait_phase_init_mode == "independent" + assert cfg.backend_profile.reset_base_qvel_limit == pytest.approx(0.05) def test_ppo_task_go2_aligns_mujoco_with_motrix_defaults(): @@ -241,69 +247,78 @@ def test_ppo_task_go2_aligns_mujoco_with_motrix_defaults(): assert cfg.algo.algorithm.entropy_coef == pytest.approx(1.0e-3) -def test_build_task_motrix_ppo_env_cfg_override_applies_go1_legacy( +def test_build_ppo_env_cfg_override_and_resolve_algo_go1( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) - env_cfg_override = mod.build_task_motrix_ppo_env_cfg_override(cfg) - - assert cfg.algo.max_iterations == 151 - assert cfg.algo.empirical_normalization is True - assert cfg.algo.policy.init_noise_std == pytest.approx(0.5) - assert cfg.algo.algorithm.learning_rate == pytest.approx(3.0e-4) - assert cfg.algo.algorithm.entropy_coef == pytest.approx(1.0e-3) - assert env_cfg_override["legacy_motrix_profile"]["enabled"] is True + env_cfg_override = mod.build_ppo_env_cfg_override(cfg) + resolved_algo = mod.resolve_ppo_algo(cfg) + + # resolved algo merges algo_motrix overrides + assert resolved_algo["empirical_normalization"] is True + assert resolved_algo["policy"]["init_noise_std"] == pytest.approx(0.5) + assert resolved_algo["algorithm"]["learning_rate"] == pytest.approx(3.0e-4) + assert resolved_algo["algorithm"]["entropy_coef"] == pytest.approx(1.0e-3) + # cfg.algo is NOT mutated + assert cfg.algo.empirical_normalization is False + # env_cfg_override has reward + backend_profile commands assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) + assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_build_task_motrix_ppo_env_cfg_override_applies_g1_legacy( +def test_build_ppo_env_cfg_override_and_resolve_algo_g1( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=g1_joystick", "training.sim_backend=motrix"]) - env_cfg_override = mod.build_task_motrix_ppo_env_cfg_override(cfg) - - assert cfg.algo.max_iterations == 151 - assert cfg.algo.empirical_normalization is True - assert cfg.algo.policy.init_noise_std == pytest.approx(0.5) - assert cfg.algo.algorithm.learning_rate == pytest.approx(3.0e-4) - assert cfg.algo.algorithm.entropy_coef == pytest.approx(5.0e-3) - assert cfg.algo.obs_groups.actor == ["policy"] - assert env_cfg_override["backend_overrides"]["enabled"] is True + env_cfg_override = mod.build_ppo_env_cfg_override(cfg) + resolved_algo = mod.resolve_ppo_algo(cfg) + + # resolved algo merges algo_motrix overrides + assert resolved_algo["empirical_normalization"] is True + assert resolved_algo["policy"]["init_noise_std"] == pytest.approx(0.5) + assert resolved_algo["algorithm"]["learning_rate"] == pytest.approx(3.0e-4) + assert resolved_algo["algorithm"]["entropy_coef"] == pytest.approx(5.0e-3) + assert resolved_algo["obs_groups"]["actor"] == ["policy"] + # cfg.algo is NOT mutated + assert cfg.algo.empirical_normalization is False + # env_cfg_override has reward + backend_profile fields assert env_cfg_override["reward_config"]["scales"]["upper_body_pose"] == pytest.approx(-0.05) + assert env_cfg_override["control_config"]["action_scale"] == pytest.approx(0.5) + assert env_cfg_override["gait_phase_init_mode"] == "independent" + assert env_cfg_override["reset_base_qvel_limit"] == pytest.approx(0.05) -def test_build_task_motrix_ppo_env_cfg_override_applies_go2_reward_motrix( +def test_build_ppo_env_cfg_override_applies_go2_reward_motrix( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=go2_joystick", "training.sim_backend=motrix"]) - env_cfg_override = mod.build_task_motrix_ppo_env_cfg_override(cfg) + env_cfg_override = mod.build_ppo_env_cfg_override(cfg) assert cfg.reward_motrix.reward.scales.tracking_lin_vel == pytest.approx(1.0) assert cfg.algo.num_envs == 1024 - assert env_cfg_override["legacy_motrix_profile"]["enabled"] is True assert env_cfg_override["domain_rand"]["randomize_kp"] is False assert env_cfg_override["domain_rand"]["randomize_kd"] is False assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["reward_config"]["scales"]["tracking_ang_vel"] == pytest.approx(0.2) -def test_build_task_motrix_ppo_env_cfg_override_respects_cli_algo_override( +def test_resolve_ppo_algo_respects_cli_algo_override( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=g1_joystick", "training.sim_backend=motrix", "algo.max_iterations=1"]) monkeypatch.setattr(sys, "argv", ["train_rsl_rl.py", "algo.max_iterations=1"]) - mod.build_task_motrix_ppo_env_cfg_override(cfg) + resolved_algo = mod.resolve_ppo_algo(cfg) - assert cfg.algo.empirical_normalization is True - assert cfg.algo.max_iterations == 1 + assert resolved_algo["empirical_normalization"] is True + assert resolved_algo["max_iterations"] == 1 def test_g1_motion_tracking_ppo_motrix_prefers_backend_specific_reward( @@ -318,12 +333,12 @@ def test_g1_motion_tracking_ppo_motrix_prefers_backend_specific_reward( cfg.reward.scales.motion_body_pos = 1.25 cfg.reward_motrix.scales.motion_body_pos = 2.5 - env_cfg_override = mod.build_task_motrix_ppo_env_cfg_override(cfg) + env_cfg_override = mod.build_ppo_env_cfg_override(cfg) assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(2.5) -def test_build_motrix_play_ppo_env_cfg_override_applies_g1_motion_tracking_play_profile( +def test_build_ppo_play_env_cfg_override_applies_g1_motion_tracking_play_profile( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) @@ -337,7 +352,7 @@ def test_build_motrix_play_ppo_env_cfg_override_applies_g1_motion_tracking_play_ lambda source_model_file, **kwargs: "/tmp/g1_motion_tracking_play_scene.xml", ) - env_cfg_override = mod.build_motrix_play_ppo_env_cfg_override(cfg) + env_cfg_override = mod.build_ppo_play_env_cfg_override(cfg) assert cfg.training.play_env_num == 128 assert env_cfg_override["render_spacing"] == pytest.approx(2.5) @@ -345,7 +360,7 @@ def test_build_motrix_play_ppo_env_cfg_override_applies_g1_motion_tracking_play_ assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(1.0) -def test_build_motrix_play_ppo_env_cfg_override_respects_cli_play_env_override( +def test_build_ppo_play_env_cfg_override_respects_cli_play_env_override( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) @@ -364,13 +379,13 @@ def test_build_motrix_play_ppo_env_cfg_override_respects_cli_play_env_override( lambda source_model_file, **kwargs: "/tmp/g1_motion_tracking_play_scene.xml", ) - env_cfg_override = mod.build_motrix_play_ppo_env_cfg_override(cfg) + env_cfg_override = mod.build_ppo_play_env_cfg_override(cfg) assert cfg.training.play_env_num == 32 assert env_cfg_override["render_spacing"] == pytest.approx(2.5) -def test_build_motrix_play_ppo_env_cfg_override_resolves_relative_ground_texture( +def test_build_ppo_play_env_cfg_override_resolves_relative_ground_texture( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) @@ -390,7 +405,7 @@ def _fake_materialize(source_model_file, **kwargs): monkeypatch.setattr(mod, "materialize_scene_visual_override", _fake_materialize) - mod.build_motrix_play_ppo_env_cfg_override(cfg) + mod.build_ppo_play_env_cfg_override(cfg) assert captured["ground_texture_file"] == str( mod.ROOT_DIR / "src/unilab/assets/robots/g1/floor.png" diff --git a/tests/training/test_training_helpers.py b/tests/training/test_training_helpers.py index cf1286e24..c9aba162d 100644 --- a/tests/training/test_training_helpers.py +++ b/tests/training/test_training_helpers.py @@ -63,17 +63,22 @@ def test_parse_checkpoint_path_uses_algo_log_name_from_cfg(tmp_path: Path): assert checkpoint_dir == run_dir -def test_backend_adapter_applies_motrix_legacy_for_matching_algo(): +def test_backend_adapter_resolve_algo_and_env_override_for_matching_algo(): cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) - env_cfg_override = BackendAdapter( - cfg, root_dir=_ROOT_DIR, algo_name="sac" - ).build_task_env_cfg_override() - - assert cfg.algo.num_envs == 4096 - assert cfg.algo.max_iterations == 2000 - assert env_cfg_override["legacy_motrix_profile"]["enabled"] is True + adapter = BackendAdapter(cfg, root_dir=_ROOT_DIR, algo_name="sac") + resolved_algo = adapter.resolve_algo_dict() + env_cfg_override = adapter.build_task_env_cfg_override() + + # resolved algo merges algo_motrix overrides + assert resolved_algo["num_envs"] == 4096 + assert resolved_algo["max_iterations"] == 2000 + # cfg.algo is NOT mutated + assert cfg.algo.num_envs == 2048 + assert cfg.algo.max_iterations == 3000 + # env_cfg_override has reward + backend_profile fields assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) + assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] def test_backend_adapter_builds_motrix_play_scene_override(): From dccdcc99648d34fe9624a202aac04a2a6d5e7b1e Mon Sep 17 00:00:00 2001 From: TATP-233 Date: Fri, 10 Apr 2026 17:55:02 +0800 Subject: [PATCH 2/5] fixup: remove algo_motrix sidecar, fold algo into base (#177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback on rev 2: - Delete conf/{ppo,offpolicy}/algo_motrix/ directories entirely - Delete BackendAdapter.resolve_algo_dict() — training code no longer interprets backend-specific algo config; cfg.algo comes straight from Hydra compose - Fold all old motrix_legacy.algo_overrides into the base algo section of each task YAML (Go2 pattern), preserving the exact pre-refactor resolved hyperparams for the Motrix path: * Go1 PPO: empirical_normalization, policy, algorithm overrides * G1 PPO: max_iterations=151 (was drifted to 220), empirical_norm, obs_groups.actor, policy, algorithm overrides * Go1 offpolicy SAC: num_envs=4096, max_iterations=2000 * G1 SAC: use_symmetry=true (resolve_sac_use_symmetry in script still disables it for motrix at the one-liner level) - Revert training scripts to read cfg.algo directly - Rewrite tests to assert resolved hyperparam equivalence with the old Motrix behavior, not the internal bag structure - Update docs to remove algo_motrix references --- AGENTS.md | 2 +- conf/offpolicy/algo_motrix/g1_sac.yaml | 1 - conf/offpolicy/algo_motrix/go1.yaml | 4 - conf/offpolicy/task/g1_sac.yaml | 1 - conf/offpolicy/task/go1_joystick.yaml | 5 +- conf/ppo/algo_motrix/g1.yaml | 9 -- conf/ppo/algo_motrix/go1.yaml | 6 - conf/ppo/task/g1_joystick.yaml | 11 +- conf/ppo/task/go1_joystick.yaml | 7 +- docs/zh_CN/00-development-architecture.md | 2 +- scripts/train_offpolicy.py | 168 ++++++++++------------ scripts/train_rsl_rl.py | 16 +-- src/unilab/training/backend_adapter.py | 60 +------- tests/config/test_locomotion_params.py | 7 +- tests/scripts/test_train_scripts.py | 127 ++++++---------- tests/training/test_training_helpers.py | 15 +- 16 files changed, 159 insertions(+), 282 deletions(-) delete mode 100644 conf/offpolicy/algo_motrix/g1_sac.yaml delete mode 100644 conf/offpolicy/algo_motrix/go1.yaml delete mode 100644 conf/ppo/algo_motrix/g1.yaml delete mode 100644 conf/ppo/algo_motrix/go1.yaml diff --git a/AGENTS.md b/AGENTS.md index 663746779..87622faca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ UniLab 是一个 **高性能、模块化、contract 驱动** 的 RL infrastructu | 区域 | 不可破坏的不变量 | |------|----------------| | Env | `NpEnvState.obs` 必须是 dict；`reset()` 返回 `(obs_dict, info_dict)`；`obs_groups_spec` 影响 wrapper 和 learner 维度。 | -| Config / Reward | reward 通过 Hydra 注入；`training.sim_backend`、`algo_motrix`、`backend_profile` 必须尊重显式 override。 | +| Config / Reward | reward 通过 Hydra 注入；`training.sim_backend` 和 `backend_profile` 必须尊重显式 override。算法超参数直接走 YAML compose，不经 Python 层解释。 | | Backend | backend-specific 逻辑留在 backend / env 适配层，不向训练脚本扩散。 | | Async | 不绕开 runner lifecycle，也不另起 collector / learner 同步协议。 | diff --git a/conf/offpolicy/algo_motrix/g1_sac.yaml b/conf/offpolicy/algo_motrix/g1_sac.yaml deleted file mode 100644 index a7fb3d245..000000000 --- a/conf/offpolicy/algo_motrix/g1_sac.yaml +++ /dev/null @@ -1 +0,0 @@ -use_symmetry: false diff --git a/conf/offpolicy/algo_motrix/go1.yaml b/conf/offpolicy/algo_motrix/go1.yaml deleted file mode 100644 index 24bad6ca7..000000000 --- a/conf/offpolicy/algo_motrix/go1.yaml +++ /dev/null @@ -1,4 +0,0 @@ -applies_to: - algo: sac -num_envs: 4096 -max_iterations: 2000 diff --git a/conf/offpolicy/task/g1_sac.yaml b/conf/offpolicy/task/g1_sac.yaml index 439ec3b63..c4dfd4522 100644 --- a/conf/offpolicy/task/g1_sac.yaml +++ b/conf/offpolicy/task/g1_sac.yaml @@ -1,6 +1,5 @@ # @package _global_ defaults: - - /algo_motrix@algo_motrix: g1_sac - override /reward: g1_sac_mujoco training: diff --git a/conf/offpolicy/task/go1_joystick.yaml b/conf/offpolicy/task/go1_joystick.yaml index df48cc482..a5478793b 100644 --- a/conf/offpolicy/task/go1_joystick.yaml +++ b/conf/offpolicy/task/go1_joystick.yaml @@ -1,14 +1,13 @@ # @package _global_ defaults: - /reward@reward_motrix: go1_sac_motrix - - /algo_motrix@algo_motrix: go1 - override /reward: go1_sac_mujoco training: task_name: Go1JoystickFlatTerrain algo: - num_envs: 2048 - max_iterations: 3000 + num_envs: 4096 + max_iterations: 2000 backend_profile: commands: vel_limit: diff --git a/conf/ppo/algo_motrix/g1.yaml b/conf/ppo/algo_motrix/g1.yaml deleted file mode 100644 index e3d75021f..000000000 --- a/conf/ppo/algo_motrix/g1.yaml +++ /dev/null @@ -1,9 +0,0 @@ -empirical_normalization: true -obs_groups: - actor: - - policy -policy: - init_noise_std: 0.5 -algorithm: - learning_rate: 3.0e-4 - entropy_coef: 5.0e-3 diff --git a/conf/ppo/algo_motrix/go1.yaml b/conf/ppo/algo_motrix/go1.yaml deleted file mode 100644 index 2e8587a7c..000000000 --- a/conf/ppo/algo_motrix/go1.yaml +++ /dev/null @@ -1,6 +0,0 @@ -empirical_normalization: true -policy: - init_noise_std: 0.5 -algorithm: - learning_rate: 3.0e-4 - entropy_coef: 1.0e-3 diff --git a/conf/ppo/task/g1_joystick.yaml b/conf/ppo/task/g1_joystick.yaml index 3683ebb10..f0a6be33f 100644 --- a/conf/ppo/task/g1_joystick.yaml +++ b/conf/ppo/task/g1_joystick.yaml @@ -1,19 +1,24 @@ # @package _global_ defaults: - /reward@reward_motrix: g1_ppo_motrix - - /algo_motrix@algo_motrix: g1 - override /reward: g1_ppo_mujoco training: task_name: G1JoystickFlatTerrain algo: num_envs: 2048 - max_iterations: 220 + max_iterations: 151 + empirical_normalization: true obs_groups: default: - policy actor: - - actor + - policy + policy: + init_noise_std: 0.5 + algorithm: + learning_rate: 3.0e-4 + entropy_coef: 5.0e-3 backend_profile: control_config: action_scale: 0.5 diff --git a/conf/ppo/task/go1_joystick.yaml b/conf/ppo/task/go1_joystick.yaml index b6e585511..6b9cf6c36 100644 --- a/conf/ppo/task/go1_joystick.yaml +++ b/conf/ppo/task/go1_joystick.yaml @@ -1,18 +1,23 @@ # @package _global_ defaults: - /reward@reward_motrix: go1_ppo_motrix - - /algo_motrix@algo_motrix: go1 - override /reward: go1_ppo_mujoco training: task_name: Go1JoystickFlatTerrain algo: max_iterations: 151 + empirical_normalization: true obs_groups: default: - policy actor: - actor + policy: + init_noise_std: 0.5 + algorithm: + learning_rate: 3.0e-4 + entropy_coef: 1.0e-3 backend_profile: commands: vel_limit: diff --git a/docs/zh_CN/00-development-architecture.md b/docs/zh_CN/00-development-architecture.md index 50c368a3e..e7b7a2937 100644 --- a/docs/zh_CN/00-development-architecture.md +++ b/docs/zh_CN/00-development-architecture.md @@ -68,7 +68,7 @@ CPU Physics Sim ──shm──► Collector / IPC ──shm──► GPU Learne UniLab 使用 dataclass + Hydra。schema 位于 `src/unilab/config/structured_configs.py`，运行时配置位于 `conf/{ppo,appo,offpolicy}/`。 -合成顺序: `{algo}/config*.yaml` -> `task=...` -> `reward[_{backend}]` -> `algo_motrix` (backend sidecar) -> CLI override -> `backend_profile` (env overrides)。 +合成顺序: `{algo}/config*.yaml` -> `task=...` -> `reward[_{backend}]` -> CLI override -> `backend_profile` (env-only overrides，motrix 时由 BackendAdapter 应用)。 - reward 必须显式注入 - 如果 backend 选择会影响 task 或 reward 行为，就必须通过 config 表达 diff --git a/scripts/train_offpolicy.py b/scripts/train_offpolicy.py index 4f5622c7d..8af6e9f69 100644 --- a/scripts/train_offpolicy.py +++ b/scripts/train_offpolicy.py @@ -74,29 +74,18 @@ def extract_play_obs(obs_dict): return obs_out -def _backend_adapter(algo_name: str, cfg: DictConfig) -> BackendAdapter: - return BackendAdapter(cfg, root_dir=ROOT_DIR, algo_name=algo_name) - - def build_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict | None: - return _backend_adapter(algo_name, cfg).build_task_env_cfg_override() - - -def resolve_offpolicy_algo(algo_name: str, cfg: DictConfig) -> dict: - return _backend_adapter(algo_name, cfg).resolve_algo_dict() + return BackendAdapter(cfg, root_dir=ROOT_DIR, algo_name=algo_name).build_task_env_cfg_override() -def resolve_sac_use_symmetry(resolved_algo: dict, cfg: DictConfig) -> bool: +def resolve_sac_use_symmetry(cfg: DictConfig) -> bool: """Symmetry augmentation currently depends on MuJoCo model APIs.""" - return bool(resolved_algo.get("use_symmetry", False) and cfg.training.sim_backend == "mujoco") + return bool(cfg.algo.use_symmetry and cfg.training.sim_backend == "mujoco") def build_runner(algo_name: str, cfg: DictConfig): """Build algorithm runner from unified Hydra config.""" - adapter = _backend_adapter(algo_name, cfg) - env_cfg_override = adapter.build_task_env_cfg_override() - algo = adapter.resolve_algo_dict() - algo_params = algo.get("algo_params", {}) + env_cfg_override = build_offpolicy_env_cfg_override(algo_name, cfg) if algo_name == "sac": from unilab.algos.torch.fast_sac.learner import FastSACLearner @@ -124,18 +113,18 @@ def build_runner(algo_name: str, cfg: DictConfig): learner_kwargs = { "obs_dim": obs_dim, "action_dim": action_dim, - "gamma": algo["gamma"], - "tau": algo["tau"], - "actor_lr": algo["actor_lr"], - "critic_lr": algo["critic_lr"], - "alpha_lr": algo_params["alpha_lr"], - "alpha_init": algo_params["alpha_init"], - "target_entropy_ratio": algo_params["target_entropy_ratio"], - "actor_hidden_dim": algo["actor_hidden_dim"], - "critic_hidden_dim": algo["critic_hidden_dim"], - "num_atoms": algo["num_atoms"], - "use_layer_norm": algo["use_layer_norm"], - "max_grad_norm": algo_params["max_grad_norm"], + "gamma": cfg.algo.gamma, + "tau": cfg.algo.tau, + "actor_lr": cfg.algo.actor_lr, + "critic_lr": cfg.algo.critic_lr, + "alpha_lr": cfg.algo.algo_params.alpha_lr, + "alpha_init": cfg.algo.algo_params.alpha_init, + "target_entropy_ratio": cfg.algo.algo_params.target_entropy_ratio, + "actor_hidden_dim": cfg.algo.actor_hidden_dim, + "critic_hidden_dim": cfg.algo.critic_hidden_dim, + "num_atoms": cfg.algo.num_atoms, + "use_layer_norm": cfg.algo.use_layer_norm, + "max_grad_norm": cfg.algo.algo_params.max_grad_norm, "use_amp": cfg.training.use_amp, "privileged_dim": privileged_dim, # symmetry not supported in multi-GPU mode (mujoco_model not picklable) @@ -149,17 +138,17 @@ def build_runner(algo_name: str, cfg: DictConfig): algo_type="sac", learner_kwargs=learner_kwargs, num_gpus=cfg.training.num_gpus, - num_envs=algo["num_envs"], - replay_buffer_n=algo["replay_buffer_n"], - batch_size=algo["batch_size"], - warmup_steps=algo["warmup_steps"], - updates_per_step=algo["updates_per_step"], - policy_frequency=algo["policy_frequency"], + num_envs=cfg.algo.num_envs, + replay_buffer_n=cfg.algo.replay_buffer_n, + batch_size=cfg.algo.batch_size, + warmup_steps=cfg.algo.warmup_steps, + updates_per_step=cfg.algo.updates_per_step, + policy_frequency=cfg.algo.policy_frequency, sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, device=device, - actor_hidden_dim=algo["actor_hidden_dim"], - use_layer_norm=algo["use_layer_norm"], + actor_hidden_dim=cfg.algo.actor_hidden_dim, + use_layer_norm=cfg.algo.use_layer_norm, obs_normalization=False, sim_backend=cfg.training.sim_backend, env_cfg_override=env_cfg_override, @@ -169,30 +158,30 @@ def build_runner(algo_name: str, cfg: DictConfig): env_name=cfg.training.task_name, env_cfg_override=env_cfg_override, device=cfg.training.device, - num_envs=algo["num_envs"], - replay_buffer_n=algo["replay_buffer_n"], - batch_size=algo["batch_size"], - warmup_steps=algo["warmup_steps"], - updates_per_step=algo["updates_per_step"], - policy_frequency=algo["policy_frequency"], - gamma=algo["gamma"], - tau=algo["tau"], - actor_lr=algo["actor_lr"], - critic_lr=algo["critic_lr"], - alpha_lr=algo_params["alpha_lr"], - alpha_init=algo_params["alpha_init"], - target_entropy_ratio=algo_params["target_entropy_ratio"], - obs_normalization=algo["obs_normalization"], - actor_hidden_dim=algo["actor_hidden_dim"], - critic_hidden_dim=algo["critic_hidden_dim"], - num_atoms=algo["num_atoms"], - use_layer_norm=algo["use_layer_norm"], - max_grad_norm=algo_params["max_grad_norm"], + num_envs=cfg.algo.num_envs, + replay_buffer_n=cfg.algo.replay_buffer_n, + batch_size=cfg.algo.batch_size, + warmup_steps=cfg.algo.warmup_steps, + updates_per_step=cfg.algo.updates_per_step, + policy_frequency=cfg.algo.policy_frequency, + gamma=cfg.algo.gamma, + tau=cfg.algo.tau, + actor_lr=cfg.algo.actor_lr, + critic_lr=cfg.algo.critic_lr, + alpha_lr=cfg.algo.algo_params.alpha_lr, + alpha_init=cfg.algo.algo_params.alpha_init, + target_entropy_ratio=cfg.algo.algo_params.target_entropy_ratio, + obs_normalization=cfg.algo.obs_normalization, + actor_hidden_dim=cfg.algo.actor_hidden_dim, + critic_hidden_dim=cfg.algo.critic_hidden_dim, + num_atoms=cfg.algo.num_atoms, + use_layer_norm=cfg.algo.use_layer_norm, + max_grad_norm=cfg.algo.algo_params.max_grad_norm, use_amp=cfg.training.use_amp, sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, sim_backend=cfg.training.sim_backend, - use_symmetry=resolve_sac_use_symmetry(algo, cfg), + use_symmetry=resolve_sac_use_symmetry(cfg), ) if algo_name == "td3": @@ -202,31 +191,31 @@ def build_runner(algo_name: str, cfg: DictConfig): env_name=cfg.training.task_name, env_cfg_override=env_cfg_override, device=cfg.training.device, - num_envs=algo["num_envs"], - replay_buffer_n=algo["replay_buffer_n"], - batch_size=algo["batch_size"], - warmup_steps=algo["warmup_steps"], - num_updates=algo["updates_per_step"], - policy_frequency=algo["policy_frequency"], + num_envs=cfg.algo.num_envs, + replay_buffer_n=cfg.algo.replay_buffer_n, + batch_size=cfg.algo.batch_size, + warmup_steps=cfg.algo.warmup_steps, + num_updates=cfg.algo.updates_per_step, + policy_frequency=cfg.algo.policy_frequency, sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, - gamma=algo["gamma"], - tau=algo["tau"], - actor_lr=algo["actor_lr"], - critic_lr=algo["critic_lr"], - actor_hidden_dim=algo["actor_hidden_dim"], - critic_hidden_dim=algo["critic_hidden_dim"], - num_atoms=algo["num_atoms"], - v_min=algo_params["v_min"], - v_max=algo_params["v_max"], - init_scale=algo_params["init_scale"], - log_std_min=algo_params["log_std_min"], - log_std_max=algo_params["log_std_max"], - policy_noise=algo_params["policy_noise"], - noise_clip=algo_params["noise_clip"], - weight_decay=algo_params["weight_decay"], - use_cdq=algo_params["use_cdq"], - obs_normalization=algo["obs_normalization"], + gamma=cfg.algo.gamma, + tau=cfg.algo.tau, + actor_lr=cfg.algo.actor_lr, + critic_lr=cfg.algo.critic_lr, + actor_hidden_dim=cfg.algo.actor_hidden_dim, + critic_hidden_dim=cfg.algo.critic_hidden_dim, + num_atoms=cfg.algo.num_atoms, + v_min=cfg.algo.algo_params.v_min, + v_max=cfg.algo.algo_params.v_max, + init_scale=cfg.algo.algo_params.init_scale, + log_std_min=cfg.algo.algo_params.log_std_min, + log_std_max=cfg.algo.algo_params.log_std_max, + policy_noise=cfg.algo.algo_params.policy_noise, + noise_clip=cfg.algo.algo_params.noise_clip, + weight_decay=cfg.algo.algo_params.weight_decay, + use_cdq=cfg.algo.algo_params.use_cdq, + obs_normalization=cfg.algo.obs_normalization, sim_backend=cfg.training.sim_backend, ) @@ -240,9 +229,7 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: from unilab.utils.algo_utils import build_actor - adapter = _backend_adapter(algo_name, cfg) - env_cfg_override = adapter.build_task_env_cfg_override() - algo = adapter.resolve_algo_dict() + env_cfg_override = build_offpolicy_env_cfg_override(algo_name, cfg) device = default_device(torch, cfg.training.device) print(f"Using device for play: {device}") @@ -261,25 +248,24 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: "sac", obs_dim, action_dim, - algo["actor_hidden_dim"], - algo["use_layer_norm"], + cfg.algo.actor_hidden_dim, + cfg.algo.use_layer_norm, device, ) elif algo_name == "td3": from unilab.algos.torch.fast_td3.learner import EmpiricalNormalization, TD3Actor - algo_params = algo.get("algo_params", {}) actor = TD3Actor( obs_dim, action_dim, cfg.training.play_env_num, - algo_params["init_scale"], - algo["actor_hidden_dim"], - algo_params["log_std_min"], - algo_params["log_std_max"], + cfg.algo.algo_params.init_scale, + cfg.algo.actor_hidden_dim, + cfg.algo.algo_params.log_std_min, + cfg.algo.algo_params.log_std_max, device, ) - if algo["obs_normalization"]: + if cfg.algo.obs_normalization: normalizer = EmpiricalNormalization(shape=obs_dim, device=device) else: raise ValueError(f"Unsupported algo: {algo_name}") @@ -288,9 +274,9 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: load_path, load_path_dir = resolve_checkpoint_path( ROOT_DIR, - algo["algo_log_name"], + cfg.algo.algo_log_name, cfg.training.task_name, - algo["load_run"], + cfg.algo.load_run, ) if not load_path or not os.path.exists(load_path): print(f"Could not find checkpoint. load_path={load_path}") diff --git a/scripts/train_rsl_rl.py b/scripts/train_rsl_rl.py index 0f8f51a30..2e2807c41 100644 --- a/scripts/train_rsl_rl.py +++ b/scripts/train_rsl_rl.py @@ -53,10 +53,6 @@ def build_ppo_play_env_cfg_override(cfg: DictConfig) -> dict: return _backend_adapter(cfg).build_play_env_cfg_override() -def resolve_ppo_algo(cfg: DictConfig) -> dict: - return _backend_adapter(cfg).resolve_algo_dict() - - def run_motrix_rsl_play_loop( wrapped_env, policy, @@ -162,9 +158,7 @@ def play_rsl_rl(cfg: DictConfig, device: str) -> str | None: def main(cfg: DictConfig) -> None: ensure_registries() - adapter = _backend_adapter(cfg) - env_cfg_override = adapter.build_task_env_cfg_override() - resolved_algo = adapter.resolve_algo_dict() + env_cfg_override = build_ppo_env_cfg_override(cfg) if torch.cuda.is_available(): device = "cuda" @@ -175,9 +169,9 @@ def main(cfg: DictConfig) -> None: print(f"Using device: {device}") # Compute effective max_iterations (supports num_timesteps override) - max_iterations = resolved_algo["max_iterations"] + max_iterations = cfg.algo.max_iterations if cfg.training.num_timesteps: - n_steps_per_iter = resolved_algo["num_steps_per_env"] * resolved_algo["num_envs"] + n_steps_per_iter = cfg.algo.num_steps_per_env * cfg.algo.num_envs max_iterations = max(1, int(cfg.training.num_timesteps / n_steps_per_iter)) print( f"Overriding max_iterations to {max_iterations} based on " @@ -211,12 +205,12 @@ def main(cfg: DictConfig) -> None: if not cfg.training.play_only: env = create_env( cfg, - num_envs=resolved_algo["num_envs"], + num_envs=cfg.algo.num_envs, env_cfg_override=env_cfg_override, ) wrapped_env = RslRlVecEnvWrapper(env, device=device) - train_cfg = dict(resolved_algo) + train_cfg = OmegaConf.to_container(cfg.algo, resolve=True) if "runner" not in train_cfg: train_cfg["runner"] = {} diff --git a/src/unilab/training/backend_adapter.py b/src/unilab/training/backend_adapter.py index 91669bd1d..63ebdda5a 100644 --- a/src/unilab/training/backend_adapter.py +++ b/src/unilab/training/backend_adapter.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from typing import Any, Callable, cast +from typing import Any, Callable from omegaconf import DictConfig, OmegaConf @@ -34,54 +34,11 @@ def __init__( def _is_motrix(self) -> bool: return bool(self.cfg.training.sim_backend == "motrix") - def resolve_algo_dict(self) -> dict[str, Any]: - """Resolve algo config, merging algo_motrix sidecar when on motrix backend. - - Returns a new dict — cfg.algo is never mutated. - """ - raw = OmegaConf.to_container(self.cfg.algo, resolve=True) - base: dict[str, Any] = cast(dict[str, Any], raw) if isinstance(raw, dict) else {} - - if not self._is_motrix(): - return base - - algo_motrix = getattr(self.cfg, "algo_motrix", None) - if algo_motrix is None: - return base - - if OmegaConf.is_config(algo_motrix): - raw_overrides = OmegaConf.to_container(algo_motrix, resolve=True) - elif isinstance(algo_motrix, dict): - raw_overrides = algo_motrix - else: - return base - - overrides: dict[str, Any] = ( - cast(dict[str, Any], raw_overrides) if isinstance(raw_overrides, dict) else {} - ) - if not overrides: - return base - - # Check applies_to filter (e.g., algo_motrix only for sac, not td3) - applies_to = overrides.pop("applies_to", None) - if applies_to and isinstance(applies_to, dict): - target_algo = applies_to.get("algo") - if ( - self.algo_name is not None - and target_algo is not None - and target_algo != self.algo_name - ): - return base - - # Merge overrides into base, respecting CLI explicit keys - self._merge_overrides(base, overrides, base_path="algo") - return base - def build_task_env_cfg_override(self) -> dict[str, Any]: """Build env_cfg_override for training/play entrypoints. - Combines reward config (via reward_motrix) with backend_profile env overrides. - Does NOT touch algo config — use resolve_algo_dict() for that. + Combines reward config (via reward_motrix sidecar) with backend_profile env overrides. + Algo config is NOT touched — training scripts read cfg.algo directly. """ env_cfg_override = extract_reward_config(self.cfg) @@ -162,17 +119,6 @@ def build_play_env_cfg_override(self) -> dict[str, Any]: ) return env_cfg_override - def _merge_overrides(self, target: dict, overrides: dict, *, base_path: str) -> None: - """Recursively merge overrides into target dict, skipping CLI-explicit keys.""" - for key, value in overrides.items(): - key_str = str(key) - path = f"{base_path}.{key_str}" - if isinstance(value, dict) and isinstance(target.get(key_str), dict): - if path not in self.explicit_keys: - self._merge_overrides(target[key_str], value, base_path=path) - elif path not in self.explicit_keys: - target[key_str] = value - def _apply_nested_overrides(self, target: Any, overrides: Any, *, base_path: str) -> None: if OmegaConf.is_config(overrides): override_dict = OmegaConf.to_container(overrides, resolve=True) diff --git a/tests/config/test_locomotion_params.py b/tests/config/test_locomotion_params.py index 3e0c0799f..defebed49 100644 --- a/tests/config/test_locomotion_params.py +++ b/tests/config/test_locomotion_params.py @@ -79,8 +79,8 @@ def test_offpolicy_sac_defaults(): with initialize_config_dir(config_dir=str(CONF_DIR / "offpolicy"), version_base="1.3"): cfg = compose("config") assert cfg.algo.algo == "sac" - # go1_joystick task sets num_envs=2048 - assert cfg.algo.num_envs == 2048 + # go1_joystick task sets num_envs=4096 (motrix-aligned hyperparams folded into base) + assert cfg.algo.num_envs == 4096 def test_offpolicy_sac_g1_task_overrides(): @@ -176,7 +176,8 @@ def test_ppo_g1_num_envs(): with initialize_config_dir(config_dir=str(CONF_DIR / "ppo"), version_base="1.3"): cfg = compose("config", overrides=["task=g1_joystick"]) assert cfg.algo.num_envs == 2048 - assert cfg.algo.max_iterations == 220 + # max_iterations=151 (motrix-aligned hyperparams folded into base) + assert cfg.algo.max_iterations == 151 def test_ppo_go2_num_envs(): diff --git a/tests/scripts/test_train_scripts.py b/tests/scripts/test_train_scripts.py index cc8bd773c..e75a78fe2 100644 --- a/tests/scripts/test_train_scripts.py +++ b/tests/scripts/test_train_scripts.py @@ -150,84 +150,68 @@ def test_offpolicy_hydra_algo_td3(): assert cfg.algo.algo == "td3" -def test_offpolicy_task_go1_exposes_algo_motrix(): - cfg = _offpolicy_cfg(["task=go1_joystick"]) +def test_offpolicy_go1_resolved_algo_matches_old_motrix_behavior(): + """Equivalence: Motrix SAC Go1 algo hyperparams match pre-refactor values.""" + cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) - assert cfg.algo_motrix.applies_to.algo == "sac" - assert cfg.algo_motrix.num_envs == 4096 - assert cfg.algo_motrix.max_iterations == 2000 + # Old motrix_legacy.algo_overrides for Go1 SAC: num_envs=4096, max_iterations=2000 + assert cfg.algo.num_envs == 4096 + assert cfg.algo.max_iterations == 2000 -def test_offpolicy_build_env_cfg_override_and_resolve_algo_go1(): +def test_offpolicy_go1_env_cfg_override_has_reward_and_commands(): cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) env_cfg_override = _offpolicy().build_offpolicy_env_cfg_override("sac", cfg) - resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - - # resolved algo merges algo_motrix overrides - assert resolved_algo["num_envs"] == 4096 - assert resolved_algo["max_iterations"] == 2000 - # cfg.algo is NOT mutated - assert cfg.algo.num_envs == 2048 - assert cfg.algo.max_iterations == 3000 - # env_cfg_override has reward + backend_profile fields + + # env_cfg_override has reward + backend_profile fields (no bag structure) assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_offpolicy_resolve_algo_skips_td3(): - cfg = _offpolicy_cfg(["algo=td3", "task=go1_joystick", "training.sim_backend=motrix"]) - - resolved_algo = _offpolicy().resolve_offpolicy_algo("td3", cfg) - - # algo_motrix go1 has applies_to.algo=sac, so td3 should NOT get those overrides - assert resolved_algo["num_envs"] == 2048 - assert resolved_algo["max_iterations"] == 3000 - - -def test_offpolicy_resolve_algo_respects_cli_algo_override( - monkeypatch: pytest.MonkeyPatch, -): - cfg = _offpolicy_cfg( - ["task=go1_joystick", "training.sim_backend=motrix", "algo.max_iterations=1"] - ) - monkeypatch.setattr(sys, "argv", ["train_offpolicy.py", "algo.max_iterations=1"]) - - resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - - assert resolved_algo["num_envs"] == 4096 - assert resolved_algo["max_iterations"] == 1 - - def test_offpolicy_resolve_sac_use_symmetry_keeps_mujoco_setting(): cfg = _offpolicy_cfg(["task=g1_sac", "training.sim_backend=mujoco"]) - resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - assert _offpolicy().resolve_sac_use_symmetry(resolved_algo, cfg) is True + assert _offpolicy().resolve_sac_use_symmetry(cfg) is True def test_offpolicy_resolve_sac_use_symmetry_disables_motrix(): cfg = _offpolicy_cfg(["task=g1_sac", "training.sim_backend=motrix"]) - resolved_algo = _offpolicy().resolve_offpolicy_algo("sac", cfg) - assert _offpolicy().resolve_sac_use_symmetry(resolved_algo, cfg) is False + assert _offpolicy().resolve_sac_use_symmetry(cfg) is False -def test_ppo_task_go1_exposes_algo_motrix(): +def test_ppo_go1_resolved_algo_matches_old_motrix_behavior(): + """Equivalence: PPO Go1 algo hyperparams match pre-refactor motrix values.""" cfg = _ppo_cfg(["task=go1_joystick"]) - assert cfg.algo_motrix.empirical_normalization is True - assert cfg.algo_motrix.policy.init_noise_std == pytest.approx(0.5) - assert cfg.algo_motrix.algorithm.learning_rate == pytest.approx(3.0e-4) - assert cfg.algo_motrix.algorithm.entropy_coef == pytest.approx(1.0e-3) + # Old motrix_legacy.algo_overrides for Go1 PPO merged into base (Go2 pattern) + assert cfg.algo.max_iterations == 151 + assert cfg.algo.empirical_normalization is True + assert cfg.algo.policy.init_noise_std == pytest.approx(0.5) + assert cfg.algo.algorithm.learning_rate == pytest.approx(3.0e-4) + assert cfg.algo.algorithm.entropy_coef == pytest.approx(1.0e-3) + +def test_ppo_g1_resolved_algo_matches_old_motrix_behavior(): + """Equivalence: PPO G1 algo hyperparams match pre-refactor motrix values. -def test_ppo_task_g1_exposes_algo_motrix_and_backend_profile(): + In particular, max_iterations=151 (the motrix value from the old bag), + NOT 220 (the old mujoco base value which has been retired). + """ + cfg = _ppo_cfg(["task=g1_joystick"]) + + assert cfg.algo.max_iterations == 151 + assert cfg.algo.empirical_normalization is True + assert cfg.algo.obs_groups.actor == ["policy"] + assert cfg.algo.policy.init_noise_std == pytest.approx(0.5) + assert cfg.algo.algorithm.learning_rate == pytest.approx(3.0e-4) + assert cfg.algo.algorithm.entropy_coef == pytest.approx(5.0e-3) + + +def test_ppo_g1_backend_profile_has_env_overrides(): cfg = _ppo_cfg(["task=g1_joystick"]) - assert cfg.algo_motrix.empirical_normalization is True - assert cfg.algo_motrix.obs_groups.actor == ["policy"] - assert cfg.algo_motrix.algorithm.entropy_coef == pytest.approx(5.0e-3) assert cfg.backend_profile.control_config.action_scale == pytest.approx(0.5) assert cfg.backend_profile.gait_phase_init_mode == "independent" assert cfg.backend_profile.reset_base_qvel_limit == pytest.approx(0.05) @@ -247,45 +231,28 @@ def test_ppo_task_go2_aligns_mujoco_with_motrix_defaults(): assert cfg.algo.algorithm.entropy_coef == pytest.approx(1.0e-3) -def test_build_ppo_env_cfg_override_and_resolve_algo_go1( +def test_build_ppo_env_cfg_override_go1_motrix( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) env_cfg_override = mod.build_ppo_env_cfg_override(cfg) - resolved_algo = mod.resolve_ppo_algo(cfg) - - # resolved algo merges algo_motrix overrides - assert resolved_algo["empirical_normalization"] is True - assert resolved_algo["policy"]["init_noise_std"] == pytest.approx(0.5) - assert resolved_algo["algorithm"]["learning_rate"] == pytest.approx(3.0e-4) - assert resolved_algo["algorithm"]["entropy_coef"] == pytest.approx(1.0e-3) - # cfg.algo is NOT mutated - assert cfg.algo.empirical_normalization is False + # env_cfg_override has reward + backend_profile commands assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_build_ppo_env_cfg_override_and_resolve_algo_g1( +def test_build_ppo_env_cfg_override_g1_motrix( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=g1_joystick", "training.sim_backend=motrix"]) env_cfg_override = mod.build_ppo_env_cfg_override(cfg) - resolved_algo = mod.resolve_ppo_algo(cfg) - - # resolved algo merges algo_motrix overrides - assert resolved_algo["empirical_normalization"] is True - assert resolved_algo["policy"]["init_noise_std"] == pytest.approx(0.5) - assert resolved_algo["algorithm"]["learning_rate"] == pytest.approx(3.0e-4) - assert resolved_algo["algorithm"]["entropy_coef"] == pytest.approx(5.0e-3) - assert resolved_algo["obs_groups"]["actor"] == ["policy"] - # cfg.algo is NOT mutated - assert cfg.algo.empirical_normalization is False - # env_cfg_override has reward + backend_profile fields + + # env_cfg_override has reward + backend_profile fields (flat, matching env cfg structure) assert env_cfg_override["reward_config"]["scales"]["upper_body_pose"] == pytest.approx(-0.05) assert env_cfg_override["control_config"]["action_scale"] == pytest.approx(0.5) assert env_cfg_override["gait_phase_init_mode"] == "independent" @@ -308,17 +275,15 @@ def test_build_ppo_env_cfg_override_applies_go2_reward_motrix( assert env_cfg_override["reward_config"]["scales"]["tracking_ang_vel"] == pytest.approx(0.2) -def test_resolve_ppo_algo_respects_cli_algo_override( +def test_ppo_cli_algo_override_wins_over_base( monkeypatch: pytest.MonkeyPatch, ): - mod = _train_rsl_rl(monkeypatch) + """CLI override takes precedence over base task algo values via Hydra compose.""" cfg = _ppo_cfg(["task=g1_joystick", "training.sim_backend=motrix", "algo.max_iterations=1"]) - monkeypatch.setattr(sys, "argv", ["train_rsl_rl.py", "algo.max_iterations=1"]) - - resolved_algo = mod.resolve_ppo_algo(cfg) - assert resolved_algo["empirical_normalization"] is True - assert resolved_algo["max_iterations"] == 1 + assert cfg.algo.max_iterations == 1 + # Other base values remain intact + assert cfg.algo.empirical_normalization is True def test_g1_motion_tracking_ppo_motrix_prefers_backend_specific_reward( diff --git a/tests/training/test_training_helpers.py b/tests/training/test_training_helpers.py index c9aba162d..36beeaade 100644 --- a/tests/training/test_training_helpers.py +++ b/tests/training/test_training_helpers.py @@ -63,22 +63,19 @@ def test_parse_checkpoint_path_uses_algo_log_name_from_cfg(tmp_path: Path): assert checkpoint_dir == run_dir -def test_backend_adapter_resolve_algo_and_env_override_for_matching_algo(): +def test_backend_adapter_env_cfg_override_for_motrix_sac_go1(): + """Env cfg override carries reward + backend_profile fields. Algo is NOT touched.""" cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) adapter = BackendAdapter(cfg, root_dir=_ROOT_DIR, algo_name="sac") - resolved_algo = adapter.resolve_algo_dict() env_cfg_override = adapter.build_task_env_cfg_override() - # resolved algo merges algo_motrix overrides - assert resolved_algo["num_envs"] == 4096 - assert resolved_algo["max_iterations"] == 2000 - # cfg.algo is NOT mutated - assert cfg.algo.num_envs == 2048 - assert cfg.algo.max_iterations == 3000 - # env_cfg_override has reward + backend_profile fields + # env_cfg_override has reward + backend_profile env fields assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] + # algo values come straight from YAML compose — no mutation, matches old motrix behavior + assert cfg.algo.num_envs == 4096 + assert cfg.algo.max_iterations == 2000 def test_backend_adapter_builds_motrix_play_scene_override(): From 0b06fadb174cfe617b8c82e43aeae9ea6888b2ec Mon Sep 17 00:00:00 2001 From: tatp-yf Date: Fri, 10 Apr 2026 22:01:55 +0800 Subject: [PATCH 3/5] Simplify training config to single task owner configs --- CONTRIBUTING.md | 2 +- conf/appo/config.yaml | 3 +- conf/appo/reward/allegro_appo.yaml | 12 - conf/appo/reward/default.yaml | 2 - .../reward/g1_motion_tracking_motrix.yaml | 20 - .../mujoco.yaml} | 17 +- conf/appo/task/g1_flip_tracking.yaml | 13 - .../g1_flip_tracking/motrix.yaml} | 9 + .../task/g1_flip_tracking/mujoco.yaml} | 9 + conf/appo/task/g1_joystick.yaml | 9 - .../task/g1_joystick/mujoco.yaml} | 8 +- conf/appo/task/g1_motion_tracking.yaml | 13 - conf/appo/task/g1_motion_tracking/motrix.yaml | 30 + conf/appo/task/g1_motion_tracking/mujoco.yaml | 30 + conf/appo/task/go1_joystick.yaml | 6 - .../go1_joystick/mujoco.yaml} | 8 +- conf/appo/task/go2_joystick.yaml | 6 - .../task/go2_joystick/mujoco.yaml} | 6 + conf/offpolicy/config.yaml | 3 +- conf/offpolicy/reward/default.yaml | 2 - conf/offpolicy/task/g1_sac.yaml | 15 - conf/offpolicy/task/go1_joystick.yaml | 15 - conf/offpolicy/task/go2_joystick.yaml | 9 - .../allegro_sac/mujoco.yaml} | 17 +- .../sac/g1_sac/motrix.yaml} | 14 +- .../sac/g1_sac/mujoco.yaml} | 14 +- .../sac/go1_joystick/motrix.yaml} | 12 + .../sac/go1_joystick/mujoco.yaml} | 7 + .../task/sac/go2_joystick/mujoco.yaml} | 8 + .../task/td3/go1_joystick/motrix.yaml | 24 + .../task/td3/go1_joystick/mujoco.yaml} | 7 + .../task/td3/go2_joystick/mujoco.yaml} | 8 + conf/ppo/config.yaml | 3 +- conf/ppo/config_mlx.yaml | 3 +- conf/ppo/reward/allegro_ppo.yaml | 12 - conf/ppo/reward/default.yaml | 2 - .../ppo/reward/g1_motion_tracking_motrix.yaml | 20 - conf/ppo/reward/g1_ppo_motrix.yaml | 22 - conf/ppo/reward/go1_ppo_motrix.yaml | 13 - conf/ppo/reward/go2_ppo_mujoco.yaml | 14 - conf/ppo/task/allegro_inhand.yaml | 14 - conf/ppo/task/allegro_inhand/mujoco.yaml | 24 + conf/ppo/task/g1_flip_tracking.yaml | 20 - conf/ppo/task/g1_flip_tracking/motrix.yaml | 35 + conf/ppo/task/g1_flip_tracking/mujoco.yaml | 35 + conf/ppo/task/g1_joystick.yaml | 30 - conf/ppo/task/g1_joystick/motrix.yaml | 47 ++ .../task/g1_joystick/mujoco.yaml} | 10 + .../motrix.yaml} | 29 +- conf/ppo/task/g1_motion_tracking/mujoco.yaml | 35 + .../motrix.yaml} | 24 +- conf/ppo/task/go1_joystick/mujoco.yaml | 23 + .../motrix.yaml} | 23 +- conf/ppo/task/go2_joystick/mujoco.yaml | 30 + docs/zh_CN/00-development-architecture.md | 5 +- docs/zh_CN/01-getting-started.md | 10 +- docs/zh_CN/02-simulation-backends.md | 15 +- docs/zh_CN/03-training.md | 30 +- docs/zh_CN/04-algorithms.md | 30 +- docs/zh_CN/05-g1-motion-tracking.md | 35 +- docs/zh_CN/CONTRIBUTING.md | 2 +- scripts/__init__.py | 1 + scripts/train_appo.py | 54 +- scripts/train_mlx_ppo.py | 68 +- scripts/train_offpolicy.py | 41 +- .../algos/torch/offpolicy/multi_gpu_runner.py | 7 +- .../manipulation/inhand_rot_allegro/README.md | 21 +- src/unilab/training/backend_adapter.py | 94 +-- src/unilab/utils/reward_utils.py | 15 +- src/unilab/utils/rsl_rl_compat.py | 4 +- tests/config/test_config_system.py | 681 +++++------------- tests/config/test_locomotion_params.py | 22 +- tests/config/test_reward_injection.py | 38 +- .../test_reward_injection_integration.py | 32 +- tests/scripts/test_train_script_configs.py | 31 +- tests/scripts/test_train_scripts.py | 157 ++-- tests/training/test_training_helpers.py | 50 +- 77 files changed, 1128 insertions(+), 1141 deletions(-) delete mode 100644 conf/appo/reward/allegro_appo.yaml delete mode 100644 conf/appo/reward/default.yaml delete mode 100644 conf/appo/reward/g1_motion_tracking_motrix.yaml rename conf/appo/task/{allegro_inhand.yaml => allegro_inhand/mujoco.yaml} (51%) delete mode 100644 conf/appo/task/g1_flip_tracking.yaml rename conf/appo/{reward/g1_motion_tracking_mujoco.yaml => task/g1_flip_tracking/motrix.yaml} (73%) rename conf/{ppo/reward/g1_motion_tracking_mujoco.yaml => appo/task/g1_flip_tracking/mujoco.yaml} (73%) delete mode 100644 conf/appo/task/g1_joystick.yaml rename conf/{ppo/reward/g1_ppo_mujoco.yaml => appo/task/g1_joystick/mujoco.yaml} (80%) delete mode 100644 conf/appo/task/g1_motion_tracking.yaml create mode 100644 conf/appo/task/g1_motion_tracking/motrix.yaml create mode 100644 conf/appo/task/g1_motion_tracking/mujoco.yaml delete mode 100644 conf/appo/task/go1_joystick.yaml rename conf/appo/{reward/go2_appo_mujoco.yaml => task/go1_joystick/mujoco.yaml} (68%) delete mode 100644 conf/appo/task/go2_joystick.yaml rename conf/{offpolicy/reward/go2_sac_mujoco.yaml => appo/task/go2_joystick/mujoco.yaml} (71%) delete mode 100644 conf/offpolicy/reward/default.yaml delete mode 100644 conf/offpolicy/task/g1_sac.yaml delete mode 100644 conf/offpolicy/task/go1_joystick.yaml delete mode 100644 conf/offpolicy/task/go2_joystick.yaml rename conf/offpolicy/task/{allegro_sac.yaml => sac/allegro_sac/mujoco.yaml} (54%) rename conf/offpolicy/{reward/g1_sac_motrix.yaml => task/sac/g1_sac/motrix.yaml} (70%) rename conf/offpolicy/{reward/g1_sac_mujoco.yaml => task/sac/g1_sac/mujoco.yaml} (70%) rename conf/offpolicy/{reward/go1_sac_motrix.yaml => task/sac/go1_joystick/motrix.yaml} (53%) rename conf/offpolicy/{reward/go1_sac_mujoco.yaml => task/sac/go1_joystick/mujoco.yaml} (65%) rename conf/{ppo/reward/go2_ppo_motrix.yaml => offpolicy/task/sac/go2_joystick/mujoco.yaml} (65%) create mode 100644 conf/offpolicy/task/td3/go1_joystick/motrix.yaml rename conf/{appo/reward/go1_appo_mujoco.yaml => offpolicy/task/td3/go1_joystick/mujoco.yaml} (65%) rename conf/{ppo/reward/go1_ppo_mujoco.yaml => offpolicy/task/td3/go2_joystick/mujoco.yaml} (65%) delete mode 100644 conf/ppo/reward/allegro_ppo.yaml delete mode 100644 conf/ppo/reward/default.yaml delete mode 100644 conf/ppo/reward/g1_motion_tracking_motrix.yaml delete mode 100644 conf/ppo/reward/g1_ppo_motrix.yaml delete mode 100644 conf/ppo/reward/go1_ppo_motrix.yaml delete mode 100644 conf/ppo/reward/go2_ppo_mujoco.yaml delete mode 100644 conf/ppo/task/allegro_inhand.yaml create mode 100644 conf/ppo/task/allegro_inhand/mujoco.yaml delete mode 100644 conf/ppo/task/g1_flip_tracking.yaml create mode 100644 conf/ppo/task/g1_flip_tracking/motrix.yaml create mode 100644 conf/ppo/task/g1_flip_tracking/mujoco.yaml delete mode 100644 conf/ppo/task/g1_joystick.yaml create mode 100644 conf/ppo/task/g1_joystick/motrix.yaml rename conf/{appo/reward/g1_appo_mujoco.yaml => ppo/task/g1_joystick/mujoco.yaml} (76%) rename conf/ppo/task/{g1_motion_tracking.yaml => g1_motion_tracking/motrix.yaml} (54%) create mode 100644 conf/ppo/task/g1_motion_tracking/mujoco.yaml rename conf/ppo/task/{go1_joystick.yaml => go1_joystick/motrix.yaml} (51%) create mode 100644 conf/ppo/task/go1_joystick/mujoco.yaml rename conf/ppo/task/{go2_joystick.yaml => go2_joystick/motrix.yaml} (55%) create mode 100644 conf/ppo/task/go2_joystick/mujoco.yaml create mode 100644 scripts/__init__.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12ad68b4d..42acaafed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -143,7 +143,7 @@ Use GitHub Issues to report bugs or propose features. UniLab uses Hydra + dataclass configuration: -- **Add a new task**: create YAML under `conf/{algo}/task/` and use `# @package _global_` +- **Add a new task entry**: create a single owner YAML under `conf/{algo}/task/...` and use `# @package _global_` - **Change hyperparameters**: edit the matching YAML or use CLI overrides such as `algo.num_envs=2048` - **Add a new algorithm**: add the dataclass in `structured_configs.py` and create the matching `conf/` directory diff --git a/conf/appo/config.yaml b/conf/appo/config.yaml index 9043e11a0..c918d7f6a 100644 --- a/conf/appo/config.yaml +++ b/conf/appo/config.yaml @@ -1,7 +1,6 @@ defaults: - _self_ - - reward: default - - task: go1_joystick + - task: go1_joystick/mujoco algo: algo: appo diff --git a/conf/appo/reward/allegro_appo.yaml b/conf/appo/reward/allegro_appo.yaml deleted file mode 100644 index 9741eb910..000000000 --- a/conf/appo/reward/allegro_appo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# @package _global_ -reward: - scales: - rotate: 1.25 - obj_linvel: -0.3 - pose_diff: -0.3 - torque: -0.1 - work: -2.0 - drop: 0.0 - angvel_clip_min: -0.5 - angvel_clip_max: 0.5 - reset_z_threshold: 0.125 diff --git a/conf/appo/reward/default.yaml b/conf/appo/reward/default.yaml deleted file mode 100644 index 96b8f030d..000000000 --- a/conf/appo/reward/default.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# @package _global_ -# No reward overrides - use env defaults diff --git a/conf/appo/reward/g1_motion_tracking_motrix.yaml b/conf/appo/reward/g1_motion_tracking_motrix.yaml deleted file mode 100644 index 361dcc6b1..000000000 --- a/conf/appo/reward/g1_motion_tracking_motrix.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# @package _group_ -scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 -std_root_pos: 0.3 -std_root_ori: 0.4 -std_body_pos: 0.3 -std_body_ori: 0.4 -std_body_lin_vel: 1.0 -std_body_ang_vel: 3.14 -std_joint_pos: 0.2 -std_joint_vel: 1.0 diff --git a/conf/appo/task/allegro_inhand.yaml b/conf/appo/task/allegro_inhand/mujoco.yaml similarity index 51% rename from conf/appo/task/allegro_inhand.yaml rename to conf/appo/task/allegro_inhand/mujoco.yaml index f83a7418d..c70791397 100644 --- a/conf/appo/task/allegro_inhand.yaml +++ b/conf/appo/task/allegro_inhand/mujoco.yaml @@ -1,10 +1,8 @@ # @package _global_ -defaults: - - override /reward: allegro_appo - +_selected_sim_backend: mujoco training: task_name: AllegroInhandRotation - + sim_backend: mujoco algo: num_envs: 16384 steps_per_env: 8 @@ -16,3 +14,14 @@ algo: obs_normalization: true critic: obs_normalization: true +reward: + scales: + rotate: 1.25 + obj_linvel: -0.3 + pose_diff: -0.3 + torque: -0.1 + work: -2.0 + drop: 0.0 + angvel_clip_min: -0.5 + angvel_clip_max: 0.5 + reset_z_threshold: 0.125 diff --git a/conf/appo/task/g1_flip_tracking.yaml b/conf/appo/task/g1_flip_tracking.yaml deleted file mode 100644 index 97bc3e909..000000000 --- a/conf/appo/task/g1_flip_tracking.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# @package _global_ -defaults: - - /reward@reward_motrix: g1_motion_tracking_motrix - - override /reward: g1_motion_tracking_mujoco - -training: - task_name: G1FlipTracking - play_steps: 1000 - -algo: - num_envs: 1024 - max_iterations: 5000 - save_interval: 500 diff --git a/conf/appo/reward/g1_motion_tracking_mujoco.yaml b/conf/appo/task/g1_flip_tracking/motrix.yaml similarity index 73% rename from conf/appo/reward/g1_motion_tracking_mujoco.yaml rename to conf/appo/task/g1_flip_tracking/motrix.yaml index 4fd6d90e2..fabf421d0 100644 --- a/conf/appo/reward/g1_motion_tracking_mujoco.yaml +++ b/conf/appo/task/g1_flip_tracking/motrix.yaml @@ -1,4 +1,13 @@ # @package _global_ +_selected_sim_backend: motrix +training: + task_name: G1FlipTracking + sim_backend: motrix + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 5000 + save_interval: 500 reward: scales: motion_global_root_pos: 0.5 diff --git a/conf/ppo/reward/g1_motion_tracking_mujoco.yaml b/conf/appo/task/g1_flip_tracking/mujoco.yaml similarity index 73% rename from conf/ppo/reward/g1_motion_tracking_mujoco.yaml rename to conf/appo/task/g1_flip_tracking/mujoco.yaml index 4fd6d90e2..223f7a98a 100644 --- a/conf/ppo/reward/g1_motion_tracking_mujoco.yaml +++ b/conf/appo/task/g1_flip_tracking/mujoco.yaml @@ -1,4 +1,13 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1FlipTracking + sim_backend: mujoco + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 5000 + save_interval: 500 reward: scales: motion_global_root_pos: 0.5 diff --git a/conf/appo/task/g1_joystick.yaml b/conf/appo/task/g1_joystick.yaml deleted file mode 100644 index b0841e31f..000000000 --- a/conf/appo/task/g1_joystick.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# @package _global_ -defaults: - - override /reward: g1_appo_mujoco - -training: - task_name: G1JoystickFlatTerrain -algo: - max_iterations: 500 - save_interval: 100 diff --git a/conf/ppo/reward/g1_ppo_mujoco.yaml b/conf/appo/task/g1_joystick/mujoco.yaml similarity index 80% rename from conf/ppo/reward/g1_ppo_mujoco.yaml rename to conf/appo/task/g1_joystick/mujoco.yaml index 21aba5a8b..945e03061 100644 --- a/conf/ppo/reward/g1_ppo_mujoco.yaml +++ b/conf/appo/task/g1_joystick/mujoco.yaml @@ -1,4 +1,11 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1JoystickFlatTerrain + sim_backend: mujoco +algo: + max_iterations: 500 + save_interval: 100 reward: scales: tracking_lin_vel: 2.0 @@ -18,4 +25,3 @@ reward: min_base_height: 0.55 max_tilt_deg: 25.0 pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] - diff --git a/conf/appo/task/g1_motion_tracking.yaml b/conf/appo/task/g1_motion_tracking.yaml deleted file mode 100644 index 644772a1c..000000000 --- a/conf/appo/task/g1_motion_tracking.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# @package _global_ -defaults: - - /reward@reward_motrix: g1_motion_tracking_motrix - - override /reward: g1_motion_tracking_mujoco - -training: - task_name: G1MotionTracking - play_steps: 1000 - -algo: - num_envs: 1024 - max_iterations: 5000 - save_interval: 500 diff --git a/conf/appo/task/g1_motion_tracking/motrix.yaml b/conf/appo/task/g1_motion_tracking/motrix.yaml new file mode 100644 index 000000000..cfa4dae4d --- /dev/null +++ b/conf/appo/task/g1_motion_tracking/motrix.yaml @@ -0,0 +1,30 @@ +# @package _global_ +_selected_sim_backend: motrix +training: + task_name: G1MotionTracking + sim_backend: motrix + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 5000 + save_interval: 500 +reward: + scales: + motion_global_root_pos: 0.5 + motion_global_root_ori: 0.5 + motion_body_pos: 1.0 + motion_body_ori: 1.0 + motion_body_lin_vel: 1.0 + motion_body_ang_vel: 1.0 + motion_joint_pos: 0.0 + motion_joint_vel: 0.0 + action_rate_l2: -0.1 + joint_limit: -10.0 + std_root_pos: 0.3 + std_root_ori: 0.4 + std_body_pos: 0.3 + std_body_ori: 0.4 + std_body_lin_vel: 1.0 + std_body_ang_vel: 3.14 + std_joint_pos: 0.2 + std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_motion_tracking/mujoco.yaml b/conf/appo/task/g1_motion_tracking/mujoco.yaml new file mode 100644 index 000000000..d39117b09 --- /dev/null +++ b/conf/appo/task/g1_motion_tracking/mujoco.yaml @@ -0,0 +1,30 @@ +# @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1MotionTracking + sim_backend: mujoco + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 5000 + save_interval: 500 +reward: + scales: + motion_global_root_pos: 0.5 + motion_global_root_ori: 0.5 + motion_body_pos: 1.0 + motion_body_ori: 1.0 + motion_body_lin_vel: 1.0 + motion_body_ang_vel: 1.0 + motion_joint_pos: 0.0 + motion_joint_vel: 0.0 + action_rate_l2: -0.1 + joint_limit: -10.0 + std_root_pos: 0.3 + std_root_ori: 0.4 + std_body_pos: 0.3 + std_body_ori: 0.4 + std_body_lin_vel: 1.0 + std_body_ang_vel: 3.14 + std_joint_pos: 0.2 + std_joint_vel: 1.0 diff --git a/conf/appo/task/go1_joystick.yaml b/conf/appo/task/go1_joystick.yaml deleted file mode 100644 index 857420a33..000000000 --- a/conf/appo/task/go1_joystick.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ -defaults: - - override /reward: go1_appo_mujoco - -training: - task_name: Go1JoystickFlatTerrain diff --git a/conf/appo/reward/go2_appo_mujoco.yaml b/conf/appo/task/go1_joystick/mujoco.yaml similarity index 68% rename from conf/appo/reward/go2_appo_mujoco.yaml rename to conf/appo/task/go1_joystick/mujoco.yaml index d2afeeb12..2e604807f 100644 --- a/conf/appo/reward/go2_appo_mujoco.yaml +++ b/conf/appo/task/go1_joystick/mujoco.yaml @@ -1,4 +1,10 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go1JoystickFlatTerrain + sim_backend: mujoco +algo: + max_iterations: 150 reward: scales: tracking_lin_vel: 1.0 @@ -8,8 +14,6 @@ reward: base_height: -100.0 action_rate: -0.005 similar_to_default: -0.1 - alive: 0.0 contact: 0.24 - swing_feet_z: 4.0 tracking_sigma: 0.25 base_height_target: 0.3 diff --git a/conf/appo/task/go2_joystick.yaml b/conf/appo/task/go2_joystick.yaml deleted file mode 100644 index 808602bd6..000000000 --- a/conf/appo/task/go2_joystick.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ -defaults: - - override /reward: go2_appo_mujoco - -training: - task_name: Go2JoystickFlatTerrain diff --git a/conf/offpolicy/reward/go2_sac_mujoco.yaml b/conf/appo/task/go2_joystick/mujoco.yaml similarity index 71% rename from conf/offpolicy/reward/go2_sac_mujoco.yaml rename to conf/appo/task/go2_joystick/mujoco.yaml index d2afeeb12..45256da05 100644 --- a/conf/offpolicy/reward/go2_sac_mujoco.yaml +++ b/conf/appo/task/go2_joystick/mujoco.yaml @@ -1,4 +1,10 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go2JoystickFlatTerrain + sim_backend: mujoco +algo: + max_iterations: 150 reward: scales: tracking_lin_vel: 1.0 diff --git a/conf/offpolicy/config.yaml b/conf/offpolicy/config.yaml index 89447555d..6a862eda2 100644 --- a/conf/offpolicy/config.yaml +++ b/conf/offpolicy/config.yaml @@ -1,8 +1,7 @@ defaults: - _self_ - algo: sac - - reward: default - - task: go1_joystick + - task: ${algo}/go1_joystick/mujoco training: task_name: Go1JoystickFlatTerrain diff --git a/conf/offpolicy/reward/default.yaml b/conf/offpolicy/reward/default.yaml deleted file mode 100644 index 96b8f030d..000000000 --- a/conf/offpolicy/reward/default.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# @package _global_ -# No reward overrides - use env defaults diff --git a/conf/offpolicy/task/g1_sac.yaml b/conf/offpolicy/task/g1_sac.yaml deleted file mode 100644 index c4dfd4522..000000000 --- a/conf/offpolicy/task/g1_sac.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# @package _global_ -defaults: - - override /reward: g1_sac_mujoco - -training: - task_name: G1WalkTaskMjSAC -algo: - num_envs: 2048 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: true - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 diff --git a/conf/offpolicy/task/go1_joystick.yaml b/conf/offpolicy/task/go1_joystick.yaml deleted file mode 100644 index a5478793b..000000000 --- a/conf/offpolicy/task/go1_joystick.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# @package _global_ -defaults: - - /reward@reward_motrix: go1_sac_motrix - - override /reward: go1_sac_mujoco - -training: - task_name: Go1JoystickFlatTerrain -algo: - num_envs: 4096 - max_iterations: 2000 -backend_profile: - commands: - vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] diff --git a/conf/offpolicy/task/go2_joystick.yaml b/conf/offpolicy/task/go2_joystick.yaml deleted file mode 100644 index 647f51905..000000000 --- a/conf/offpolicy/task/go2_joystick.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# @package _global_ -defaults: - - override /reward: go2_sac_mujoco - -training: - task_name: Go2JoystickFlatTerrain -algo: - num_envs: 1024 - max_iterations: 3000 diff --git a/conf/offpolicy/task/allegro_sac.yaml b/conf/offpolicy/task/sac/allegro_sac/mujoco.yaml similarity index 54% rename from conf/offpolicy/task/allegro_sac.yaml rename to conf/offpolicy/task/sac/allegro_sac/mujoco.yaml index d82ba7897..20845bd76 100644 --- a/conf/offpolicy/task/allegro_sac.yaml +++ b/conf/offpolicy/task/sac/allegro_sac/mujoco.yaml @@ -1,9 +1,8 @@ # @package _global_ -defaults: - - override /reward: default - +_selected_sim_backend: mujoco training: task_name: AllegroInhandRotationSac + sim_backend: mujoco algo: num_envs: 2048 batch_size: 16384 @@ -17,3 +16,15 @@ algo: alpha_init: 0.001 target_entropy_ratio: 0.0 max_grad_norm: 1.0 +reward: + scales: + rotate: 0.0 + alive: 10.0 + obj_linvel: -0.0 + pose_diff: -0.0 + torque: -0.0 + work: -0.0 + drop: -4000.0 + angvel_clip_min: -0.5 + angvel_clip_max: 0.5 + reset_z_threshold: 0.125 diff --git a/conf/offpolicy/reward/g1_sac_motrix.yaml b/conf/offpolicy/task/sac/g1_sac/motrix.yaml similarity index 70% rename from conf/offpolicy/reward/g1_sac_motrix.yaml rename to conf/offpolicy/task/sac/g1_sac/motrix.yaml index 4ab0951c2..63721337e 100644 --- a/conf/offpolicy/reward/g1_sac_motrix.yaml +++ b/conf/offpolicy/task/sac/g1_sac/motrix.yaml @@ -1,4 +1,17 @@ # @package _global_ +_selected_sim_backend: motrix +training: + task_name: G1WalkTaskMjSAC + sim_backend: motrix +algo: + num_envs: 2048 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + use_symmetry: false + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 reward: scales: tracking_lin_vel: 2.2 @@ -19,4 +32,3 @@ reward: feet_phase_tracking_sigma: 0.008 close_feet_threshold: 0.15 pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] - diff --git a/conf/offpolicy/reward/g1_sac_mujoco.yaml b/conf/offpolicy/task/sac/g1_sac/mujoco.yaml similarity index 70% rename from conf/offpolicy/reward/g1_sac_mujoco.yaml rename to conf/offpolicy/task/sac/g1_sac/mujoco.yaml index 14b90af3e..a6feb5b49 100644 --- a/conf/offpolicy/reward/g1_sac_mujoco.yaml +++ b/conf/offpolicy/task/sac/g1_sac/mujoco.yaml @@ -1,4 +1,17 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1WalkTaskMjSAC + sim_backend: mujoco +algo: + num_envs: 2048 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + use_symmetry: true + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 reward: scales: tracking_lin_vel: 2.0 @@ -19,4 +32,3 @@ reward: feet_phase_tracking_sigma: 0.008 close_feet_threshold: 0.15 pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] - diff --git a/conf/offpolicy/reward/go1_sac_motrix.yaml b/conf/offpolicy/task/sac/go1_joystick/motrix.yaml similarity index 53% rename from conf/offpolicy/reward/go1_sac_motrix.yaml rename to conf/offpolicy/task/sac/go1_joystick/motrix.yaml index 3f5dee7ad..4bcad056c 100644 --- a/conf/offpolicy/reward/go1_sac_motrix.yaml +++ b/conf/offpolicy/task/sac/go1_joystick/motrix.yaml @@ -1,4 +1,16 @@ # @package _global_ +_selected_sim_backend: motrix +training: + task_name: Go1JoystickFlatTerrain + sim_backend: motrix +algo: + num_envs: 4096 + max_iterations: 2000 +env: + commands: + vel_limit: + - [0.5, 0.0, 0.0] + - [0.5, 0.0, 0.0] reward: scales: tracking_lin_vel: 1.0 diff --git a/conf/offpolicy/reward/go1_sac_mujoco.yaml b/conf/offpolicy/task/sac/go1_joystick/mujoco.yaml similarity index 65% rename from conf/offpolicy/reward/go1_sac_mujoco.yaml rename to conf/offpolicy/task/sac/go1_joystick/mujoco.yaml index 67e30db7a..8ecc25d87 100644 --- a/conf/offpolicy/reward/go1_sac_mujoco.yaml +++ b/conf/offpolicy/task/sac/go1_joystick/mujoco.yaml @@ -1,4 +1,11 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go1JoystickFlatTerrain + sim_backend: mujoco +algo: + num_envs: 2048 + max_iterations: 3000 reward: scales: tracking_lin_vel: 1.0 diff --git a/conf/ppo/reward/go2_ppo_motrix.yaml b/conf/offpolicy/task/sac/go2_joystick/mujoco.yaml similarity index 65% rename from conf/ppo/reward/go2_ppo_motrix.yaml rename to conf/offpolicy/task/sac/go2_joystick/mujoco.yaml index 958f54e45..af559c653 100644 --- a/conf/ppo/reward/go2_ppo_motrix.yaml +++ b/conf/offpolicy/task/sac/go2_joystick/mujoco.yaml @@ -1,4 +1,11 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go2JoystickFlatTerrain + sim_backend: mujoco +algo: + num_envs: 1024 + max_iterations: 3000 reward: scales: tracking_lin_vel: 1.0 @@ -8,6 +15,7 @@ reward: base_height: -100.0 action_rate: -0.005 similar_to_default: -0.1 + alive: 0.0 contact: 0.24 swing_feet_z: 4.0 tracking_sigma: 0.25 diff --git a/conf/offpolicy/task/td3/go1_joystick/motrix.yaml b/conf/offpolicy/task/td3/go1_joystick/motrix.yaml new file mode 100644 index 000000000..4c7c21278 --- /dev/null +++ b/conf/offpolicy/task/td3/go1_joystick/motrix.yaml @@ -0,0 +1,24 @@ +# @package _global_ +_selected_sim_backend: motrix +training: + task_name: Go1JoystickFlatTerrain + sim_backend: motrix +algo: + num_envs: 2048 + max_iterations: 3000 +env: + commands: + vel_limit: + - [0.5, 0.0, 0.0] + - [0.5, 0.0, 0.0] +reward: + scales: + tracking_lin_vel: 1.0 + tracking_ang_vel: 0.2 + lin_vel_z: -5.0 + ang_vel_xy: -0.1 + base_height: -100.0 + action_rate: -0.005 + similar_to_default: -0.1 + tracking_sigma: 0.25 + base_height_target: 0.3 diff --git a/conf/appo/reward/go1_appo_mujoco.yaml b/conf/offpolicy/task/td3/go1_joystick/mujoco.yaml similarity index 65% rename from conf/appo/reward/go1_appo_mujoco.yaml rename to conf/offpolicy/task/td3/go1_joystick/mujoco.yaml index 67e30db7a..8ecc25d87 100644 --- a/conf/appo/reward/go1_appo_mujoco.yaml +++ b/conf/offpolicy/task/td3/go1_joystick/mujoco.yaml @@ -1,4 +1,11 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go1JoystickFlatTerrain + sim_backend: mujoco +algo: + num_envs: 2048 + max_iterations: 3000 reward: scales: tracking_lin_vel: 1.0 diff --git a/conf/ppo/reward/go1_ppo_mujoco.yaml b/conf/offpolicy/task/td3/go2_joystick/mujoco.yaml similarity index 65% rename from conf/ppo/reward/go1_ppo_mujoco.yaml rename to conf/offpolicy/task/td3/go2_joystick/mujoco.yaml index 958f54e45..af559c653 100644 --- a/conf/ppo/reward/go1_ppo_mujoco.yaml +++ b/conf/offpolicy/task/td3/go2_joystick/mujoco.yaml @@ -1,4 +1,11 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go2JoystickFlatTerrain + sim_backend: mujoco +algo: + num_envs: 1024 + max_iterations: 3000 reward: scales: tracking_lin_vel: 1.0 @@ -8,6 +15,7 @@ reward: base_height: -100.0 action_rate: -0.005 similar_to_default: -0.1 + alive: 0.0 contact: 0.24 swing_feet_z: 4.0 tracking_sigma: 0.25 diff --git a/conf/ppo/config.yaml b/conf/ppo/config.yaml index c8697498d..07a3f7714 100644 --- a/conf/ppo/config.yaml +++ b/conf/ppo/config.yaml @@ -1,7 +1,6 @@ defaults: - _self_ - - reward: default - - task: go1_joystick + - task: go1_joystick/mujoco algo: algo: ppo diff --git a/conf/ppo/config_mlx.yaml b/conf/ppo/config_mlx.yaml index 2f5772ddc..9eaacb767 100644 --- a/conf/ppo/config_mlx.yaml +++ b/conf/ppo/config_mlx.yaml @@ -1,7 +1,6 @@ defaults: - _self_ - - reward: default - - task: go1_joystick + - task: go1_joystick/mujoco algo: algo: ppo diff --git a/conf/ppo/reward/allegro_ppo.yaml b/conf/ppo/reward/allegro_ppo.yaml deleted file mode 100644 index 9741eb910..000000000 --- a/conf/ppo/reward/allegro_ppo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# @package _global_ -reward: - scales: - rotate: 1.25 - obj_linvel: -0.3 - pose_diff: -0.3 - torque: -0.1 - work: -2.0 - drop: 0.0 - angvel_clip_min: -0.5 - angvel_clip_max: 0.5 - reset_z_threshold: 0.125 diff --git a/conf/ppo/reward/default.yaml b/conf/ppo/reward/default.yaml deleted file mode 100644 index 96b8f030d..000000000 --- a/conf/ppo/reward/default.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# @package _global_ -# No reward overrides - use env defaults diff --git a/conf/ppo/reward/g1_motion_tracking_motrix.yaml b/conf/ppo/reward/g1_motion_tracking_motrix.yaml deleted file mode 100644 index 19934d355..000000000 --- a/conf/ppo/reward/g1_motion_tracking_motrix.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# @package _group_ -scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.05 - joint_limit: -10.0 -std_root_pos: 0.3 -std_root_ori: 0.4 -std_body_pos: 0.3 -std_body_ori: 0.4 -std_body_lin_vel: 1.0 -std_body_ang_vel: 3.14 -std_joint_pos: 0.2 -std_joint_vel: 1.0 diff --git a/conf/ppo/reward/g1_ppo_motrix.yaml b/conf/ppo/reward/g1_ppo_motrix.yaml deleted file mode 100644 index c27836651..000000000 --- a/conf/ppo/reward/g1_ppo_motrix.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# @package _global_ -reward: - scales: - tracking_lin_vel: 3.0 - tracking_ang_vel: 0.25 - forward_progress: 1.5 - under_speed: -1.0 - upper_body_pose: -0.05 - feet_phase: 0.5 - lin_vel_z: -1.0 - ang_vel_xy: -0.2 - base_height: -120.0 - orientation: -2.5 - action_rate: -0.005 - pose: -0.05 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.5 - max_tilt_deg: 35.0 diff --git a/conf/ppo/reward/go1_ppo_motrix.yaml b/conf/ppo/reward/go1_ppo_motrix.yaml deleted file mode 100644 index 251245a88..000000000 --- a/conf/ppo/reward/go1_ppo_motrix.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# @package _global_ -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 diff --git a/conf/ppo/reward/go2_ppo_mujoco.yaml b/conf/ppo/reward/go2_ppo_mujoco.yaml deleted file mode 100644 index 958f54e45..000000000 --- a/conf/ppo/reward/go2_ppo_mujoco.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# @package _global_ -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 diff --git a/conf/ppo/task/allegro_inhand.yaml b/conf/ppo/task/allegro_inhand.yaml deleted file mode 100644 index e14167c4d..000000000 --- a/conf/ppo/task/allegro_inhand.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# @package _global_ -defaults: - - override /reward: allegro_ppo - -training: - task_name: AllegroInhandRotation -algo: - num_envs: 16384 - num_steps_per_env: 8 - max_iterations: 501 - empirical_normalization: true - algorithm: - value_loss_coef: 4.0 - desired_kl: 0.02 diff --git a/conf/ppo/task/allegro_inhand/mujoco.yaml b/conf/ppo/task/allegro_inhand/mujoco.yaml new file mode 100644 index 000000000..ea0347243 --- /dev/null +++ b/conf/ppo/task/allegro_inhand/mujoco.yaml @@ -0,0 +1,24 @@ +# @package _global_ +_selected_sim_backend: mujoco +training: + task_name: AllegroInhandRotation + sim_backend: mujoco +algo: + num_envs: 16384 + num_steps_per_env: 8 + max_iterations: 501 + empirical_normalization: true + algorithm: + value_loss_coef: 4.0 + desired_kl: 0.02 +reward: + scales: + rotate: 1.25 + obj_linvel: -0.3 + pose_diff: -0.3 + torque: -0.1 + work: -2.0 + drop: 0.0 + angvel_clip_min: -0.5 + angvel_clip_max: 0.5 + reset_z_threshold: 0.125 diff --git a/conf/ppo/task/g1_flip_tracking.yaml b/conf/ppo/task/g1_flip_tracking.yaml deleted file mode 100644 index 57cd893eb..000000000 --- a/conf/ppo/task/g1_flip_tracking.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# @package _global_ -defaults: - - /reward@reward_motrix: g1_motion_tracking_motrix - - override /reward: g1_motion_tracking_mujoco - -training: - task_name: G1FlipTracking - play_steps: 1000 -algo: - num_envs: 1024 - # max_iterations: 5000 - max_iterations: 30000 - save_interval: 500 - obs_groups: - default: - - policy - actor: - - actor - algorithm: - entropy_coef: 0.005 diff --git a/conf/ppo/task/g1_flip_tracking/motrix.yaml b/conf/ppo/task/g1_flip_tracking/motrix.yaml new file mode 100644 index 000000000..733d98ee3 --- /dev/null +++ b/conf/ppo/task/g1_flip_tracking/motrix.yaml @@ -0,0 +1,35 @@ +# @package _global_ +_selected_sim_backend: motrix +training: + task_name: G1FlipTracking + sim_backend: motrix + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 30000 + save_interval: 500 + obs_groups: + actor: + - actor + algorithm: + entropy_coef: 0.005 +reward: + scales: + motion_global_root_pos: 1.0 + motion_global_root_ori: 0.5 + motion_body_pos: 1.0 + motion_body_ori: 1.0 + motion_body_lin_vel: 1.0 + motion_body_ang_vel: 1.0 + motion_joint_pos: 0.0 + motion_joint_vel: 0.0 + action_rate_l2: -0.05 + joint_limit: -10.0 + std_root_pos: 0.3 + std_root_ori: 0.4 + std_body_pos: 0.3 + std_body_ori: 0.4 + std_body_lin_vel: 1.0 + std_body_ang_vel: 3.14 + std_joint_pos: 0.2 + std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_flip_tracking/mujoco.yaml b/conf/ppo/task/g1_flip_tracking/mujoco.yaml new file mode 100644 index 000000000..70cd1265a --- /dev/null +++ b/conf/ppo/task/g1_flip_tracking/mujoco.yaml @@ -0,0 +1,35 @@ +# @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1FlipTracking + sim_backend: mujoco + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 30000 + save_interval: 500 + obs_groups: + actor: + - actor + algorithm: + entropy_coef: 0.005 +reward: + scales: + motion_global_root_pos: 0.5 + motion_global_root_ori: 0.5 + motion_body_pos: 1.0 + motion_body_ori: 1.0 + motion_body_lin_vel: 1.0 + motion_body_ang_vel: 1.0 + motion_joint_pos: 0.0 + motion_joint_vel: 0.0 + action_rate_l2: -0.1 + joint_limit: -10.0 + std_root_pos: 0.3 + std_root_ori: 0.4 + std_body_pos: 0.3 + std_body_ori: 0.4 + std_body_lin_vel: 1.0 + std_body_ang_vel: 3.14 + std_joint_pos: 0.2 + std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_joystick.yaml b/conf/ppo/task/g1_joystick.yaml deleted file mode 100644 index f0a6be33f..000000000 --- a/conf/ppo/task/g1_joystick.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# @package _global_ -defaults: - - /reward@reward_motrix: g1_ppo_motrix - - override /reward: g1_ppo_mujoco - -training: - task_name: G1JoystickFlatTerrain -algo: - num_envs: 2048 - max_iterations: 151 - empirical_normalization: true - obs_groups: - default: - - policy - actor: - - policy - policy: - init_noise_std: 0.5 - algorithm: - learning_rate: 3.0e-4 - entropy_coef: 5.0e-3 -backend_profile: - control_config: - action_scale: 0.5 - commands: - vel_limit: - - [0.45, 0.0, 0.0] - - [0.55, 0.0, 0.0] - gait_phase_init_mode: independent - reset_base_qvel_limit: 0.05 diff --git a/conf/ppo/task/g1_joystick/motrix.yaml b/conf/ppo/task/g1_joystick/motrix.yaml new file mode 100644 index 000000000..984fc8f25 --- /dev/null +++ b/conf/ppo/task/g1_joystick/motrix.yaml @@ -0,0 +1,47 @@ +# @package _global_ +_selected_sim_backend: motrix +training: + task_name: G1JoystickFlatTerrain + sim_backend: motrix +algo: + num_envs: 2048 + max_iterations: 151 + empirical_normalization: true + obs_groups: + actor: + - policy + policy: + init_noise_std: 0.5 + algorithm: + learning_rate: 3.0e-4 + entropy_coef: 5.0e-3 +env: + control_config: + action_scale: 0.5 + commands: + vel_limit: + - [0.45, 0.0, 0.0] + - [0.55, 0.0, 0.0] + gait_phase_init_mode: independent + reset_base_qvel_limit: 0.05 +reward: + scales: + tracking_lin_vel: 3.0 + tracking_ang_vel: 0.25 + forward_progress: 1.5 + under_speed: -1.0 + upper_body_pose: -0.05 + feet_phase: 0.5 + lin_vel_z: -1.0 + ang_vel_xy: -0.2 + base_height: -120.0 + orientation: -2.5 + action_rate: -0.005 + pose: -0.05 + tracking_sigma: 0.25 + gait_frequency: 1.5 + feet_phase_swing_height: 0.09 + feet_phase_tracking_sigma: 0.008 + base_height_target: 0.754 + min_base_height: 0.5 + max_tilt_deg: 35.0 diff --git a/conf/appo/reward/g1_appo_mujoco.yaml b/conf/ppo/task/g1_joystick/mujoco.yaml similarity index 76% rename from conf/appo/reward/g1_appo_mujoco.yaml rename to conf/ppo/task/g1_joystick/mujoco.yaml index db453d5d7..3e27ae247 100644 --- a/conf/appo/reward/g1_appo_mujoco.yaml +++ b/conf/ppo/task/g1_joystick/mujoco.yaml @@ -1,4 +1,14 @@ # @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1JoystickFlatTerrain + sim_backend: mujoco +algo: + num_envs: 2048 + max_iterations: 220 + obs_groups: + actor: + - actor reward: scales: tracking_lin_vel: 2.0 diff --git a/conf/ppo/task/g1_motion_tracking.yaml b/conf/ppo/task/g1_motion_tracking/motrix.yaml similarity index 54% rename from conf/ppo/task/g1_motion_tracking.yaml rename to conf/ppo/task/g1_motion_tracking/motrix.yaml index 81fc76d4f..40ae40e30 100644 --- a/conf/ppo/task/g1_motion_tracking.yaml +++ b/conf/ppo/task/g1_motion_tracking/motrix.yaml @@ -1,19 +1,14 @@ # @package _global_ -defaults: - - /reward@reward_motrix: g1_motion_tracking_motrix - - override /reward: g1_motion_tracking_mujoco - +_selected_sim_backend: motrix training: task_name: G1MotionTracking + sim_backend: motrix play_steps: 1000 algo: num_envs: 1024 - # max_iterations: 5000 max_iterations: 15000 save_interval: 500 obs_groups: - default: - - policy actor: - actor algorithm: @@ -31,3 +26,23 @@ motrix_play_only: skybox_rgb1: [0.90, 0.90, 0.91] skybox_rgb2: [0.68, 0.68, 0.70] ground_texrepeat: [0.25, 0.25] +reward: + scales: + motion_global_root_pos: 1.0 + motion_global_root_ori: 0.5 + motion_body_pos: 1.0 + motion_body_ori: 1.0 + motion_body_lin_vel: 1.0 + motion_body_ang_vel: 1.0 + motion_joint_pos: 0.0 + motion_joint_vel: 0.0 + action_rate_l2: -0.05 + joint_limit: -10.0 + std_root_pos: 0.3 + std_root_ori: 0.4 + std_body_pos: 0.3 + std_body_ori: 0.4 + std_body_lin_vel: 1.0 + std_body_ang_vel: 3.14 + std_joint_pos: 0.2 + std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_motion_tracking/mujoco.yaml b/conf/ppo/task/g1_motion_tracking/mujoco.yaml new file mode 100644 index 000000000..ea89d13b6 --- /dev/null +++ b/conf/ppo/task/g1_motion_tracking/mujoco.yaml @@ -0,0 +1,35 @@ +# @package _global_ +_selected_sim_backend: mujoco +training: + task_name: G1MotionTracking + sim_backend: mujoco + play_steps: 1000 +algo: + num_envs: 1024 + max_iterations: 15000 + save_interval: 500 + obs_groups: + actor: + - actor + algorithm: + entropy_coef: 0.005 +reward: + scales: + motion_global_root_pos: 0.5 + motion_global_root_ori: 0.5 + motion_body_pos: 1.0 + motion_body_ori: 1.0 + motion_body_lin_vel: 1.0 + motion_body_ang_vel: 1.0 + motion_joint_pos: 0.0 + motion_joint_vel: 0.0 + action_rate_l2: -0.1 + joint_limit: -10.0 + std_root_pos: 0.3 + std_root_ori: 0.4 + std_body_pos: 0.3 + std_body_ori: 0.4 + std_body_lin_vel: 1.0 + std_body_ang_vel: 3.14 + std_joint_pos: 0.2 + std_joint_vel: 1.0 diff --git a/conf/ppo/task/go1_joystick.yaml b/conf/ppo/task/go1_joystick/motrix.yaml similarity index 51% rename from conf/ppo/task/go1_joystick.yaml rename to conf/ppo/task/go1_joystick/motrix.yaml index 6b9cf6c36..41d118007 100644 --- a/conf/ppo/task/go1_joystick.yaml +++ b/conf/ppo/task/go1_joystick/motrix.yaml @@ -1,25 +1,33 @@ # @package _global_ -defaults: - - /reward@reward_motrix: go1_ppo_motrix - - override /reward: go1_ppo_mujoco - +_selected_sim_backend: motrix training: task_name: Go1JoystickFlatTerrain + sim_backend: motrix algo: max_iterations: 151 - empirical_normalization: true obs_groups: - default: - - policy actor: - actor + empirical_normalization: true policy: init_noise_std: 0.5 algorithm: learning_rate: 3.0e-4 entropy_coef: 1.0e-3 -backend_profile: +env: commands: vel_limit: - [0.5, 0.0, 0.0] - [0.5, 0.0, 0.0] +reward: + scales: + tracking_lin_vel: 1.0 + tracking_ang_vel: 0.2 + lin_vel_z: -5.0 + ang_vel_xy: -0.1 + base_height: -100.0 + action_rate: -0.005 + similar_to_default: -0.1 + swing_feet_z: 4.0 + tracking_sigma: 0.25 + base_height_target: 0.3 diff --git a/conf/ppo/task/go1_joystick/mujoco.yaml b/conf/ppo/task/go1_joystick/mujoco.yaml new file mode 100644 index 000000000..ae3b409e5 --- /dev/null +++ b/conf/ppo/task/go1_joystick/mujoco.yaml @@ -0,0 +1,23 @@ +# @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go1JoystickFlatTerrain + sim_backend: mujoco +algo: + max_iterations: 151 + obs_groups: + actor: + - actor +reward: + scales: + tracking_lin_vel: 1.0 + tracking_ang_vel: 0.2 + lin_vel_z: -5.0 + ang_vel_xy: -0.1 + base_height: -100.0 + action_rate: -0.005 + similar_to_default: -0.1 + contact: 0.24 + swing_feet_z: 4.0 + tracking_sigma: 0.25 + base_height_target: 0.3 diff --git a/conf/ppo/task/go2_joystick.yaml b/conf/ppo/task/go2_joystick/motrix.yaml similarity index 55% rename from conf/ppo/task/go2_joystick.yaml rename to conf/ppo/task/go2_joystick/motrix.yaml index 1659aa119..44095ce5c 100644 --- a/conf/ppo/task/go2_joystick.yaml +++ b/conf/ppo/task/go2_joystick/motrix.yaml @@ -1,17 +1,13 @@ # @package _global_ -defaults: - - /reward@reward_motrix: go2_ppo_motrix - - override /reward: go2_ppo_mujoco - +_selected_sim_backend: motrix training: task_name: Go2JoystickFlatTerrain + sim_backend: motrix algo: num_envs: 1024 max_iterations: 151 empirical_normalization: true obs_groups: - default: - - policy actor: - actor policy: @@ -19,7 +15,7 @@ algo: algorithm: learning_rate: 3.0e-4 entropy_coef: 1.0e-3 -backend_profile: +env: commands: vel_limit: - [0.5, 0.0, 0.0] @@ -27,3 +23,16 @@ backend_profile: domain_rand: randomize_kp: false randomize_kd: false +reward: + scales: + tracking_lin_vel: 1.0 + tracking_ang_vel: 0.2 + lin_vel_z: -5.0 + ang_vel_xy: -0.1 + base_height: -100.0 + action_rate: -0.005 + similar_to_default: -0.1 + contact: 0.24 + swing_feet_z: 4.0 + tracking_sigma: 0.25 + base_height_target: 0.3 diff --git a/conf/ppo/task/go2_joystick/mujoco.yaml b/conf/ppo/task/go2_joystick/mujoco.yaml new file mode 100644 index 000000000..68b5ae467 --- /dev/null +++ b/conf/ppo/task/go2_joystick/mujoco.yaml @@ -0,0 +1,30 @@ +# @package _global_ +_selected_sim_backend: mujoco +training: + task_name: Go2JoystickFlatTerrain + sim_backend: mujoco +algo: + num_envs: 1024 + max_iterations: 151 + empirical_normalization: true + obs_groups: + actor: + - actor + policy: + init_noise_std: 0.5 + algorithm: + learning_rate: 3.0e-4 + entropy_coef: 1.0e-3 +reward: + scales: + tracking_lin_vel: 1.0 + tracking_ang_vel: 0.2 + lin_vel_z: -5.0 + ang_vel_xy: -0.1 + base_height: -100.0 + action_rate: -0.005 + similar_to_default: -0.1 + contact: 0.24 + swing_feet_z: 4.0 + tracking_sigma: 0.25 + base_height_target: 0.3 diff --git a/docs/zh_CN/00-development-architecture.md b/docs/zh_CN/00-development-architecture.md index e7b7a2937..dde9f03a1 100644 --- a/docs/zh_CN/00-development-architecture.md +++ b/docs/zh_CN/00-development-architecture.md @@ -68,8 +68,11 @@ CPU Physics Sim ──shm──► Collector / IPC ──shm──► GPU Learne UniLab 使用 dataclass + Hydra。schema 位于 `src/unilab/config/structured_configs.py`，运行时配置位于 `conf/{ppo,appo,offpolicy}/`。 -合成顺序: `{algo}/config*.yaml` -> `task=...` -> `reward[_{backend}]` -> CLI override -> `backend_profile` (env-only overrides，motrix 时由 BackendAdapter 应用)。 +合成顺序: `{algo}/config*.yaml` -> `task=...` -> CLI override。 +- `task` 是唯一 owner 配置入口：同一个 task + backend（offpolicy 再加 algo）对应一个 YAML，里面直接放这个组合的 `training.task_name` / `training.sim_backend` / `reward` / `env` / task-specific `algo` +- PPO / APPO 入口形如 `conf/{ppo,appo}/task//.yaml`；offpolicy 入口形如 `conf/offpolicy/task///.yaml` +- 这里的 `task` 不是旧设计里“只表达任务、再去别处拼 backend/reward/algo”的 group；它本身就是最终 owner 配置入口 - reward 必须显式注入 - 如果 backend 选择会影响 task 或 reward 行为，就必须通过 config 表达 - 动态 override 必须尊重 CLI diff --git a/docs/zh_CN/01-getting-started.md b/docs/zh_CN/01-getting-started.md index 812e7b1b1..a2d05ff1c 100644 --- a/docs/zh_CN/01-getting-started.md +++ b/docs/zh_CN/01-getting-started.md @@ -51,21 +51,21 @@ uv sync --index-url https://pypi.tuna.tsinghua.edu.cn/simple ### 训练一个最小任务 ```bash -uv run python scripts/train_rsl_rl.py task=go1_joystick +uv run python scripts/train_rsl_rl.py task=go1_joystick/mujoco ``` ### 常用入口脚本 ```bash # PPO (RSL-RL) -uv run python scripts/train_rsl_rl.py task=go1_joystick +uv run python scripts/train_rsl_rl.py task=go1_joystick/mujoco # APPO -uv run python scripts/train_appo.py task=go1_joystick +uv run python scripts/train_appo.py task=go1_joystick/mujoco # SAC / TD3 -uv run python scripts/train_offpolicy.py algo=sac task=go1_joystick -uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick +uv run python scripts/train_offpolicy.py algo=sac task=sac/go1_joystick/mujoco +uv run python scripts/train_offpolicy.py algo=td3 task=td3/go1_joystick/mujoco ``` ### 验证环境 diff --git a/docs/zh_CN/02-simulation-backends.md b/docs/zh_CN/02-simulation-backends.md index 3d10ed888..5584285e2 100644 --- a/docs/zh_CN/02-simulation-backends.md +++ b/docs/zh_CN/02-simulation-backends.md @@ -36,14 +36,21 @@ UniLab 当前支持两个仿真后端: ## Select A Backend -默认后端是 `mujoco`。通过 Hydra 参数 `training.sim_backend` 切换到 `motrix`。 +默认后端是 `mujoco`。通过 Hydra config choice `sim_backend` 切换到 `motrix`。 + +实际要改参数时，不再去拆着找 `reward` / `backend preset` / `algo preset`。直接改对应的 `task` 文件： + +- PPO / APPO: `conf/{ppo,appo}/task//.yaml` +- offpolicy: `conf/offpolicy/task///.yaml` + +现在没有单独的 `reward/`、`backend preset`、`sim_backend/` 配置组。`task/` 是唯一 owner 入口，不再是旧的拆分式 task 配置。 ```bash # 默认 MuJoCo -uv run python scripts/train_rsl_rl.py task=go1_joystick +uv run python scripts/train_rsl_rl.py task=go1_joystick/mujoco # 显式指定 Motrix -uv run python scripts/train_rsl_rl.py task=go1_joystick training.sim_backend=motrix +uv run python scripts/train_rsl_rl.py task=go1_joystick/motrix ``` ## Playback Differences @@ -54,7 +61,7 @@ uv run python scripts/train_rsl_rl.py task=go1_joystick training.sim_backend=mot 对 G1 motion tracking 来说，目前已验证的 Motrix 路径是 `PPO (torch) + motrix` 和 `APPO (torch) + motrix`。`scripts/play_interactive.py` 仍然沿用 MuJoCo 路径。 ```bash -uv run python scripts/train_rsl_rl.py task=go1_joystick training.play_only=true +uv run python scripts/train_rsl_rl.py task=go1_joystick/mujoco training.play_only=true ``` ## Notes diff --git a/docs/zh_CN/03-training.md b/docs/zh_CN/03-training.md index fc522da54..954ffe2dc 100644 --- a/docs/zh_CN/03-training.md +++ b/docs/zh_CN/03-training.md @@ -17,20 +17,20 @@ ```bash # PPO (RSL-RL) -uv run python scripts/train_rsl_rl.py task=go1_joystick +uv run python scripts/train_rsl_rl.py task=go1_joystick/mujoco # PPO (MLX, Apple Silicon) -uv run python scripts/train_mlx_ppo.py task=go1_joystick +uv run python scripts/train_mlx_ppo.py task=go1_joystick/mujoco # APPO -uv run python scripts/train_appo.py task=go1_joystick +uv run python scripts/train_appo.py task=go1_joystick/mujoco # Off-policy -uv run python scripts/train_offpolicy.py algo=sac task=go1_joystick -uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick +uv run python scripts/train_offpolicy.py algo=sac task=sac/go1_joystick/mujoco +uv run python scripts/train_offpolicy.py algo=td3 task=td3/go1_joystick/mujoco # CLI override -uv run python scripts/train_offpolicy.py algo=sac task=g1_sac algo.num_envs=2048 algo.max_iterations=1000 +uv run python scripts/train_offpolicy.py algo=sac task=sac/g1_sac/mujoco algo.num_envs=2048 algo.max_iterations=1000 ``` 训练脚本默认会在训练结束后自动进入回放。 @@ -45,18 +45,18 @@ run 目录命名格式是 `YYYY-MM-DD_HH-MM-SS_`，例如 `2026-03- ```bash # 回放最新结果 -uv run python scripts/train_rsl_rl.py task=go2_joystick training.play_only=true -uv run python scripts/train_offpolicy.py algo=sac task=go2_joystick training.play_only=true +uv run python scripts/train_rsl_rl.py task=go2_joystick/mujoco training.play_only=true +uv run python scripts/train_offpolicy.py algo=sac task=sac/go2_joystick/mujoco training.play_only=true # 回放指定 run -uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick training.play_only=true training.load_run="2024-02-04_12-00-00" +uv run python scripts/train_offpolicy.py algo=td3 task=td3/go1_joystick/mujoco training.play_only=true training.load_run="2024-02-04_12-00-00" ``` ## Resume Training ```bash -uv run python scripts/train_rsl_rl.py task=go2_joystick training.load_run="2024-02-04_12-00-00" -uv run python scripts/train_offpolicy.py algo=sac task=go2_joystick training.load_run="2024-02-04_12-00-00" +uv run python scripts/train_rsl_rl.py task=go2_joystick/mujoco training.load_run="2024-02-04_12-00-00" +uv run python scripts/train_offpolicy.py algo=sac task=sac/go2_joystick/mujoco training.load_run="2024-02-04_12-00-00" ``` ## Hydra Overrides @@ -68,7 +68,7 @@ uv run python scripts/train_offpolicy.py algo=sac task=go2_joystick training.loa uv run python scripts/train_*.py [config_group=value] [key.subkey=value] # 常见参数 -task=go1_joystick +task=go1_joystick/mujoco algo=sac training.play_only=true training.no_play=true @@ -95,11 +95,11 @@ uv run python scripts/train_offpolicy.py --cfg job ```bash # 基本用法 -uv run python scripts/train_rsl_rl.py task=go1_joystick training.logger=wandb +uv run python scripts/train_rsl_rl.py task=go1_joystick/mujoco training.logger=wandb # 共享 project / entity uv run python scripts/train_appo.py \ - task=go1_joystick \ + task=go1_joystick/mujoco \ training.logger=wandb \ training.wandb_project=unilab-benchmark \ training.wandb_entity=my-team @@ -107,7 +107,7 @@ uv run python scripts/train_appo.py \ # 按 task 分组 uv run python scripts/train_offpolicy.py \ algo=sac \ - task=go2_joystick \ + task=sac/go2_joystick/mujoco \ training.logger=wandb \ training.wandb_project=unilab-benchmark \ training.wandb_group=go2_joystick diff --git a/docs/zh_CN/04-algorithms.md b/docs/zh_CN/04-algorithms.md index 1010cb5e7..70dd7c6b6 100644 --- a/docs/zh_CN/04-algorithms.md +++ b/docs/zh_CN/04-algorithms.md @@ -22,30 +22,30 @@ APPO 是 UniLab 的异步 PPO 实现，带有 V-trace importance-sampling 修正 ```bash # 默认训练 -uv run python scripts/train_appo.py task=go1_joystick +uv run python scripts/train_appo.py task=go1_joystick/mujoco # 指定环境数和迭代数 -uv run python scripts/train_appo.py task=go2_joystick algo.num_envs=2048 algo.max_iterations=300 +uv run python scripts/train_appo.py task=go2_joystick/mujoco algo.num_envs=2048 algo.max_iterations=300 # 调整 replay queue 深度 -uv run python scripts/train_appo.py task=go1_joystick training.replay_queue_size=2 +uv run python scripts/train_appo.py task=go1_joystick/mujoco training.replay_queue_size=2 # 跳过自动回放 -uv run python scripts/train_appo.py task=go1_joystick training.no_play=true +uv run python scripts/train_appo.py task=go1_joystick/mujoco training.no_play=true ``` ### Playback ```bash -uv run python scripts/train_appo.py task=go1_joystick training.play_only=true -uv run python scripts/train_appo.py task=go1_joystick training.play_only=true training.load_run="2026-03-16_01-35-12_mujoco" +uv run python scripts/train_appo.py task=go1_joystick/mujoco training.play_only=true +uv run python scripts/train_appo.py task=go1_joystick/mujoco training.play_only=true training.load_run="2026-03-16_01-35-12_mujoco" ``` ### Key Parameters | 参数 | 默认值 | 说明 | |------|--------|------| -| `task` | `go2_joystick` | task 配置名 | +| `task` | `go1_joystick/mujoco` | 单个 task 配置入口，内部同时定义 task + backend | | `algo.max_iterations` | 150 | 最大训练迭代数 | | `algo.num_envs` | 2048 | 并行环境数量 | | `algo.steps_per_env` | 24 | 每个 env 的 rollout 长度 | @@ -84,21 +84,21 @@ FastSAC 和 FastTD3 使用同一套异步多进程架构，通过 shared memory ```bash # 基本训练 -uv run python scripts/train_offpolicy.py algo=sac task=go2_joystick -uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick +uv run python scripts/train_offpolicy.py algo=sac task=sac/go2_joystick/mujoco +uv run python scripts/train_offpolicy.py algo=td3 task=td3/go1_joystick/mujoco # 异步采集模式 -uv run python scripts/train_offpolicy.py algo=sac task=go2_joystick training.no_sync_collection=true +uv run python scripts/train_offpolicy.py algo=sac task=sac/go2_joystick/mujoco training.no_sync_collection=true # 跳过自动回放 -uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick training.no_play=true +uv run python scripts/train_offpolicy.py algo=td3 task=td3/go1_joystick/mujoco training.no_play=true ``` ### Playback ```bash -uv run python scripts/train_offpolicy.py algo=sac task=go2_joystick training.play_only=true -uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick training.play_only=true training.load_run="2024-02-04_12-00-00" +uv run python scripts/train_offpolicy.py algo=sac task=sac/go2_joystick/mujoco training.play_only=true +uv run python scripts/train_offpolicy.py algo=td3 task=td3/go1_joystick/mujoco training.play_only=true training.load_run="2024-02-04_12-00-00" ``` ### Key Parameters @@ -106,11 +106,11 @@ uv run python scripts/train_offpolicy.py algo=td3 task=go1_joystick training.pla | 参数 | 默认值 | 说明 | |------|--------|------| | `algo` | `sac` | 算法选择 | -| `task` | `go1_joystick` | task 配置名 | +| `task` | `sac/go1_joystick/mujoco` | 单个 task 配置入口，内部同时定义 algo + task + backend | | `algo.max_iterations` | 500 (SAC) / 5000 (TD3) | 最大训练迭代数 | | `algo.num_envs` | 4096 | 并行环境数量 | | `training.device` | 自动检测 | learner 设备 | -| `training.sim_backend` | `mujoco` | 仿真后端 | +| `conf/*/task/...` | - | 唯一 owner 配置入口；reward/env/backend-specific algo 都在这里改 | | `training.no_sync_collection` | false | 启用异步采集 | | `training.env_steps_per_sync` | 1 | 同步模式下每轮采集步数 | | `training.play_only` | false | 仅回放 | diff --git a/docs/zh_CN/05-g1-motion-tracking.md b/docs/zh_CN/05-g1-motion-tracking.md index ce300a9f3..dc7905709 100644 --- a/docs/zh_CN/05-g1-motion-tracking.md +++ b/docs/zh_CN/05-g1-motion-tracking.md @@ -2,52 +2,51 @@ 语言: 简体中文 -UniLab 当前提供两个 G1 whole-body motion tracking task： +UniLab 当前提供两个 G1 whole-body motion tracking task family： -- Hydra task：`g1_motion_tracking`（兼容历史默认） +- task family：`g1_motion_tracking` - 注册环境名：`G1MotionTracking` -- Hydra task：`g1_flip_tracking`（flip 专用 profile） +- task family：`g1_flip_tracking`（flip 专用 profile） - 注册环境名：`G1FlipTracking` - 后端注册：`mujoco` 和 `motrix` - 已提交的 Motrix 特化配置：PPO 和 APPO 的 motion-tracking reward - `g1_motion_tracking` 默认 motion：`src/unilab/assets/motions/g1/dance1_subject2_part.npz` - `g1_flip_tracking` 默认 motion：`src/unilab/assets/motions/g1/flip_360_001__A304.npz` +- 实际训练入口统一写成 `task=/` ## Environment Entrypoints ```bash # PPO (RSL-RL, MuJoCo) -uv run python scripts/train_rsl_rl.py task=g1_motion_tracking +uv run python scripts/train_rsl_rl.py task=g1_motion_tracking/mujoco # PPO (RSL-RL, MuJoCo, flip profile) -uv run python scripts/train_rsl_rl.py task=g1_flip_tracking +uv run python scripts/train_rsl_rl.py task=g1_flip_tracking/mujoco # PPO (RSL-RL, Motrix) -uv run python scripts/train_rsl_rl.py task=g1_motion_tracking training.sim_backend=motrix +uv run python scripts/train_rsl_rl.py task=g1_motion_tracking/motrix # PPO (RSL-RL, Motrix, flip profile) -uv run python scripts/train_rsl_rl.py task=g1_flip_tracking training.sim_backend=motrix +uv run python scripts/train_rsl_rl.py task=g1_flip_tracking/motrix # APPO (MuJoCo) -uv run python scripts/train_appo.py task=g1_motion_tracking +uv run python scripts/train_appo.py task=g1_motion_tracking/mujoco # APPO (Motrix) -uv run python scripts/train_appo.py task=g1_motion_tracking training.sim_backend=motrix +uv run python scripts/train_appo.py task=g1_motion_tracking/motrix # 回放最新 checkpoint -uv run python scripts/train_rsl_rl.py task=g1_motion_tracking training.play_only=true +uv run python scripts/train_rsl_rl.py task=g1_motion_tracking/mujoco training.play_only=true # Motrix PPO 回放会打开原生 renderer -uv run python scripts/train_rsl_rl.py task=g1_motion_tracking \ - training.sim_backend=motrix \ +uv run python scripts/train_rsl_rl.py task=g1_motion_tracking/motrix \ training.play_only=true # APPO MuJoCo 回放 -uv run python scripts/train_appo.py task=g1_motion_tracking training.play_only=true +uv run python scripts/train_appo.py task=g1_motion_tracking/mujoco training.play_only=true # APPO Motrix 回放会打开原生 renderer -uv run python scripts/train_appo.py task=g1_motion_tracking \ - training.sim_backend=motrix \ +uv run python scripts/train_appo.py task=g1_motion_tracking/motrix \ training.play_only=true ``` @@ -121,7 +120,7 @@ uv run python scripts/motion/replay_npz.py \ ## Configuration Note -`task=g1_motion_tracking` 默认读取环境配置里的单个 `motion_file`（历史默认是 `dance1_subject2_part.npz`）。`task=g1_flip_tracking` 提供 flip 专用默认 profile（更保守的 reset 随机化与 termination）。 +`task=g1_motion_tracking/mujoco` 默认读取环境配置里的单个 `motion_file`（历史默认是 `dance1_subject2_part.npz`）。`task=g1_flip_tracking/mujoco` 提供 flip 专用默认 profile（更保守的 reset 随机化与 termination）。 PPO 默认训练预算也做了分流：`g1_motion_tracking` 保持历史默认 `max_iterations=15000`，`g1_flip_tracking` 使用更长的 `max_iterations=30000`。 @@ -142,8 +141,8 @@ motion_file: 验证 Motrix 路径时，优先使用训练脚本自带的 play mode，而不是只支持 MuJoCo 的调试脚本: ```bash -uv run python scripts/train_rsl_rl.py task=g1_motion_tracking training.sim_backend=motrix -uv run python scripts/train_appo.py task=g1_motion_tracking training.sim_backend=motrix +uv run python scripts/train_rsl_rl.py task=g1_motion_tracking/motrix +uv run python scripts/train_appo.py task=g1_motion_tracking/motrix ``` ## Navigation diff --git a/docs/zh_CN/CONTRIBUTING.md b/docs/zh_CN/CONTRIBUTING.md index 88213198f..a2d9d98d8 100644 --- a/docs/zh_CN/CONTRIBUTING.md +++ b/docs/zh_CN/CONTRIBUTING.md @@ -143,7 +143,7 @@ uv run pytest -m veryslow -v UniLab 使用 Hydra + dataclass 配置系统: -- **添加新任务**: 在 `conf/{algo}/task/` 下创建 YAML，并使用 `# @package _global_` +- **添加新的 task 配置入口**: 在 `conf/{algo}/task/...` 下创建单一 owner YAML，并使用 `# @package _global_` - **修改超参数**: 编辑对应 YAML，或使用 `algo.num_envs=2048` 这样的 CLI override - **添加新算法**: 在 `structured_configs.py` 中添加 dataclass，并创建对应的 `conf/` 目录 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ + diff --git a/scripts/train_appo.py b/scripts/train_appo.py index a5a1eb289..2a3008fae 100644 --- a/scripts/train_appo.py +++ b/scripts/train_appo.py @@ -6,6 +6,7 @@ import os import sys from pathlib import Path +from typing import Any, cast import hydra import torch @@ -14,7 +15,13 @@ ROOT_DIR = Path(__file__).parent.parent sys.path.append(str(ROOT_DIR)) -from unilab.training import create_env, ensure_registries, get_log_root, render_play_mode +from unilab.training import ( + BackendAdapter, + create_env, + ensure_registries, + get_log_root, + render_play_mode, +) from unilab.utils.experiment_tracking import ExperimentTracker @@ -22,10 +29,13 @@ def build_appo_runner_kwargs( cfg: DictConfig, env_cfg_override: dict | None, collector_device: str | None, - rl_cfg: dict | None = None, + rl_cfg: dict[str, Any] | None = None, ) -> dict: if rl_cfg is None: - rl_cfg = OmegaConf.to_container(cfg.algo, resolve=True) + rl_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) + if not isinstance(rl_cfg_raw, dict): + raise TypeError("cfg.algo must resolve to a dict") + rl_cfg = cast(dict[str, Any], rl_cfg_raw) runner_kwargs = { "env_name": cfg.training.task_name, @@ -97,16 +107,17 @@ def _get_log_root(cfg: DictConfig) -> str: return str(get_log_root(ROOT_DIR, cfg)) -def play_appo(cfg: DictConfig, rl_cfg: dict) -> str | None: +def play_appo(cfg: DictConfig, rl_cfg: dict[str, Any]) -> str | None: """Play mode for APPO.""" import numpy as np from rsl_rl.utils import resolve_callable from tensordict import TensorDict - from unilab.utils.reward_utils import extract_reward_config from unilab.utils.rsl_rl_compat import convert_config_v3_to_v4, is_rsl_rl_v4, is_rsl_rl_v5 - env_cfg_override = extract_reward_config(cfg) + env_cfg_override = BackendAdapter( + cfg, root_dir=ROOT_DIR, algo_name="appo" + ).build_task_env_cfg_override() device = cfg.training.device or ( "cuda" @@ -117,15 +128,21 @@ def play_appo(cfg: DictConfig, rl_cfg: dict) -> str | None: ) print(f"Using device for play: {device}") - env = create_env( - cfg, - num_envs=cfg.training.play_env_num, - env_cfg_override=env_cfg_override, + env = cast( + Any, + create_env( + cfg, + num_envs=cfg.training.play_env_num, + env_cfg_override=env_cfg_override, + ), ) from unilab.utils.obs_utils import get_obs_dims obs_dim, privileged_dim = get_obs_dims(env.obs_groups_spec) - action_dim = env.action_space.shape[0] + action_shape = env.action_space.shape + if action_shape is None: + raise ValueError("env.action_space.shape must be defined") + action_dim = int(action_shape[0]) rl_cfg_dict = dict(rl_cfg) if "obs_groups" not in rl_cfg_dict: @@ -183,6 +200,10 @@ def play_appo(cfg: DictConfig, rl_cfg: dict) -> str | None: raise return None + if load_path_dir is None: + print(f"Could not resolve checkpoint directory. load_path_dir={load_path_dir}") + return None + output_video = os.path.join(load_path_dir, "play_video.mp4") print(f"Rendering video to {output_video}...") @@ -231,12 +252,15 @@ def play_appo(cfg: DictConfig, rl_cfg: dict) -> str | None: def main(cfg: DictConfig) -> None: ensure_registries() - from unilab.utils.reward_utils import extract_reward_config - - env_cfg_override = extract_reward_config(cfg) + env_cfg_override = BackendAdapter( + cfg, root_dir=ROOT_DIR, algo_name="appo" + ).build_task_env_cfg_override() # Convert algo config to plain dict for APPORunner / RSL-RL internals - rl_cfg = OmegaConf.to_container(cfg.algo, resolve=True) + rl_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) + if not isinstance(rl_cfg_raw, dict): + raise TypeError("cfg.algo must resolve to a dict") + rl_cfg = cast(dict[str, Any], rl_cfg_raw) if cfg.training.log_dir is None: timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") diff --git a/scripts/train_mlx_ppo.py b/scripts/train_mlx_ppo.py index 7f85b033f..dd5dc0bae 100644 --- a/scripts/train_mlx_ppo.py +++ b/scripts/train_mlx_ppo.py @@ -13,6 +13,7 @@ import time from collections import deque from pathlib import Path +from typing import Any, cast import hydra import mlx.core as mx @@ -26,6 +27,7 @@ from unilab.algos.mlx.common import EmpiricalDiscountedVariationNormalization, RolloutBuffer from unilab.algos.mlx.ppo import MLPActorCritic, PPOConfig, PPOTrainer from unilab.training import ( + BackendAdapter, create_env, ensure_registries, get_log_root, @@ -65,8 +67,14 @@ def __init__(self, log_dir: Path) -> None: self._writer = EventFileWriter(str(log_dir)) def add_scalar(self, tag: str, value: float, step: int) -> None: - summary = self._Summary(value=[self._Summary.Value(tag=tag, simple_value=float(value))]) - event = self._Event(wall_time=time.time(), step=int(step), summary=summary) + summary = cast(Any, self._Summary()) + summary_value = summary.value.add() + summary_value.tag = tag + summary_value.simple_value = float(value) + event = cast(Any, self._Event()) + event.wall_time = time.time() + event.step = int(step) + event.summary.CopyFrom(summary) self._writer.add_event(event) def flush(self) -> None: @@ -77,11 +85,11 @@ def close(self) -> None: def get_latest_run(log_dir: Path) -> Path | None: - return get_latest_run_common(log_dir) + return cast(Path | None, get_latest_run_common(log_dir)) def get_latest_checkpoint(run_dir: Path) -> Path | None: - return get_latest_checkpoint_common(run_dir, suffix=".safetensors") + return cast(Path | None, get_latest_checkpoint_common(run_dir, suffix=".safetensors")) def save_trainer_state(path: Path, trainer: PPOTrainer, iteration: int) -> None: @@ -136,24 +144,30 @@ def _get_physics_state_snapshot(env) -> np.ndarray: def _get_log_root(cfg: DictConfig) -> Path: - return get_log_root(ROOT_DIR, cfg) + return cast(Path, get_log_root(ROOT_DIR, cfg)) def play_mlx_ppo(cfg: DictConfig, dtype, use_fp16: bool, resolved_sim_backend: str) -> str | None: """Play mode for MLX PPO.""" - from unilab.utils.reward_utils import extract_reward_config - - env_cfg_override = extract_reward_config(cfg) + env_cfg_override = BackendAdapter( + cfg, root_dir=ROOT_DIR, algo_name="ppo" + ).build_task_env_cfg_override() play_model_dtype = mx.float32 if use_fp16 else dtype play_env_num = cfg.training.play_env_num - env = create_env( - cfg, - num_envs=play_env_num, - env_cfg_override=env_cfg_override, + env = cast( + Any, + create_env( + cfg, + num_envs=play_env_num, + env_cfg_override=env_cfg_override, + ), ) obs_dim = sum(env.obs_groups_spec.values()) - action_dim = env.action_space.shape[0] + action_shape = env.action_space.shape + if action_shape is None: + raise ValueError("env.action_space.shape must be defined") + action_dim = int(action_shape[0]) model = build_model(cfg.algo, obs_dim, action_dim, dtype=play_model_dtype) task_log_root = _get_log_root(cfg) / cfg.training.task_name @@ -182,7 +196,7 @@ def _play_step(current_obs): actions = mx.where(mx.isfinite(actions_mx), actions_mx, mx.zeros_like(actions_mx)) actions = actions.astype(dtype) if getattr(actions, "dtype", None) != dtype else actions state = env.step(np.asarray(actions)) - raw_obs = flatten_obs_dict(state.obs) + raw_obs = mx.array(flatten_obs_dict(state.obs)) return mx.nan_to_num(raw_obs, nan=0.0, posinf=0.0, neginf=0.0) if resolved_sim_backend == "motrix": @@ -276,14 +290,17 @@ def log(msg: str) -> None: ) tracker.start() - from unilab.utils.reward_utils import extract_reward_config - - env_cfg_override = extract_reward_config(cfg) + env_cfg_override = BackendAdapter( + cfg, root_dir=ROOT_DIR, algo_name="ppo" + ).build_task_env_cfg_override() - env = create_env( - cfg, - num_envs=cfg.algo.num_envs, - env_cfg_override=env_cfg_override, + env = cast( + Any, + create_env( + cfg, + num_envs=cfg.algo.num_envs, + env_cfg_override=env_cfg_override, + ), ) if env.state is None: env.init_state() @@ -292,7 +309,10 @@ def log(msg: str) -> None: obs = mx.array(flatten_obs_dict(obs_dict)) obs_dim = sum(env.obs_groups_spec.values()) - action_dim = env.action_space.shape[0] + action_shape = env.action_space.shape + if action_shape is None: + raise ValueError("env.action_space.shape must be defined") + action_dim = int(action_shape[0]) model = build_model(cfg.algo, obs_dim, action_dim, dtype=model_dtype) ppo_cfg = PPOConfig( @@ -384,8 +404,8 @@ def log(msg: str) -> None: episode_returns = np.zeros((cfg.algo.num_envs,), dtype=(np.float16 if use_fp16 else np.float32)) episode_lengths = np.zeros((cfg.algo.num_envs,), dtype=np.int32) - reward_window = deque(maxlen=100) - length_window = deque(maxlen=100) + reward_window: deque[float] = deque(maxlen=100) + length_window: deque[int] = deque(maxlen=100) collection_size = num_steps * cfg.algo.num_envs total_time = 0.0 best_mean_reward = float("-inf") diff --git a/scripts/train_offpolicy.py b/scripts/train_offpolicy.py index 8af6e9f69..24dee5e3e 100644 --- a/scripts/train_offpolicy.py +++ b/scripts/train_offpolicy.py @@ -6,6 +6,7 @@ import os import sys from pathlib import Path +from typing import Any, cast import hydra from omegaconf import DictConfig @@ -64,7 +65,7 @@ def resolve_play_obs_dim(obs_groups_spec: dict[str, int]) -> int: from unilab.utils.obs_utils import get_obs_dims obs_dim, _ = get_obs_dims(obs_groups_spec) - return obs_dim + return int(obs_dim) def extract_play_obs(obs_dict): @@ -74,13 +75,11 @@ def extract_play_obs(obs_dict): return obs_out -def build_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict | None: - return BackendAdapter(cfg, root_dir=ROOT_DIR, algo_name=algo_name).build_task_env_cfg_override() - - -def resolve_sac_use_symmetry(cfg: DictConfig) -> bool: - """Symmetry augmentation currently depends on MuJoCo model APIs.""" - return bool(cfg.algo.use_symmetry and cfg.training.sim_backend == "mujoco") +def build_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict[str, Any] | None: + return cast( + dict[str, Any] | None, + BackendAdapter(cfg, root_dir=ROOT_DIR, algo_name=algo_name).build_task_env_cfg_override(), + ) def build_runner(algo_name: str, cfg: DictConfig): @@ -181,7 +180,7 @@ def build_runner(algo_name: str, cfg: DictConfig): sync_collection=not cfg.training.no_sync_collection, env_steps_per_sync=cfg.training.env_steps_per_sync, sim_backend=cfg.training.sim_backend, - use_symmetry=resolve_sac_use_symmetry(cfg), + use_symmetry=cfg.algo.use_symmetry, ) if algo_name == "td3": @@ -234,13 +233,19 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: device = default_device(torch, cfg.training.device) print(f"Using device for play: {device}") - env = create_env( - cfg, - num_envs=cfg.training.play_env_num, - env_cfg_override=env_cfg_override, + env = cast( + Any, + create_env( + cfg, + num_envs=cfg.training.play_env_num, + env_cfg_override=env_cfg_override, + ), ) obs_dim = resolve_play_obs_dim(env.obs_groups_spec) - action_dim = env.action_space.shape[0] + action_shape = env.action_space.shape + if action_shape is None: + raise ValueError("env.action_space.shape must be defined") + action_dim = int(action_shape[0]) normalizer = None if algo_name == "sac": @@ -253,6 +258,8 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: device, ) elif algo_name == "td3": + import torch + from unilab.algos.torch.fast_td3.learner import EmpiricalNormalization, TD3Actor actor = TD3Actor( @@ -263,7 +270,7 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: cfg.algo.actor_hidden_dim, cfg.algo.algo_params.log_std_min, cfg.algo.algo_params.log_std_max, - device, + torch.device(device), ) if cfg.algo.obs_normalization: normalizer = EmpiricalNormalization(shape=obs_dim, device=device) @@ -336,6 +343,10 @@ def _policy_step(obs_np: np.ndarray) -> np.ndarray: raise return None + if load_path_dir is None: + print(f"Could not resolve checkpoint directory. load_path_dir={load_path_dir}") + return None + output_video = os.path.join(load_path_dir, "play_video.mp4") print("Collecting physics states...") with torch.inference_mode(): diff --git a/src/unilab/algos/torch/offpolicy/multi_gpu_runner.py b/src/unilab/algos/torch/offpolicy/multi_gpu_runner.py index 6d3579a66..b5d1894c2 100644 --- a/src/unilab/algos/torch/offpolicy/multi_gpu_runner.py +++ b/src/unilab/algos/torch/offpolicy/multi_gpu_runner.py @@ -17,7 +17,7 @@ import time from collections import defaultdict, deque from datetime import timedelta -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, cast import torch import torch.distributed as dist @@ -40,7 +40,10 @@ def _find_free_port() -> int: def _broadcast_initial_params(learner: FastSACLearner, rank: int) -> None: """Broadcast rank-0 initial parameters to all workers for consistent starting point.""" - for model in [learner.actor, learner.qnet]: + for model in ( + cast(torch.nn.Module, learner.actor), + cast(torch.nn.Module, learner.qnet), + ): for p in model.parameters(): dist.broadcast(p.data, src=0) dist.broadcast(learner.log_alpha.data, src=0) diff --git a/src/unilab/envs/manipulation/inhand_rot_allegro/README.md b/src/unilab/envs/manipulation/inhand_rot_allegro/README.md index f902a9712..e255aa839 100644 --- a/src/unilab/envs/manipulation/inhand_rot_allegro/README.md +++ b/src/unilab/envs/manipulation/inhand_rot_allegro/README.md @@ -55,27 +55,26 @@ The output is a `(N, 23)` float32 array: ```bash # From the UniLab root directory -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco ``` **Common options (Hydra syntax):** ```bash # Override number of environments -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation algo.num_envs=8192 +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco algo.num_envs=8192 # Resume from a previous run -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation \ +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco \ training.load_run=2026-03-10_22-50-13 # Train without rendering a play video afterwards -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation training.no_play=true +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco training.no_play=true # Use MuJoCo backend (default) -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation training.sim_backend=mujoco +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco -# Use Motrix backend -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation training.sim_backend=motrix +# Motrix preset is not configured for AllegroInhandRotation ``` Training logs are saved to `logs/rsl_rl_train/AllegroInhandRotation//`. @@ -85,14 +84,14 @@ Training logs are saved to `logs/rsl_rl_train/AllegroInhandRotation// ### Step 3 — Evaluate / render video ```bash -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation training.play_only=true +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco training.play_only=true # Custom camera -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation training.play_only=true \ +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco training.play_only=true \ training.cam_distance=1.5 training.cam_elevation=-30 training.cam_azimuth=45 # Load a specific run -uv run python scripts/train_rsl_rl.py task=AllegroInhandRotation training.play_only=true \ +uv run python scripts/train_rsl_rl.py task=allegro_inhand/mujoco training.play_only=true \ training.load_run=2026-03-10_22-50-13 ``` @@ -135,7 +134,7 @@ inhand_rot_allegro/ Key parameters are in `rotation.py` (`RewardConfig`, `DomainRandConfig`) and `unilab/config/manipulation_params.py` (`ppo_config`). -**PPO hyperparameters** (configured via `conf/ppo/task/allegro_inhand.yaml` and `conf/ppo/reward/allegro_ppo.yaml`): +**PPO hyperparameters** (configured via `conf/ppo/task/allegro_inhand/mujoco.yaml`): | Parameter | Value | |---|---| diff --git a/src/unilab/training/backend_adapter.py b/src/unilab/training/backend_adapter.py index 63ebdda5a..1ebb367f4 100644 --- a/src/unilab/training/backend_adapter.py +++ b/src/unilab/training/backend_adapter.py @@ -13,7 +13,7 @@ class BackendAdapter: - """Apply backend-specific config and env overrides while respecting CLI overrides.""" + """Build env/play overrides from the final composed config.""" def __init__( self, @@ -34,43 +34,23 @@ def __init__( def _is_motrix(self) -> bool: return bool(self.cfg.training.sim_backend == "motrix") - def build_task_env_cfg_override(self) -> dict[str, Any]: - """Build env_cfg_override for training/play entrypoints. + def _assert_backend_choice_consistent(self) -> None: + selected_backend = getattr(self.cfg, "_selected_sim_backend", None) + if selected_backend is None: + return + resolved_backend = str(self.cfg.training.sim_backend) + if str(selected_backend) != resolved_backend: + raise ValueError( + "Hydra task selection is inconsistent with training.sim_backend. " + "Use `task=...` to switch backends instead of overriding " + "`training.sim_backend` directly." + ) - Combines reward config (via reward_motrix sidecar) with backend_profile env overrides. - Algo config is NOT touched — training scripts read cfg.algo directly. - """ + def build_task_env_cfg_override(self) -> dict[str, Any]: + """Build env_cfg_override from the resolved reward + env sections.""" + self._assert_backend_choice_consistent() env_cfg_override = extract_reward_config(self.cfg) - - if not self._is_motrix(): - return env_cfg_override - - profile = getattr(self.cfg, "backend_profile", None) - if profile is None: - return env_cfg_override - - if OmegaConf.is_config(profile): - profile_dict = OmegaConf.to_container(profile, resolve=True) - elif isinstance(profile, dict): - profile_dict = profile - else: - return env_cfg_override - - if not isinstance(profile_dict, dict): - return env_cfg_override - - reward_explicit = any( - key == "reward" - or key.startswith("reward.") - or key.startswith("reward@") - or key.startswith("reward_") - for key in self.explicit_keys - ) - for key, value in profile_dict.items(): - key_str = str(key) - if key_str == "reward_config" and reward_explicit: - continue - env_cfg_override[key_str] = value + env_cfg_override.update(self._to_plain_dict(getattr(self.cfg, "env", None))) return env_cfg_override @@ -120,13 +100,7 @@ def build_play_env_cfg_override(self) -> dict[str, Any]: return env_cfg_override def _apply_nested_overrides(self, target: Any, overrides: Any, *, base_path: str) -> None: - if OmegaConf.is_config(overrides): - override_dict = OmegaConf.to_container(overrides, resolve=True) - elif isinstance(overrides, dict): - override_dict = overrides - else: - return - + override_dict = self._to_plain_dict(overrides) if not isinstance(override_dict, dict): return @@ -144,31 +118,21 @@ def _apply_nested_overrides(self, target: Any, overrides: Any, *, base_path: str target[key_str] = value def _apply_env_profile(self, env_cfg_override: dict[str, Any], env_profile: Any) -> None: - if OmegaConf.is_config(env_profile): - env_profile_dict = OmegaConf.to_container(env_profile, resolve=True) - elif isinstance(env_profile, dict): - env_profile_dict = env_profile - else: - return - - if not isinstance(env_profile_dict, dict): - return - - reward_explicit = any( - key == "reward" - or key.startswith("reward.") - or key.startswith("reward@") - or key.startswith("reward_") - for key in self.explicit_keys - ) - for key, value in env_profile_dict.items(): - key_str = str(key) - if key_str == "reward_config" and reward_explicit: - continue - env_cfg_override[key_str] = value + env_cfg_override.update(self._to_plain_dict(env_profile)) def _resolve_root_relative_path(self, path_value: str) -> str: candidate = Path(path_value) if candidate.is_absolute(): return str(candidate) return str((self.root_dir / candidate).resolve()) + + def _to_plain_dict(self, value: Any) -> dict[str, Any]: + if OmegaConf.is_config(value): + resolved = OmegaConf.to_container(value, resolve=True) + elif isinstance(value, dict): + resolved = value + else: + return {} + if not isinstance(resolved, dict): + return {} + return {str(key): item for key, item in resolved.items()} diff --git a/src/unilab/utils/reward_utils.py b/src/unilab/utils/reward_utils.py index 045110bf4..ee68d18fc 100644 --- a/src/unilab/utils/reward_utils.py +++ b/src/unilab/utils/reward_utils.py @@ -12,7 +12,7 @@ def _to_reward_dict(value: object, *, error_message: str) -> RewardDict: resolved = OmegaConf.to_container(value, resolve=True) if not isinstance(resolved, dict): raise ValueError(error_message) - # Hydra mounts like `/reward@reward_motrix=...` resolve to {"reward": {...}}. + # Some reward configs are mounted as a full `reward:` section. # Env config injection expects the inner reward mapping. if set(resolved) == {"reward"} and isinstance(resolved["reward"], dict): return cast(RewardDict, resolved["reward"]) @@ -20,18 +20,7 @@ def _to_reward_dict(value: object, *, error_message: str) -> RewardDict: def resolve_reward_dict(cfg: DictConfig) -> RewardDict: - """Resolve the reward config, preferring backend-specific overrides when present.""" - sim_backend = OmegaConf.select(cfg, "training.sim_backend") - if sim_backend: - backend_reward = OmegaConf.select(cfg, f"reward_{sim_backend}") - if backend_reward: - reward_dict = _to_reward_dict( - backend_reward, - error_message="Backend-specific reward config must resolve to a mapping.", - ) - if reward_dict: - return reward_dict - + """Resolve the reward config from the final composed config.""" reward_cfg = OmegaConf.select(cfg, "reward") if not reward_cfg: raise ValueError( diff --git a/src/unilab/utils/rsl_rl_compat.py b/src/unilab/utils/rsl_rl_compat.py index 86d234f71..1f0df9bbe 100644 --- a/src/unilab/utils/rsl_rl_compat.py +++ b/src/unilab/utils/rsl_rl_compat.py @@ -33,14 +33,14 @@ def is_rsl_rl_v4() -> bool: """Check if the installed rsl_rl version is 4.x or above.""" version_str = get_rsl_rl_version() print(version_str) - return Version(version_str) >= Version("4.0.0") + return bool(Version(version_str) >= Version("4.0.0")) def is_rsl_rl_v5() -> bool: """Check if the installed rsl_rl version is 4.x or above.""" version_str = get_rsl_rl_version() print(version_str) - return Version(version_str) >= Version("5.0.0") + return bool(Version(version_str) >= Version("5.0.0")) def convert_config_v3_to_v4(cfg: dict) -> dict: diff --git a/tests/config/test_config_system.py b/tests/config/test_config_system.py index 1f0509d4b..52a9db7cd 100644 --- a/tests/config/test_config_system.py +++ b/tests/config/test_config_system.py @@ -1,16 +1,16 @@ -"""Comprehensive config system verification tests. +"""Config system verification tests. These tests enforce that: -1. Every Hydra YAML config composes without errors -2. Every non-default task has a non-empty reward config that reaches cfg.reward -3. Every reward YAML that exists is reachable via some (algo, task, reward) combination -4. The reward config injection chain (Hydra → env_cfg_override → registry.make) works end-to-end -5. sim_backend is present in every algo's training config +1. Base Hydra configs compose without legacy config groups. +2. Every supported runtime variant resolves through exactly one task owner file. +3. Final reward/env/algo sections are present on the composed config, not mounted by Python glue. +4. Backend-specific hyperparameters preserve the intended pre-refactor behavior. """ from __future__ import annotations from pathlib import Path +from typing import Any, cast import pytest from hydra import compose, initialize_config_dir @@ -18,52 +18,98 @@ from omegaconf import OmegaConf CONF_DIR = Path(__file__).parent.parent.parent / "conf" - - -# --------------------------------------------------------------------------- -# helpers -# --------------------------------------------------------------------------- +_PPO_MLX_TASKS = {"go1_joystick", "go2_joystick", "g1_joystick"} def _compose(algo_dir: str, config_name: str = "config", overrides: list[str] | None = None): - """Compose a Hydra config from the given algo directory.""" + normalized_overrides = _normalize_overrides(algo_dir, overrides) + GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / algo_dir), version_base="1.3"): - return compose(config_name, overrides=overrides or []) - + return compose(config_name, overrides=normalized_overrides) -def _assert_reward_populated(cfg, algo: str, task: str): - """Assert that cfg.reward exists and contains non-empty scales.""" - assert hasattr(cfg, "reward"), ( - f"[{algo}/{task}] cfg.reward missing — check defaults list has 'reward: default'" - ) - reward = cfg.reward - assert reward, f"[{algo}/{task}] cfg.reward is empty — task file should override /reward" - reward_dict = OmegaConf.to_container(reward, resolve=True) - assert "scales" in reward_dict, f"[{algo}/{task}] cfg.reward has no 'scales' key" - assert len(reward_dict["scales"]) > 0, f"[{algo}/{task}] cfg.reward.scales is empty" +def _normalize_overrides(algo_dir: str, overrides: list[str] | None) -> list[str]: + algo = "sac" + normalized: list[str] = [] + task_selected = False -def _assert_reward_section_loadable(section, label: str): - """Assert that a composed reward-like section has the expected structure.""" - reward_dict = OmegaConf.to_container(section, resolve=True) - assert isinstance(reward_dict, dict), f"{label} must resolve to a mapping" - assert "scales" in reward_dict, f"{label} must have 'scales' key" + for override in overrides or []: + if override.startswith("algo="): + algo = override.split("=", 1)[1] + normalized.append(override) + continue + if override.startswith("task="): + task_selected = True + normalized.append(override) + continue + normalized.append(override) + if not task_selected: + if algo_dir == "offpolicy": + normalized.append(f"task={algo}/go1_joystick/mujoco") + else: + normalized.append("task=go1_joystick/mujoco") -def _assert_sim_backend_configurable(cfg, algo: str): - """Assert that training.sim_backend exists and defaults to mujoco.""" - assert hasattr(cfg.training, "sim_backend"), ( - f"[{algo}] training.sim_backend missing from config" - ) - assert cfg.training.sim_backend == "mujoco", ( - f"[{algo}] training.sim_backend should default to 'mujoco', got '{cfg.training.sim_backend}'" - ) + return normalized -# --------------------------------------------------------------------------- -# 1. Every algo config composes at all (smoke test) -# --------------------------------------------------------------------------- +def _assert_reward_populated(cfg, label: str): + assert hasattr(cfg, "reward"), f"{label} missing cfg.reward" + reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) + assert isinstance(reward_dict, dict), f"{label} reward must resolve to mapping" + assert "scales" in reward_dict, f"{label} reward must contain scales" + assert len(reward_dict["scales"]) > 0, f"{label} reward.scales must be non-empty" + + +def _supported_task_cases() -> list[tuple[str, str, str, str, str, list[str]]]: + cases: list[tuple[str, str, str, str, str, list[str]]] = [] + + for algo_dir in ["ppo", "appo"]: + root = CONF_DIR / algo_dir / "task" + for task_dir in sorted(path for path in root.iterdir() if path.is_dir()): + for backend_file in sorted(task_dir.glob("*.yaml")): + cases.append( + ( + algo_dir, + "config", + task_dir.name, + backend_file.stem, + str(backend_file.relative_to(CONF_DIR)), + [f"task={task_dir.name}/{backend_file.stem}"], + ) + ) + if algo_dir == "ppo" and task_dir.name in _PPO_MLX_TASKS: + cases.append( + ( + algo_dir, + "config_mlx", + task_dir.name, + backend_file.stem, + str(backend_file.relative_to(CONF_DIR)), + [f"task={task_dir.name}/{backend_file.stem}"], + ) + ) + + offpolicy_root = CONF_DIR / "offpolicy" / "task" + for algo_root in sorted(path for path in offpolicy_root.iterdir() if path.is_dir()): + for task_dir in sorted(path for path in algo_root.iterdir() if path.is_dir()): + for backend_file in sorted(task_dir.glob("*.yaml")): + cases.append( + ( + "offpolicy", + "config", + task_dir.name, + backend_file.stem, + str(backend_file.relative_to(CONF_DIR)), + [ + f"algo={algo_root.name}", + f"task={algo_root.name}/{task_dir.name}/{backend_file.stem}", + ], + ) + ) + + return cases @pytest.mark.parametrize( @@ -76,491 +122,136 @@ def _assert_sim_backend_configurable(cfg, algo: str): ], ) def test_algo_config_composes(algo_dir: str, config_name: str): - """Each base algorithm config should compose without errors.""" - cfg = _compose(algo_dir, config_name) - assert cfg.training.task_name, f"[{algo_dir}/{config_name}] task_name is empty" - - -# --------------------------------------------------------------------------- -# 2. Every algo config has sim_backend in training section -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "algo_dir,config_name", - [ - ("offpolicy", "config"), - ("appo", "config"), - ("ppo", "config"), - ("ppo", "config_mlx"), - ], -) -def test_sim_backend_in_config(algo_dir: str, config_name: str): - """Every algo config must have training.sim_backend.""" - cfg = _compose(algo_dir, config_name) - _assert_sim_backend_configurable(cfg, algo_dir) - - -# --------------------------------------------------------------------------- -# 3. Every algo config has 'reward' in defaults -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "algo_dir,config_name", - [ - ("offpolicy", "config"), - ("appo", "config"), - ("ppo", "config"), - ("ppo", "config_mlx"), - ], -) -def test_reward_key_in_config(algo_dir: str, config_name: str): - """Every algo config must declare the reward config group so overrides work. - - Note: default.yaml is intentionally empty, so cfg.reward won't exist with defaults. - The test verifies that a non-default reward override can be applied without error. - """ - # Just loading with default is fine (no cfg.reward expected) cfg = _compose(algo_dir, config_name) - assert cfg.training.task_name, "base config should load" - - # But we must be able to override reward explicitly - # (this would fail if 'reward: default' is missing from the defaults list) - reward_dir = CONF_DIR / algo_dir / "reward" - if reward_dir.exists(): - non_default = sorted(f.stem for f in reward_dir.glob("*.yaml") if f.stem != "default") - if non_default: - direct_reward = None - for reward_name in non_default: - cfg2 = _compose(algo_dir, config_name, overrides=[f"reward={reward_name}"]) - if hasattr(cfg2, "reward"): - direct_reward = reward_name - break - - assert direct_reward is not None, ( - f"[{algo_dir}/{config_name}] no direct reward override produced cfg.reward " - f"— add 'reward: default' to defaults list" - ) - - -# --------------------------------------------------------------------------- -# 4. Every (algo × task) combination composes correctly -# --------------------------------------------------------------------------- - - -# offpolicy tasks -@pytest.mark.parametrize( - "task,expected_env", - [ - ("go1_joystick", "Go1JoystickFlatTerrain"), - ("go2_joystick", "Go2JoystickFlatTerrain"), - ("g1_sac", "G1WalkTaskMjSAC"), - ("allegro_sac", "AllegroInhandRotationSac"), - ], -) -def test_offpolicy_task_composes(task: str, expected_env: str): - cfg = _compose("offpolicy", overrides=[f"task={task}"]) - assert cfg.training.task_name == expected_env - - -# appo tasks -@pytest.mark.parametrize( - "task,expected_env", - [ - ("go1_joystick", "Go1JoystickFlatTerrain"), - ("go2_joystick", "Go2JoystickFlatTerrain"), - ("g1_joystick", "G1JoystickFlatTerrain"), - ("g1_flip_tracking", "G1FlipTracking"), - ], -) -def test_appo_task_composes(task: str, expected_env: str): - cfg = _compose("appo", overrides=[f"task={task}"]) - assert cfg.training.task_name == expected_env - - -# ppo tasks (rsl_rl + mlx) -@pytest.mark.parametrize( - "task,expected_env", - [ - ("go1_joystick", "Go1JoystickFlatTerrain"), - ("go2_joystick", "Go2JoystickFlatTerrain"), - ("g1_joystick", "G1JoystickFlatTerrain"), - ("g1_motion_tracking", "G1MotionTracking"), - ("g1_flip_tracking", "G1FlipTracking"), - ("allegro_inhand", "AllegroInhandRotation"), - ], -) -def test_ppo_task_composes(task: str, expected_env: str): - cfg = _compose("ppo", overrides=[f"task={task}"]) - assert cfg.training.task_name == expected_env - - -@pytest.mark.parametrize( - "task,expected_env", - [ - ("go1_joystick", "Go1JoystickFlatTerrain"), - ("go2_joystick", "Go2JoystickFlatTerrain"), - ("g1_joystick", "G1JoystickFlatTerrain"), - ], -) -def test_mlx_ppo_task_composes(task: str, expected_env: str): - cfg = _compose("ppo", "config_mlx", overrides=[f"task={task}"]) - assert cfg.training.task_name == expected_env - - -# --------------------------------------------------------------------------- -# 5. Tasks with non-default reward must populate cfg.reward.scales -# (g1_motion_tracking uses env defaults, excluded here) -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "task", - ["go1_joystick", "g1_sac"], -) -def test_offpolicy_locomotion_reward_populated(task: str): - """Offpolicy locomotion tasks with reward override must have non-empty scales.""" - cfg = _compose("offpolicy", overrides=[f"task={task}"]) - _assert_reward_populated(cfg, "offpolicy", task) - - -@pytest.mark.parametrize( - "task", - ["go1_joystick", "go2_joystick", "g1_joystick"], -) -def test_ppo_locomotion_reward_populated(task: str): - """PPO locomotion tasks must have non-empty reward scales.""" - cfg = _compose("ppo", overrides=[f"task={task}"]) - _assert_reward_populated(cfg, "ppo", task) - - -@pytest.mark.parametrize( - "task", - ["go1_joystick", "go2_joystick", "g1_joystick"], -) -def test_mlx_ppo_locomotion_reward_populated(task: str): - """MLX PPO locomotion tasks must have non-empty reward scales (shared with PPO).""" - cfg = _compose("ppo", "config_mlx", overrides=[f"task={task}"]) - _assert_reward_populated(cfg, "mlx_ppo", task) - - -# --------------------------------------------------------------------------- -# 6. Every reward YAML is loadable (no typos, valid structure) -# --------------------------------------------------------------------------- - - -def _all_reward_yamls(): - """Discover all reward YAML files across all algo directories.""" - results = [] - for algo_dir in ["offpolicy", "appo", "ppo"]: - reward_dir = CONF_DIR / algo_dir / "reward" - if reward_dir.exists(): - for f in sorted(reward_dir.glob("*.yaml")): - results.append((algo_dir, f.stem)) - return results - - -@pytest.mark.parametrize("algo_dir,reward_name", _all_reward_yamls()) -def test_reward_yaml_loadable(algo_dir: str, reward_name: str): - """Every reward YAML file must compose through its supported override path.""" - config_name = "config" - label = f"[{algo_dir}/reward/{reward_name}]" - cfg = _compose(algo_dir, config_name, overrides=[f"reward={reward_name}"]) - if reward_name == "default": - # default.yaml is intentionally empty — cfg.reward won't exist - return - if hasattr(cfg, "reward"): - _assert_reward_section_loadable(cfg.reward, label) - return - - # G1 motion-tracking motrix rewards are sidecar backend-specific configs that - # are selected via `reward@reward_motrix=...` on the task config. - if algo_dir in {"appo", "ppo"} and reward_name == "g1_motion_tracking_motrix": - cfg = _compose( - algo_dir, - config_name, - overrides=[ - "task=g1_motion_tracking", - f"reward@reward_motrix={reward_name}", - ], + assert cfg.training.task_name + assert cfg.training.sim_backend == "mujoco" + + +def test_legacy_config_groups_removed(): + for path in [ + CONF_DIR / "ppo" / "reward", + CONF_DIR / "ppo" / "backend_task_preset", + CONF_DIR / "ppo" / "algo_preset", + CONF_DIR / "ppo" / "sim_backend", + CONF_DIR / "appo" / "reward", + CONF_DIR / "appo" / "backend_task_preset", + CONF_DIR / "appo" / "sim_backend", + CONF_DIR / "offpolicy" / "reward", + CONF_DIR / "offpolicy" / "backend_task_preset", + CONF_DIR / "offpolicy" / "algo_preset", + CONF_DIR / "offpolicy" / "sim_backend", + ]: + assert not path.exists(), f"legacy config group should be removed: {path}" + + +def test_task_files_keep_full_identity(): + for path in sorted(CONF_DIR.glob("*/task/**/*.yaml")): + cfg = OmegaConf.load(path) + cfg_dict_raw = OmegaConf.to_container(cfg, resolve=True) or {} + assert isinstance(cfg_dict_raw, dict) + cfg_dict = cast(dict[str, Any], cfg_dict_raw) + assert "_selected_sim_backend" in cfg_dict, f"task missing backend marker: {path}" + training_raw = cfg_dict.get("training", {}) + assert isinstance(training_raw, dict) + training = cast(dict[str, Any], training_raw) + assert "task_name" in training, f"task missing task_name: {path}" + assert "sim_backend" in training, f"task missing sim_backend: {path}" + assert training["sim_backend"] == cfg_dict["_selected_sim_backend"], ( + f"task backend marker mismatch: {path}" ) - assert hasattr(cfg, "reward_motrix"), f"{label} cfg.reward_motrix missing after compose" - _assert_reward_section_loadable(cfg.reward_motrix, label) - return - - pytest.fail(f"{label} cfg.reward missing after compose") - - -# --------------------------------------------------------------------------- -# 7. Motrix reward override composes correctly via CLI -# --------------------------------------------------------------------------- @pytest.mark.parametrize( - "algo_dir,task,reward", - [ - ("offpolicy", "go1_joystick", "go1_sac_motrix"), - ("offpolicy", "g1_sac", "g1_sac_motrix"), - ("ppo", "go1_joystick", "go1_ppo_motrix"), - ("ppo", "go2_joystick", "go2_ppo_motrix"), - ("ppo", "g1_joystick", "g1_ppo_motrix"), - ], + "algo_dir,config_name,task,backend,task_file,overrides", + _supported_task_cases(), ) -def test_motrix_reward_override(algo_dir: str, task: str, reward: str): - """Motrix reward configs must compose with sim_backend=motrix.""" - cfg = _compose( - algo_dir, - overrides=[f"task={task}", f"reward={reward}", "training.sim_backend=motrix"], - ) - assert cfg.training.sim_backend == "motrix" - _assert_reward_populated(cfg, algo_dir, task) +def test_supported_task_composes( + algo_dir: str, + config_name: str, + task: str, + backend: str, + task_file: str, + overrides: list[str], +): + cfg = _compose(algo_dir, config_name, overrides=overrides) + assert cfg.training.task_name, f"{task_file} should resolve task_name" + assert cfg.training.sim_backend == backend, f"{task_file} should set backend" + _assert_reward_populated(cfg, task_file) -# --------------------------------------------------------------------------- -# 8. Reward config injection end-to-end: Hydra → env_cfg_override → env -# --------------------------------------------------------------------------- +def test_offpolicy_go1_motrix_sac_preserves_legacy_behavior(): + cfg = _compose("offpolicy", overrides=["algo=sac", "task=sac/go1_joystick/motrix"]) -def test_reward_injection_reaches_env_go1(): - """Verify reward config actually reaches the environment via registry.make.""" - from unilab.base import registry - from unilab.utils.algo_utils import ensure_registries + assert cfg.algo.num_envs == 4096 + assert cfg.algo.max_iterations == 2000 + assert cfg.reward.scales.tracking_lin_vel == pytest.approx(1.0) + assert cfg.env.commands.vel_limit == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] - ensure_registries() - cfg = _compose("ppo", overrides=["task=go1_joystick"]) - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override = {"reward_config": reward_dict} +def test_offpolicy_go1_motrix_td3_stays_isolated_from_sac_override(): + cfg = _compose("offpolicy", overrides=["algo=td3", "task=td3/go1_joystick/motrix"]) - env = registry.make( - "Go1JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override=env_cfg_override, - ) - # Verify the injected config matches what we specified - actual_scales = env._cfg.reward_config.scales - expected_scales = reward_dict["scales"] - for key, expected_val in expected_scales.items(): - assert key in actual_scales, f"Missing reward scale key: {key}" - assert actual_scales[key] == pytest.approx(expected_val), ( - f"Reward scale '{key}' mismatch: {actual_scales[key]} != {expected_val}" - ) - env.close() + assert cfg.algo.num_envs == 2048 + assert cfg.algo.max_iterations == 3000 + assert cfg.reward.scales.tracking_lin_vel == pytest.approx(1.0) + assert cfg.env.commands.vel_limit == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_reward_injection_reaches_env_go2(): - """Verify reward config reaches Go2 env.""" - from unilab.base import registry - from unilab.utils.algo_utils import ensure_registries +def test_offpolicy_g1_sac_motrix_preserves_backend_specific_algo_value(): + mujoco_cfg = _compose("offpolicy", overrides=["algo=sac", "task=sac/g1_sac/mujoco"]) + motrix_cfg = _compose("offpolicy", overrides=["algo=sac", "task=sac/g1_sac/motrix"]) - ensure_registries() + assert mujoco_cfg.algo.use_symmetry is True + assert motrix_cfg.algo.use_symmetry is False - cfg = _compose("ppo", overrides=["task=go2_joystick"]) - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override = {"reward_config": reward_dict} - env = registry.make( - "Go2JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override=env_cfg_override, - ) - actual_scales = env._cfg.reward_config.scales - assert actual_scales["tracking_lin_vel"] == pytest.approx(1.0) - env.close() +def test_ppo_g1_backend_specific_hyperparams_remain_separate(): + mujoco_cfg = _compose("ppo", overrides=["task=g1_joystick/mujoco"]) + motrix_cfg = _compose("ppo", overrides=["task=g1_joystick/motrix"]) + assert mujoco_cfg.algo.max_iterations == 220 + assert mujoco_cfg.algo.empirical_normalization is False + assert mujoco_cfg.algo.obs_groups.actor == ["actor"] -def test_go2_ppo_reward_keys_match_env_reward_functions(): - """Go2 PPO reward config must not carry dead/stale reward scale names.""" - from unilab.base import registry - from unilab.utils.algo_utils import ensure_registries + assert motrix_cfg.algo.max_iterations == 151 + assert motrix_cfg.algo.empirical_normalization is True + assert motrix_cfg.algo.obs_groups.actor == ["policy"] + assert motrix_cfg.env.control_config.action_scale == pytest.approx(0.5) - ensure_registries() - cfg = _compose("ppo", overrides=["task=go2_joystick"]) - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override = {"reward_config": reward_dict} +def test_ppo_go1_motrix_preserves_reward_and_algo_values(): + cfg = _compose("ppo", overrides=["task=go1_joystick/motrix"]) - env = registry.make( - "Go2JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override=env_cfg_override, - ) - try: - assert set(reward_dict["scales"]).issubset(env._reward_fns.keys()) - finally: - env.close() + assert cfg.algo.max_iterations == 151 + assert cfg.algo.empirical_normalization is True + assert cfg.algo.policy.init_noise_std == pytest.approx(0.5) + assert cfg.algo.algorithm.learning_rate == pytest.approx(3.0e-4) + assert cfg.reward.scales.tracking_lin_vel == pytest.approx(1.0) + assert cfg.env.commands.vel_limit == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_reward_injection_reaches_env_g1(): - """Verify reward config reaches G1 PPO env.""" - from unilab.base import registry - from unilab.utils.algo_utils import ensure_registries +def test_ppo_go2_motrix_preserves_backend_env_overrides(): + cfg = _compose("ppo", overrides=["task=go2_joystick/motrix"]) - ensure_registries() + assert cfg.algo.num_envs == 1024 + assert cfg.algo.empirical_normalization is True + assert cfg.env.domain_rand.randomize_kp is False + assert cfg.env.domain_rand.randomize_kd is False - cfg = _compose("ppo", overrides=["task=g1_joystick"]) - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override = {"reward_config": reward_dict} - env = registry.make( - "G1JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override=env_cfg_override, - ) - actual_scales = env._cfg.reward_config.scales - assert "feet_phase" in actual_scales, "G1 PPO should have feet_phase scale" - assert actual_scales["tracking_lin_vel"] == pytest.approx(2.0) - env.close() - - -def test_reward_injection_changes_values(): - """Verify that injecting a different reward config actually changes env behavior.""" - from unilab.base import registry - from unilab.utils.algo_utils import ensure_registries - - ensure_registries() - - # Create env with default reward config - default_reward = { - "scales": { - "tracking_lin_vel": 1.0, - "tracking_ang_vel": 0.2, - "lin_vel_z": -5.0, - "ang_vel_xy": -0.1, - "base_height": -100.0, - "action_rate": -0.005, - "similar_to_default": -0.1, - "contact": 0.24, - }, - "tracking_sigma": 0.25, - "base_height_target": 0.3, - } - env_default = registry.make( - "Go1JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override={"reward_config": default_reward}, - ) - default_scales = dict(env_default._cfg.reward_config.scales) - env_default.close() - - # Create env with custom reward (double tracking_lin_vel) - custom_override = { - "reward_config": { - "scales": {**default_scales, "tracking_lin_vel": 999.0}, - "tracking_sigma": 0.25, - "base_height_target": 0.3, - } - } - env_custom = registry.make( - "Go1JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override=custom_override, +def test_cli_override_beats_task_defaults(): + cfg = _compose( + "ppo", + overrides=["task=g1_joystick/motrix", "algo.max_iterations=1"], ) - assert env_custom._cfg.reward_config.scales["tracking_lin_vel"] == pytest.approx(999.0) - env_custom.close() - - -# --------------------------------------------------------------------------- -# 9. Default reward (empty) does not break env creation -# --------------------------------------------------------------------------- - - -def test_default_reward_creates_env_with_hardcoded_defaults(): - """Reward config must be provided - env creation should fail without it.""" - from unilab.base import registry - from unilab.utils.algo_utils import ensure_registries - - ensure_registries() - - # Should fail without reward_config - with pytest.raises(ValueError, match="reward_config must be provided"): - registry.make("Go1JoystickFlatTerrain", num_envs=1, sim_backend="mujoco") - - -# --------------------------------------------------------------------------- -# 10. Reward YAML values match env dataclass defaults (consistency check) -# --------------------------------------------------------------------------- - - -def test_go1_ppo_mujoco_matches_env_defaults(): - """RewardConfig no longer has defaults - this test is obsolete.""" - pytest.skip("RewardConfig defaults removed - all configs must come from Hydra YAML") - - -def test_go2_ppo_mujoco_matches_env_defaults(): - """RewardConfig no longer has defaults - this test is obsolete.""" - pytest.skip("RewardConfig defaults removed - all configs must come from Hydra YAML") - - -def test_g1_ppo_mujoco_matches_env_defaults(): - """RewardConfigPPO no longer has defaults - this test is obsolete.""" - pytest.skip("RewardConfigPPO defaults removed - all configs must come from Hydra YAML") - - -# --------------------------------------------------------------------------- -# 11. Verify training scripts extract reward config correctly (unit test) -# --------------------------------------------------------------------------- - - -def test_offpolicy_reward_extraction_pattern(): - """Verify the reward extraction logic used in train_offpolicy.py.""" - cfg = _compose("offpolicy", overrides=["task=go1_joystick"]) - # Simulate train_offpolicy.py build_runner logic - env_cfg_override = None - if hasattr(cfg, "reward") and cfg.reward: - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override = {"reward_config": reward_dict} - assert env_cfg_override is not None, "env_cfg_override should not be None for go1_joystick" - assert "scales" in env_cfg_override["reward_config"] - assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) - - -def test_ppo_reward_extraction_pattern(): - """Verify the reward extraction logic used in train_rsl_rl.py.""" - cfg = _compose("ppo", overrides=["task=g1_joystick"]) - env_cfg_override: dict = {} - if hasattr(cfg, "reward") and cfg.reward: - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override["reward_config"] = reward_dict - assert "reward_config" in env_cfg_override, "PPO g1_joystick should extract reward" - assert env_cfg_override["reward_config"]["scales"]["feet_phase"] == pytest.approx(1.0) - - -def test_motion_tracking_reward_extraction_pattern(): - """Verify g1_motion_tracking now extracts explicit motion-tracking reward config.""" - cfg = _compose("ppo", overrides=["task=g1_motion_tracking"]) - env_cfg_override: dict = {} - if hasattr(cfg, "reward") and cfg.reward: - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override["reward_config"] = reward_dict - assert "reward_config" in env_cfg_override - assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(1.0) + assert cfg.algo.max_iterations == 1 + assert cfg.algo.empirical_normalization is True -def test_appo_reward_extraction_pattern(): - """Verify APPO go1 task extracts its default reward override.""" - cfg = _compose("appo", overrides=["task=go1_joystick"]) - env_cfg_override: dict = {} - if hasattr(cfg, "reward") and cfg.reward: - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override["reward_config"] = reward_dict - assert "reward_config" in env_cfg_override - assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) +def test_offpolicy_allegro_sac_has_explicit_reward_config(): + cfg = _compose("offpolicy", overrides=["algo=sac", "task=sac/allegro_sac/mujoco"]) -def test_appo_custom_reward_extraction(): - """Verify APPO with explicit reward override extracts correctly.""" - cfg = _compose("appo", overrides=["task=go1_joystick", "reward=go1_appo_mujoco"]) - env_cfg_override: dict = {} - if hasattr(cfg, "reward") and cfg.reward: - reward_dict = OmegaConf.to_container(cfg.reward, resolve=True) - env_cfg_override["reward_config"] = reward_dict - assert "reward_config" in env_cfg_override - assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) + assert cfg.reward.scales.alive == pytest.approx(10.0) + assert cfg.reward.reset_z_threshold == pytest.approx(0.125) diff --git a/tests/config/test_locomotion_params.py b/tests/config/test_locomotion_params.py index defebed49..fd39a2c22 100644 --- a/tests/config/test_locomotion_params.py +++ b/tests/config/test_locomotion_params.py @@ -79,8 +79,7 @@ def test_offpolicy_sac_defaults(): with initialize_config_dir(config_dir=str(CONF_DIR / "offpolicy"), version_base="1.3"): cfg = compose("config") assert cfg.algo.algo == "sac" - # go1_joystick task sets num_envs=4096 (motrix-aligned hyperparams folded into base) - assert cfg.algo.num_envs == 4096 + assert cfg.algo.num_envs == 2048 def test_offpolicy_sac_g1_task_overrides(): @@ -89,7 +88,7 @@ def test_offpolicy_sac_g1_task_overrides(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "offpolicy"), version_base="1.3"): - cfg = compose("config", overrides=["algo=sac", "task=g1_sac"]) + cfg = compose("config", overrides=["algo=sac", "task=sac/g1_sac/mujoco"]) assert cfg.algo.num_envs == 2048 assert cfg.algo.max_iterations == 5000 assert cfg.algo.use_symmetry is True @@ -119,7 +118,7 @@ def test_offpolicy_go2_task_overrides(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "offpolicy"), version_base="1.3"): - cfg = compose("config", overrides=["algo=sac", "task=go2_joystick"]) + cfg = compose("config", overrides=["algo=sac", "task=sac/go2_joystick/mujoco"]) assert cfg.algo.num_envs == 1024 assert cfg.training.task_name == "Go2JoystickFlatTerrain" @@ -146,7 +145,7 @@ def test_appo_g1_task_overrides(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "appo"), version_base="1.3"): - cfg = compose("config", overrides=["task=g1_joystick"]) + cfg = compose("config", overrides=["task=g1_joystick/mujoco"]) assert cfg.algo.max_iterations == 500 assert cfg.algo.save_interval == 100 assert cfg.training.task_name == "G1JoystickFlatTerrain" @@ -163,7 +162,7 @@ def test_ppo_go1_max_iterations(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "ppo"), version_base="1.3"): - cfg = compose("config", overrides=["task=go1_joystick"]) + cfg = compose("config", overrides=["task=go1_joystick/mujoco"]) assert cfg.algo.max_iterations == 151 assert "actor" in cfg.algo.obs_groups @@ -174,10 +173,9 @@ def test_ppo_g1_num_envs(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "ppo"), version_base="1.3"): - cfg = compose("config", overrides=["task=g1_joystick"]) + cfg = compose("config", overrides=["task=g1_joystick/mujoco"]) assert cfg.algo.num_envs == 2048 - # max_iterations=151 (motrix-aligned hyperparams folded into base) - assert cfg.algo.max_iterations == 151 + assert cfg.algo.max_iterations == 220 def test_ppo_go2_num_envs(): @@ -186,7 +184,7 @@ def test_ppo_go2_num_envs(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "ppo"), version_base="1.3"): - cfg = compose("config", overrides=["task=go2_joystick"]) + cfg = compose("config", overrides=["task=go2_joystick/mujoco"]) assert cfg.algo.num_envs == 1024 assert cfg.algo.max_iterations == 151 @@ -197,7 +195,7 @@ def test_ppo_g1_motion_tracking(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "ppo"), version_base="1.3"): - cfg = compose("config", overrides=["task=g1_motion_tracking"]) + cfg = compose("config", overrides=["task=g1_motion_tracking/mujoco"]) assert cfg.training.task_name == "G1MotionTracking" assert cfg.algo.max_iterations == 15000 assert cfg.algo.algorithm.entropy_coef == pytest.approx(0.005) @@ -209,6 +207,6 @@ def test_ppo_g1_flip_tracking(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(CONF_DIR / "ppo"), version_base="1.3"): - cfg = compose("config", overrides=["task=g1_flip_tracking"]) + cfg = compose("config", overrides=["task=g1_flip_tracking/mujoco"]) assert cfg.training.task_name == "G1FlipTracking" assert cfg.algo.max_iterations == 30000 diff --git a/tests/config/test_reward_injection.py b/tests/config/test_reward_injection.py index aa6378857..57d8f6223 100644 --- a/tests/config/test_reward_injection.py +++ b/tests/config/test_reward_injection.py @@ -1,5 +1,7 @@ """Test reward config injection system.""" +from typing import Any, cast + import pytest from hydra import compose, initialize from omegaconf import OmegaConf @@ -8,7 +10,7 @@ def test_reward_config_loading_g1(): """Test G1 SAC reward config loads correctly.""" with initialize(config_path="../../conf/offpolicy", version_base="1.3"): - cfg = compose(config_name="config", overrides=["task=g1_sac"]) + cfg = compose(config_name="config", overrides=["task=sac/g1_sac/mujoco"]) assert hasattr(cfg, "reward") assert cfg.reward.scales.tracking_lin_vel == 2.0 assert cfg.reward.scales.alive == 10.0 @@ -18,20 +20,20 @@ def test_reward_config_loading_g1(): def test_reward_config_loading_go1(): """Test Go1 reward config loads correctly.""" with initialize(config_path="../../conf/offpolicy", version_base="1.3"): - cfg = compose(config_name="config", overrides=["task=go1_joystick"]) + cfg = compose(config_name="config", overrides=["task=sac/go1_joystick/mujoco"]) assert hasattr(cfg, "reward") assert cfg.reward.scales.tracking_lin_vel == 1.0 assert cfg.reward.scales.contact == 0.24 -def test_resolve_reward_dict_unwraps_backend_mounted_reward(): - """Mounted backend reward groups should resolve to the inner reward mapping.""" +def test_resolve_reward_dict_reads_task_reward(): + """Task-backend configs should expose the final reward mapping directly.""" from unilab.utils.reward_utils import resolve_reward_dict with initialize(config_path="../../conf/ppo", version_base="1.3"): cfg = compose( config_name="config", - overrides=["task=go2_joystick", "training.sim_backend=motrix"], + overrides=["task=go2_joystick/motrix"], ) reward_dict = resolve_reward_dict(cfg) @@ -60,11 +62,14 @@ def test_reward_config_conversion(): "close_feet_threshold": 0.15, "pose_weights": [0.01] * 29, } - env = registry.make( - "G1WalkTaskMjSAC", - num_envs=1, - sim_backend="mujoco", - env_cfg_override={"reward_config": g1_dict}, + env = cast( + Any, + registry.make( + "G1WalkTaskMjSAC", + num_envs=1, + sim_backend="mujoco", + env_cfg_override={"reward_config": g1_dict}, + ), ) assert hasattr(env._cfg.reward_config, "scales") assert env._cfg.reward_config.scales["tracking_lin_vel"] == 2.0 @@ -76,11 +81,14 @@ def test_reward_config_conversion(): "tracking_sigma": 0.25, "base_height_target": 0.3, } - env = registry.make( - "Go1JoystickFlatTerrain", - num_envs=1, - sim_backend="mujoco", - env_cfg_override={"reward_config": go1_dict}, + env = cast( + Any, + registry.make( + "Go1JoystickFlatTerrain", + num_envs=1, + sim_backend="mujoco", + env_cfg_override={"reward_config": go1_dict}, + ), ) assert hasattr(env._cfg.reward_config, "scales") assert env._cfg.reward_config.scales["tracking_lin_vel"] == 1.0 diff --git a/tests/integration/test_reward_injection_integration.py b/tests/integration/test_reward_injection_integration.py index adec345d2..042033eee 100644 --- a/tests/integration/test_reward_injection_integration.py +++ b/tests/integration/test_reward_injection_integration.py @@ -1,5 +1,7 @@ """Integration test for reward config injection in training.""" +from typing import Any, cast + import numpy as np import pytest @@ -14,7 +16,7 @@ def test_reward_injection_in_training(): cfg = compose( config_name="config", overrides=[ - "task=g1_sac", + "task=sac/g1_sac/mujoco", "algo.max_iterations=1", "algo.num_envs=64", "training.no_play=true", @@ -56,11 +58,14 @@ def test_reward_override_propagation(): ) # Create env with override - env = registry.make( - "Go1JoystickFlatTerrain", - num_envs=4, - sim_backend="mujoco", - env_cfg_override={"reward_config": custom_config}, + env = cast( + Any, + registry.make( + "Go1JoystickFlatTerrain", + num_envs=4, + sim_backend="mujoco", + env_cfg_override={"reward_config": custom_config}, + ), ) # Verify override was applied @@ -69,7 +74,7 @@ def test_reward_override_propagation(): # Test reward computation uses overridden scales env.init_state() - state = env.reset(np.array([0, 1, 2, 3], dtype=np.int32))[0] + env.reset(np.array([0, 1, 2, 3], dtype=np.int32)) # Take a step and verify reward is computed actions = np.zeros((4, env.action_space.shape[0]), dtype=np.float32) @@ -116,11 +121,14 @@ def test_zero_scale_skips_computation(): base_height_target=0.3, ) - env = registry.make( - "Go1JoystickFlatTerrain", - num_envs=2, - sim_backend="mujoco", - env_cfg_override={"reward_config": custom_config}, + env = cast( + Any, + registry.make( + "Go1JoystickFlatTerrain", + num_envs=2, + sim_backend="mujoco", + env_cfg_override={"reward_config": custom_config}, + ), ) # Verify only non-zero scales are in config diff --git a/tests/scripts/test_train_script_configs.py b/tests/scripts/test_train_script_configs.py index 2587d293e..362c07531 100644 --- a/tests/scripts/test_train_script_configs.py +++ b/tests/scripts/test_train_script_configs.py @@ -13,13 +13,34 @@ pytest.importorskip("mujoco") +def _mlx_runtime_usable() -> bool: + """Probe whether importing mlx.core is safe in a subprocess on this host.""" + if sys.platform != "darwin": + return True + result = subprocess.run( + [sys.executable, "-c", "import mlx.core"], capture_output=True, text=True, timeout=10 + ) + return result.returncode == 0 + + +_MLX_RUNTIME_USABLE = _mlx_runtime_usable() + + @pytest.mark.veryslow @pytest.mark.parametrize( "task", - ["go1_joystick", "go2_joystick", "g1_joystick", "g1_motion_tracking", "g1_flip_tracking"], + [ + "go1_joystick/mujoco", + "go2_joystick/mujoco", + "g1_joystick/mujoco", + "g1_motion_tracking/mujoco", + "g1_flip_tracking/mujoco", + ], ) def test_appo_task_configs_load(task): - """APPO can start training with all task configs.""" + """APPO can start training with all supported task configs.""" + if not _MLX_RUNTIME_USABLE: + pytest.skip("mlx runtime aborts in subprocess on this host") result = subprocess.run( [ sys.executable, @@ -38,10 +59,12 @@ def test_appo_task_configs_load(task): @pytest.mark.veryslow @pytest.mark.parametrize( "task", - ["go1_joystick", "go2_joystick"], + ["sac/go1_joystick/mujoco", "sac/go2_joystick/mujoco"], ) def test_offpolicy_task_configs_load(task): - """Off-policy SAC can start training with all task configs.""" + """Off-policy SAC can start training with all supported task configs.""" + if not _MLX_RUNTIME_USABLE: + pytest.skip("mlx runtime aborts in subprocess on this host") result = subprocess.run( [ sys.executable, diff --git a/tests/scripts/test_train_scripts.py b/tests/scripts/test_train_scripts.py index e75a78fe2..3621606a3 100644 --- a/tests/scripts/test_train_scripts.py +++ b/tests/scripts/test_train_scripts.py @@ -11,6 +11,7 @@ import importlib.util import sys from pathlib import Path +from typing import Any, cast from unittest.mock import MagicMock import pytest @@ -21,7 +22,31 @@ _CONF_DIR = Path(__file__).parent.parent.parent / "conf" -def _load_script(name: str): +def _normalize_overrides(overrides: list[str] | None, *, offpolicy: bool = False) -> list[str]: + normalized: list[str] = [] + algo = "sac" + task_selected = False + + for override in overrides or []: + if override.startswith("algo="): + algo = override.split("=", 1)[1] + normalized.append(override) + continue + if override.startswith("task="): + task_selected = True + normalized.append(override) + continue + normalized.append(override) + + if not task_selected: + if offpolicy: + normalized.append(f"task={algo}/go1_joystick/mujoco") + else: + normalized.append("task=go1_joystick/mujoco") + return normalized + + +def _load_script(name: str) -> Any: """Load a scripts/.py as a fresh module (no __init__ required).""" path = _SCRIPTS_DIR / f"{name}.py" spec = importlib.util.spec_from_file_location(name, path) @@ -37,10 +62,8 @@ def _load_script(name: str): try: import sys as _sys - import mlx.core # noqa: F401 - - _HAS_MLX = _sys.platform == "darwin" -except ImportError: + _HAS_MLX = _sys.platform == "darwin" and importlib.util.find_spec("mlx.core") is not None +except Exception: _HAS_MLX = False try: @@ -59,19 +82,19 @@ def _load_script(name: str): def _offpolicy_cfg(overrides=None): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(_CONF_DIR / "offpolicy"), version_base="1.3"): - return compose("config", overrides=overrides or []) + return compose("config", overrides=_normalize_overrides(overrides, offpolicy=True)) def _ppo_cfg(overrides=None): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(_CONF_DIR / "ppo"), version_base="1.3"): - return compose("config", overrides=overrides or []) + return compose("config", overrides=_normalize_overrides(overrides)) def _appo_cfg(overrides=None): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(_CONF_DIR / "appo"), version_base="1.3"): - return compose("config", overrides=overrides or []) + return compose("config", overrides=_normalize_overrides(overrides)) def _train_rsl_rl(monkeypatch: pytest.MonkeyPatch): @@ -81,9 +104,9 @@ def _train_rsl_rl(monkeypatch: pytest.MonkeyPatch): if module_name == "unilab" or module_name.startswith("unilab."): monkeypatch.delitem(sys.modules, module_name, raising=False) - runners_mod = types.ModuleType("rsl_rl.runners") + runners_mod = cast(Any, types.ModuleType("rsl_rl.runners")) runners_mod.OnPolicyRunner = object - rsl_pkg = types.ModuleType("rsl_rl") + rsl_pkg = cast(Any, types.ModuleType("rsl_rl")) rsl_pkg.runners = runners_mod monkeypatch.setitem(sys.modules, "rsl_rl", rsl_pkg) monkeypatch.setitem(sys.modules, "rsl_rl.runners", runners_mod) @@ -151,41 +174,36 @@ def test_offpolicy_hydra_algo_td3(): def test_offpolicy_go1_resolved_algo_matches_old_motrix_behavior(): - """Equivalence: Motrix SAC Go1 algo hyperparams match pre-refactor values.""" - cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) + """Equivalence: Motrix SAC Go1 algo hyperparams match legacy values.""" + cfg = _offpolicy_cfg(["task=sac/go1_joystick/motrix"]) - # Old motrix_legacy.algo_overrides for Go1 SAC: num_envs=4096, max_iterations=2000 + # Legacy Motrix SAC Go1 values: num_envs=4096, max_iterations=2000 assert cfg.algo.num_envs == 4096 assert cfg.algo.max_iterations == 2000 def test_offpolicy_go1_env_cfg_override_has_reward_and_commands(): - cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) + cfg = _offpolicy_cfg(["task=sac/go1_joystick/motrix"]) env_cfg_override = _offpolicy().build_offpolicy_env_cfg_override("sac", cfg) - # env_cfg_override has reward + backend_profile fields (no bag structure) + # env_cfg_override has reward + env preset fields (no bag structure) assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] -def test_offpolicy_resolve_sac_use_symmetry_keeps_mujoco_setting(): - cfg = _offpolicy_cfg(["task=g1_sac", "training.sim_backend=mujoco"]) - - assert _offpolicy().resolve_sac_use_symmetry(cfg) is True +def test_offpolicy_g1_sac_backend_scoped_use_symmetry(): + mujoco_cfg = _offpolicy_cfg(["task=sac/g1_sac/mujoco"]) + motrix_cfg = _offpolicy_cfg(["task=sac/g1_sac/motrix"]) - -def test_offpolicy_resolve_sac_use_symmetry_disables_motrix(): - cfg = _offpolicy_cfg(["task=g1_sac", "training.sim_backend=motrix"]) - - assert _offpolicy().resolve_sac_use_symmetry(cfg) is False + assert mujoco_cfg.algo.use_symmetry is True + assert motrix_cfg.algo.use_symmetry is False def test_ppo_go1_resolved_algo_matches_old_motrix_behavior(): """Equivalence: PPO Go1 algo hyperparams match pre-refactor motrix values.""" - cfg = _ppo_cfg(["task=go1_joystick"]) + cfg = _ppo_cfg(["task=go1_joystick/motrix"]) - # Old motrix_legacy.algo_overrides for Go1 PPO merged into base (Go2 pattern) assert cfg.algo.max_iterations == 151 assert cfg.algo.empirical_normalization is True assert cfg.algo.policy.init_noise_std == pytest.approx(0.5) @@ -199,7 +217,7 @@ def test_ppo_g1_resolved_algo_matches_old_motrix_behavior(): In particular, max_iterations=151 (the motrix value from the old bag), NOT 220 (the old mujoco base value which has been retired). """ - cfg = _ppo_cfg(["task=g1_joystick"]) + cfg = _ppo_cfg(["task=g1_joystick/motrix"]) assert cfg.algo.max_iterations == 151 assert cfg.algo.empirical_normalization is True @@ -209,16 +227,24 @@ def test_ppo_g1_resolved_algo_matches_old_motrix_behavior(): assert cfg.algo.algorithm.entropy_coef == pytest.approx(5.0e-3) -def test_ppo_g1_backend_profile_has_env_overrides(): - cfg = _ppo_cfg(["task=g1_joystick"]) +def test_ppo_g1_mujoco_base_hyperparams_remain_separate(): + cfg = _ppo_cfg(["task=g1_joystick/mujoco"]) + + assert cfg.algo.max_iterations == 220 + assert cfg.algo.empirical_normalization is False + assert cfg.algo.obs_groups.actor == ["actor"] + - assert cfg.backend_profile.control_config.action_scale == pytest.approx(0.5) - assert cfg.backend_profile.gait_phase_init_mode == "independent" - assert cfg.backend_profile.reset_base_qvel_limit == pytest.approx(0.05) +def test_ppo_g1_env_preset_has_env_overrides(): + cfg = _ppo_cfg(["task=g1_joystick/motrix"]) + + assert cfg.env.control_config.action_scale == pytest.approx(0.5) + assert cfg.env.gait_phase_init_mode == "independent" + assert cfg.env.reset_base_qvel_limit == pytest.approx(0.05) def test_ppo_task_go2_aligns_mujoco_with_motrix_defaults(): - cfg = _ppo_cfg(["task=go2_joystick"]) + cfg = _ppo_cfg(["task=go2_joystick/mujoco"]) assert cfg.algo.num_envs == 1024 assert cfg.reward.scales.tracking_lin_vel == pytest.approx(1.0) @@ -235,11 +261,11 @@ def test_build_ppo_env_cfg_override_go1_motrix( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) - cfg = _ppo_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) + cfg = _ppo_cfg(["task=go1_joystick/motrix"]) env_cfg_override = mod.build_ppo_env_cfg_override(cfg) - # env_cfg_override has reward + backend_profile commands + # env_cfg_override has reward + env preset commands assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] @@ -248,11 +274,11 @@ def test_build_ppo_env_cfg_override_g1_motrix( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) - cfg = _ppo_cfg(["task=g1_joystick", "training.sim_backend=motrix"]) + cfg = _ppo_cfg(["task=g1_joystick/motrix"]) env_cfg_override = mod.build_ppo_env_cfg_override(cfg) - # env_cfg_override has reward + backend_profile fields (flat, matching env cfg structure) + # env_cfg_override has reward + env preset fields (flat, matching env cfg structure) assert env_cfg_override["reward_config"]["scales"]["upper_body_pose"] == pytest.approx(-0.05) assert env_cfg_override["control_config"]["action_scale"] == pytest.approx(0.5) assert env_cfg_override["gait_phase_init_mode"] == "independent" @@ -263,11 +289,11 @@ def test_build_ppo_env_cfg_override_applies_go2_reward_motrix( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) - cfg = _ppo_cfg(["task=go2_joystick", "training.sim_backend=motrix"]) + cfg = _ppo_cfg(["task=go2_joystick/motrix"]) env_cfg_override = mod.build_ppo_env_cfg_override(cfg) - assert cfg.reward_motrix.reward.scales.tracking_lin_vel == pytest.approx(1.0) + assert cfg.reward.scales.tracking_lin_vel == pytest.approx(1.0) assert cfg.algo.num_envs == 1024 assert env_cfg_override["domain_rand"]["randomize_kp"] is False assert env_cfg_override["domain_rand"]["randomize_kd"] is False @@ -279,7 +305,7 @@ def test_ppo_cli_algo_override_wins_over_base( monkeypatch: pytest.MonkeyPatch, ): """CLI override takes precedence over base task algo values via Hydra compose.""" - cfg = _ppo_cfg(["task=g1_joystick", "training.sim_backend=motrix", "algo.max_iterations=1"]) + cfg = _ppo_cfg(["task=g1_joystick/motrix", "algo.max_iterations=1"]) assert cfg.algo.max_iterations == 1 # Other base values remain intact @@ -290,26 +316,21 @@ def test_g1_motion_tracking_ppo_motrix_prefers_backend_specific_reward( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) - cfg = _ppo_cfg(["task=g1_motion_tracking", "training.sim_backend=motrix"]) + cfg = _ppo_cfg(["task=g1_motion_tracking/motrix"]) assert cfg.reward.scales.motion_body_pos == pytest.approx(1.0) - assert cfg.reward_motrix.scales.motion_body_pos == pytest.approx(1.0) - cfg.reward.scales.motion_body_pos = 1.25 - cfg.reward_motrix.scales.motion_body_pos = 2.5 env_cfg_override = mod.build_ppo_env_cfg_override(cfg) - assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(2.5) + assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(1.25) def test_build_ppo_play_env_cfg_override_applies_g1_motion_tracking_play_profile( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) - cfg = _ppo_cfg( - ["task=g1_motion_tracking", "training.sim_backend=motrix", "training.play_only=true"] - ) + cfg = _ppo_cfg(["task=g1_motion_tracking/motrix", "training.play_only=true"]) monkeypatch.setattr( mod, @@ -331,8 +352,7 @@ def test_build_ppo_play_env_cfg_override_respects_cli_play_env_override( mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg( [ - "task=g1_motion_tracking", - "training.sim_backend=motrix", + "task=g1_motion_tracking/motrix", "training.play_only=true", "training.play_env_num=32", ] @@ -354,9 +374,7 @@ def test_build_ppo_play_env_cfg_override_resolves_relative_ground_texture( monkeypatch: pytest.MonkeyPatch, ): mod = _train_rsl_rl(monkeypatch) - cfg = _ppo_cfg( - ["task=g1_motion_tracking", "training.sim_backend=motrix", "training.play_only=true"] - ) + cfg = _ppo_cfg(["task=g1_motion_tracking/motrix", "training.play_only=true"]) cfg.motrix_play_only.scene_override.ground_texture_file = ( "src/unilab/assets/robots/g1/floor.png" ) @@ -442,41 +460,26 @@ def step(self, actions): def test_g1_motion_tracking_appo_reward_extraction_prefers_backend_specific_reward(): from unilab.utils.reward_utils import extract_reward_config - cfg = _appo_cfg(["task=g1_motion_tracking", "training.sim_backend=motrix"]) + cfg = _appo_cfg(["task=g1_motion_tracking/motrix"]) assert cfg.reward.scales.motion_body_pos == pytest.approx(1.0) - assert cfg.reward_motrix.scales.motion_body_pos == pytest.approx(1.0) - cfg.reward.scales.motion_body_pos = 1.5 - cfg.reward_motrix.scales.motion_body_pos = 3.0 env_cfg_override = extract_reward_config(cfg) - assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(3.0) + assert env_cfg_override["reward_config"]["scales"]["motion_body_pos"] == pytest.approx(1.5) -def test_g1_motion_tracking_ppo_can_override_backend_reward_from_reward_group(): - cfg = _ppo_cfg( - [ - "task=g1_motion_tracking", - "training.sim_backend=motrix", - "reward@reward_motrix=g1_motion_tracking_motrix", - ] - ) +def test_g1_motion_tracking_ppo_task_exposes_final_reward(): + cfg = _ppo_cfg(["task=g1_motion_tracking/motrix"]) - assert cfg.reward_motrix.scales.motion_body_pos == pytest.approx(1.0) + assert cfg.reward.scales.motion_body_pos == pytest.approx(1.0) -def test_g1_motion_tracking_appo_can_override_backend_reward_from_reward_group(): - cfg = _appo_cfg( - [ - "task=g1_motion_tracking", - "training.sim_backend=motrix", - "reward@reward_motrix=g1_motion_tracking_motrix", - ] - ) +def test_g1_motion_tracking_appo_task_exposes_final_reward(): + cfg = _appo_cfg(["task=g1_motion_tracking/motrix"]) - assert cfg.reward_motrix.scales.motion_body_pos == pytest.approx(1.0) + assert cfg.reward.scales.motion_body_pos == pytest.approx(1.0) # --------------------------------------------------------------------------- @@ -486,7 +489,7 @@ def test_g1_motion_tracking_appo_can_override_backend_reward_from_reward_group() def test_build_appo_runner_kwargs_forwards_sim_backend(): mod = _train_appo() - cfg = _appo_cfg(["task=g1_motion_tracking", "training.sim_backend=motrix"]) + cfg = _appo_cfg(["task=g1_motion_tracking/motrix"]) runner_kwargs = mod.build_appo_runner_kwargs( cfg, diff --git a/tests/training/test_training_helpers.py b/tests/training/test_training_helpers.py index 36beeaade..5783b0339 100644 --- a/tests/training/test_training_helpers.py +++ b/tests/training/test_training_helpers.py @@ -17,16 +17,40 @@ _CONF_DIR = _ROOT_DIR / "conf" +def _normalize_overrides(overrides: list[str] | None, *, offpolicy: bool = False) -> list[str]: + normalized: list[str] = [] + algo = "sac" + task_selected = False + + for override in overrides or []: + if override.startswith("algo="): + algo = override.split("=", 1)[1] + normalized.append(override) + continue + if override.startswith("task="): + task_selected = True + normalized.append(override) + continue + normalized.append(override) + + if not task_selected: + if offpolicy: + normalized.append(f"task={algo}/go1_joystick/mujoco") + else: + normalized.append("task=go1_joystick/mujoco") + return normalized + + def _ppo_cfg(overrides: list[str] | None = None): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(_CONF_DIR / "ppo"), version_base="1.3"): - return compose("config", overrides=overrides or []) + return compose("config", overrides=_normalize_overrides(overrides)) def _offpolicy_cfg(overrides: list[str] | None = None): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(_CONF_DIR / "offpolicy"), version_base="1.3"): - return compose("config", overrides=overrides or []) + return compose("config", overrides=_normalize_overrides(overrides, offpolicy=True)) def test_get_latest_run_and_checkpoint_support_shared_checkpoint_resolution(tmp_path: Path): @@ -64,13 +88,13 @@ def test_parse_checkpoint_path_uses_algo_log_name_from_cfg(tmp_path: Path): def test_backend_adapter_env_cfg_override_for_motrix_sac_go1(): - """Env cfg override carries reward + backend_profile fields. Algo is NOT touched.""" - cfg = _offpolicy_cfg(["task=go1_joystick", "training.sim_backend=motrix"]) + """Env cfg override carries reward + env preset fields. Algo is NOT touched.""" + cfg = _offpolicy_cfg(["task=sac/go1_joystick/motrix"]) adapter = BackendAdapter(cfg, root_dir=_ROOT_DIR, algo_name="sac") env_cfg_override = adapter.build_task_env_cfg_override() - # env_cfg_override has reward + backend_profile env fields + # env_cfg_override has reward + env preset fields assert env_cfg_override["reward_config"]["scales"]["tracking_lin_vel"] == pytest.approx(1.0) assert env_cfg_override["commands"]["vel_limit"] == [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]] # algo values come straight from YAML compose — no mutation, matches old motrix behavior @@ -79,9 +103,7 @@ def test_backend_adapter_env_cfg_override_for_motrix_sac_go1(): def test_backend_adapter_builds_motrix_play_scene_override(): - cfg = _ppo_cfg( - ["task=g1_motion_tracking", "training.sim_backend=motrix", "training.play_only=true"] - ) + cfg = _ppo_cfg(["task=g1_motion_tracking/motrix", "training.play_only=true"]) captured: dict[str, object] = {} def _fake_materializer(source_model_file: str, **kwargs) -> str: @@ -102,3 +124,15 @@ def _fake_materializer(source_model_file: str, **kwargs) -> str: assert captured["ground_texture_file"] == str( _ROOT_DIR / "src/unilab/assets/robots/g1/floor.png" ) + + +def test_backend_adapter_rejects_direct_training_sim_backend_override(): + GlobalHydra.instance().clear() + with initialize_config_dir(config_dir=str(_CONF_DIR / "offpolicy"), version_base="1.3"): + cfg = compose( + "config", + overrides=["task=sac/go1_joystick/mujoco", "training.sim_backend=motrix"], + ) + + with pytest.raises(ValueError, match="Use `task=...`"): + BackendAdapter(cfg, root_dir=_ROOT_DIR, algo_name="sac").build_task_env_cfg_override() From 2e7606a31c088c29b49a220133090d8be35a74d7 Mon Sep 17 00:00:00 2001 From: tatp-yf Date: Fri, 10 Apr 2026 23:01:42 +0800 Subject: [PATCH 4/5] Clarify task backend selection contract --- AGENTS.md | 2 +- docs/zh_CN/00-development-architecture.md | 3 ++- docs/zh_CN/02-simulation-backends.md | 3 ++- docs/zh_CN/03-training.md | 2 ++ src/unilab/training/backend_adapter.py | 11 ++++++----- tests/training/test_training_helpers.py | 4 ++-- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 87622faca..efd14c518 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ UniLab 是一个 **高性能、模块化、contract 驱动** 的 RL infrastructu | 区域 | 不可破坏的不变量 | |------|----------------| | Env | `NpEnvState.obs` 必须是 dict；`reset()` 返回 `(obs_dict, info_dict)`；`obs_groups_spec` 影响 wrapper 和 learner 维度。 | -| Config / Reward | reward 通过 Hydra 注入；`training.sim_backend` 和 `backend_profile` 必须尊重显式 override。算法超参数直接走 YAML compose，不经 Python 层解释。 | +| Config / Reward | reward 通过 Hydra 注入；后端切换必须通过 `task=/` 选择 owner YAML，`training.sim_backend` 只是 owner YAML 的身份字段，不能单独 override 来切后端。算法超参数直接走 YAML compose，不经 Python 层解释。 | | Backend | backend-specific 逻辑留在 backend / env 适配层，不向训练脚本扩散。 | | Async | 不绕开 runner lifecycle，也不另起 collector / learner 同步协议。 | diff --git a/docs/zh_CN/00-development-architecture.md b/docs/zh_CN/00-development-architecture.md index dde9f03a1..228988ae5 100644 --- a/docs/zh_CN/00-development-architecture.md +++ b/docs/zh_CN/00-development-architecture.md @@ -73,9 +73,10 @@ UniLab 使用 dataclass + Hydra。schema 位于 `src/unilab/config/structured_co - `task` 是唯一 owner 配置入口：同一个 task + backend（offpolicy 再加 algo）对应一个 YAML，里面直接放这个组合的 `training.task_name` / `training.sim_backend` / `reward` / `env` / task-specific `algo` - PPO / APPO 入口形如 `conf/{ppo,appo}/task//.yaml`；offpolicy 入口形如 `conf/offpolicy/task///.yaml` - 这里的 `task` 不是旧设计里“只表达任务、再去别处拼 backend/reward/algo”的 group；它本身就是最终 owner 配置入口 +- `training.sim_backend` 是 task owner YAML 的身份字段，不是独立的 backend switch；切换后端必须改 `task=.../`，不能只 override `training.sim_backend` - reward 必须显式注入 - 如果 backend 选择会影响 task 或 reward 行为，就必须通过 config 表达 -- 动态 override 必须尊重 CLI +- 动态 override 必须尊重 CLI，但不能破坏 task owner 的 backend identity --- diff --git a/docs/zh_CN/02-simulation-backends.md b/docs/zh_CN/02-simulation-backends.md index 5584285e2..6218603e0 100644 --- a/docs/zh_CN/02-simulation-backends.md +++ b/docs/zh_CN/02-simulation-backends.md @@ -36,7 +36,7 @@ UniLab 当前支持两个仿真后端: ## Select A Backend -默认后端是 `mujoco`。通过 Hydra config choice `sim_backend` 切换到 `motrix`。 +默认后端是 `mujoco`。通过 `task=/` 切换到 `motrix`，不要用 `training.sim_backend=motrix` 单独切换后端。 实际要改参数时，不再去拆着找 `reward` / `backend preset` / `algo preset`。直接改对应的 `task` 文件： @@ -44,6 +44,7 @@ UniLab 当前支持两个仿真后端: - offpolicy: `conf/offpolicy/task///.yaml` 现在没有单独的 `reward/`、`backend preset`、`sim_backend/` 配置组。`task/` 是唯一 owner 入口，不再是旧的拆分式 task 配置。 +`training.sim_backend` 由 owner YAML 设置，只用于标识最终选择的后端；如果它和 `task` 路径里的 backend 不一致，训练入口会拒绝这种混合配置。 ```bash # 默认 MuJoCo diff --git a/docs/zh_CN/03-training.md b/docs/zh_CN/03-training.md index 954ffe2dc..6a96145e5 100644 --- a/docs/zh_CN/03-training.md +++ b/docs/zh_CN/03-training.md @@ -78,6 +78,8 @@ algo.num_envs=2048 algo.max_iterations=1000 ``` +`task` 是后端选择入口，例如 `task=go1_joystick/motrix`。`training.sim_backend` 由对应的 task owner YAML 设置，只用于标识最终后端；不要用 `training.sim_backend=motrix` 单独切换后端。 + 查看完整合成配置: ```bash diff --git a/src/unilab/training/backend_adapter.py b/src/unilab/training/backend_adapter.py index 1ebb367f4..a2a30db58 100644 --- a/src/unilab/training/backend_adapter.py +++ b/src/unilab/training/backend_adapter.py @@ -34,21 +34,22 @@ def __init__( def _is_motrix(self) -> bool: return bool(self.cfg.training.sim_backend == "motrix") - def _assert_backend_choice_consistent(self) -> None: + def _assert_task_backend_identity(self) -> None: selected_backend = getattr(self.cfg, "_selected_sim_backend", None) if selected_backend is None: return resolved_backend = str(self.cfg.training.sim_backend) if str(selected_backend) != resolved_backend: raise ValueError( - "Hydra task selection is inconsistent with training.sim_backend. " - "Use `task=...` to switch backends instead of overriding " - "`training.sim_backend` directly." + "Task owner config is inconsistent with training.sim_backend. " + "`task=/` is the backend selection contract; " + "`training.sim_backend` is the selected task owner's identity field, " + "not a standalone backend switch." ) def build_task_env_cfg_override(self) -> dict[str, Any]: """Build env_cfg_override from the resolved reward + env sections.""" - self._assert_backend_choice_consistent() + self._assert_task_backend_identity() env_cfg_override = extract_reward_config(self.cfg) env_cfg_override.update(self._to_plain_dict(getattr(self.cfg, "env", None))) diff --git a/tests/training/test_training_helpers.py b/tests/training/test_training_helpers.py index 5783b0339..465fc80f0 100644 --- a/tests/training/test_training_helpers.py +++ b/tests/training/test_training_helpers.py @@ -126,7 +126,7 @@ def _fake_materializer(source_model_file: str, **kwargs) -> str: ) -def test_backend_adapter_rejects_direct_training_sim_backend_override(): +def test_backend_adapter_rejects_mixed_task_backend_identity(): GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(_CONF_DIR / "offpolicy"), version_base="1.3"): cfg = compose( @@ -134,5 +134,5 @@ def test_backend_adapter_rejects_direct_training_sim_backend_override(): overrides=["task=sac/go1_joystick/mujoco", "training.sim_backend=motrix"], ) - with pytest.raises(ValueError, match="Use `task=...`"): + with pytest.raises(ValueError, match="backend selection contract"): BackendAdapter(cfg, root_dir=_ROOT_DIR, algo_name="sac").build_task_env_cfg_override() From 152e838780a5b676a5df5931eb7eb24709c9c3a0 Mon Sep 17 00:00:00 2001 From: tatp-yf Date: Fri, 10 Apr 2026 23:33:31 +0800 Subject: [PATCH 5/5] Tighten task owner config boundaries --- conf/appo/task/allegro_inhand/mujoco.yaml | 1 - conf/appo/task/g1_flip_tracking/motrix.yaml | 1 - conf/appo/task/g1_flip_tracking/mujoco.yaml | 1 - conf/appo/task/g1_joystick/mujoco.yaml | 1 - conf/appo/task/g1_motion_tracking/motrix.yaml | 1 - conf/appo/task/g1_motion_tracking/mujoco.yaml | 1 - conf/appo/task/go1_joystick/mujoco.yaml | 1 - conf/appo/task/go2_joystick/mujoco.yaml | 1 - .../task/sac/allegro_sac/mujoco.yaml | 1 - conf/offpolicy/task/sac/g1_sac/motrix.yaml | 1 - conf/offpolicy/task/sac/g1_sac/mujoco.yaml | 1 - .../task/sac/go1_joystick/motrix.yaml | 1 - .../task/sac/go1_joystick/mujoco.yaml | 1 - .../task/sac/go2_joystick/motrix.yaml | 1 - .../task/sac/go2_joystick/mujoco.yaml | 1 - .../task/td3/go1_joystick/motrix.yaml | 1 - .../task/td3/go1_joystick/mujoco.yaml | 1 - .../task/td3/go2_joystick/motrix.yaml | 1 - .../task/td3/go2_joystick/mujoco.yaml | 1 - conf/ppo/task/allegro_inhand/mujoco.yaml | 1 - conf/ppo/task/g1_flip_tracking/motrix.yaml | 1 - conf/ppo/task/g1_flip_tracking/mujoco.yaml | 1 - conf/ppo/task/g1_joystick/motrix.yaml | 1 - conf/ppo/task/g1_joystick/mujoco.yaml | 1 - conf/ppo/task/g1_motion_tracking/motrix.yaml | 10 ++-- conf/ppo/task/g1_motion_tracking/mujoco.yaml | 1 - conf/ppo/task/go1_joystick/motrix.yaml | 1 - conf/ppo/task/go1_joystick/mujoco.yaml | 1 - conf/ppo/task/go2_joystick/motrix.yaml | 1 - conf/ppo/task/go2_joystick/mujoco.yaml | 1 - docs/zh_CN/02-simulation-backends.md | 2 +- src/unilab/training/backend_adapter.py | 60 ++----------------- tests/config/test_config_system.py | 18 +++--- tests/scripts/test_train_scripts.py | 7 +-- tests/training/test_training_helpers.py | 15 +---- 35 files changed, 23 insertions(+), 118 deletions(-) diff --git a/conf/appo/task/allegro_inhand/mujoco.yaml b/conf/appo/task/allegro_inhand/mujoco.yaml index c70791397..19ebd4df2 100644 --- a/conf/appo/task/allegro_inhand/mujoco.yaml +++ b/conf/appo/task/allegro_inhand/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: AllegroInhandRotation sim_backend: mujoco diff --git a/conf/appo/task/g1_flip_tracking/motrix.yaml b/conf/appo/task/g1_flip_tracking/motrix.yaml index fabf421d0..7767bc52c 100644 --- a/conf/appo/task/g1_flip_tracking/motrix.yaml +++ b/conf/appo/task/g1_flip_tracking/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: G1FlipTracking sim_backend: motrix diff --git a/conf/appo/task/g1_flip_tracking/mujoco.yaml b/conf/appo/task/g1_flip_tracking/mujoco.yaml index 223f7a98a..6ddf5fe3f 100644 --- a/conf/appo/task/g1_flip_tracking/mujoco.yaml +++ b/conf/appo/task/g1_flip_tracking/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1FlipTracking sim_backend: mujoco diff --git a/conf/appo/task/g1_joystick/mujoco.yaml b/conf/appo/task/g1_joystick/mujoco.yaml index 945e03061..95412f145 100644 --- a/conf/appo/task/g1_joystick/mujoco.yaml +++ b/conf/appo/task/g1_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/appo/task/g1_motion_tracking/motrix.yaml b/conf/appo/task/g1_motion_tracking/motrix.yaml index cfa4dae4d..567b0d745 100644 --- a/conf/appo/task/g1_motion_tracking/motrix.yaml +++ b/conf/appo/task/g1_motion_tracking/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: G1MotionTracking sim_backend: motrix diff --git a/conf/appo/task/g1_motion_tracking/mujoco.yaml b/conf/appo/task/g1_motion_tracking/mujoco.yaml index d39117b09..637ea275c 100644 --- a/conf/appo/task/g1_motion_tracking/mujoco.yaml +++ b/conf/appo/task/g1_motion_tracking/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1MotionTracking sim_backend: mujoco diff --git a/conf/appo/task/go1_joystick/mujoco.yaml b/conf/appo/task/go1_joystick/mujoco.yaml index 2e604807f..b77e2215d 100644 --- a/conf/appo/task/go1_joystick/mujoco.yaml +++ b/conf/appo/task/go1_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go1JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/appo/task/go2_joystick/mujoco.yaml b/conf/appo/task/go2_joystick/mujoco.yaml index 45256da05..8344a74c7 100644 --- a/conf/appo/task/go2_joystick/mujoco.yaml +++ b/conf/appo/task/go2_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go2JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/offpolicy/task/sac/allegro_sac/mujoco.yaml b/conf/offpolicy/task/sac/allegro_sac/mujoco.yaml index 20845bd76..88f8c1d52 100644 --- a/conf/offpolicy/task/sac/allegro_sac/mujoco.yaml +++ b/conf/offpolicy/task/sac/allegro_sac/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: AllegroInhandRotationSac sim_backend: mujoco diff --git a/conf/offpolicy/task/sac/g1_sac/motrix.yaml b/conf/offpolicy/task/sac/g1_sac/motrix.yaml index 63721337e..2a2a7279c 100644 --- a/conf/offpolicy/task/sac/g1_sac/motrix.yaml +++ b/conf/offpolicy/task/sac/g1_sac/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: G1WalkTaskMjSAC sim_backend: motrix diff --git a/conf/offpolicy/task/sac/g1_sac/mujoco.yaml b/conf/offpolicy/task/sac/g1_sac/mujoco.yaml index a6feb5b49..997fd5c1d 100644 --- a/conf/offpolicy/task/sac/g1_sac/mujoco.yaml +++ b/conf/offpolicy/task/sac/g1_sac/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1WalkTaskMjSAC sim_backend: mujoco diff --git a/conf/offpolicy/task/sac/go1_joystick/motrix.yaml b/conf/offpolicy/task/sac/go1_joystick/motrix.yaml index 4bcad056c..03b0fe931 100644 --- a/conf/offpolicy/task/sac/go1_joystick/motrix.yaml +++ b/conf/offpolicy/task/sac/go1_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: Go1JoystickFlatTerrain sim_backend: motrix diff --git a/conf/offpolicy/task/sac/go1_joystick/mujoco.yaml b/conf/offpolicy/task/sac/go1_joystick/mujoco.yaml index 8ecc25d87..2072e9821 100644 --- a/conf/offpolicy/task/sac/go1_joystick/mujoco.yaml +++ b/conf/offpolicy/task/sac/go1_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go1JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/offpolicy/task/sac/go2_joystick/motrix.yaml b/conf/offpolicy/task/sac/go2_joystick/motrix.yaml index 23e54c0b9..60919ec35 100644 --- a/conf/offpolicy/task/sac/go2_joystick/motrix.yaml +++ b/conf/offpolicy/task/sac/go2_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: Go2JoystickFlatTerrain sim_backend: motrix diff --git a/conf/offpolicy/task/sac/go2_joystick/mujoco.yaml b/conf/offpolicy/task/sac/go2_joystick/mujoco.yaml index af559c653..4ef0fbde3 100644 --- a/conf/offpolicy/task/sac/go2_joystick/mujoco.yaml +++ b/conf/offpolicy/task/sac/go2_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go2JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/offpolicy/task/td3/go1_joystick/motrix.yaml b/conf/offpolicy/task/td3/go1_joystick/motrix.yaml index 4c7c21278..870ea7504 100644 --- a/conf/offpolicy/task/td3/go1_joystick/motrix.yaml +++ b/conf/offpolicy/task/td3/go1_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: Go1JoystickFlatTerrain sim_backend: motrix diff --git a/conf/offpolicy/task/td3/go1_joystick/mujoco.yaml b/conf/offpolicy/task/td3/go1_joystick/mujoco.yaml index 8ecc25d87..2072e9821 100644 --- a/conf/offpolicy/task/td3/go1_joystick/mujoco.yaml +++ b/conf/offpolicy/task/td3/go1_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go1JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/offpolicy/task/td3/go2_joystick/motrix.yaml b/conf/offpolicy/task/td3/go2_joystick/motrix.yaml index 23e54c0b9..60919ec35 100644 --- a/conf/offpolicy/task/td3/go2_joystick/motrix.yaml +++ b/conf/offpolicy/task/td3/go2_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: Go2JoystickFlatTerrain sim_backend: motrix diff --git a/conf/offpolicy/task/td3/go2_joystick/mujoco.yaml b/conf/offpolicy/task/td3/go2_joystick/mujoco.yaml index af559c653..4ef0fbde3 100644 --- a/conf/offpolicy/task/td3/go2_joystick/mujoco.yaml +++ b/conf/offpolicy/task/td3/go2_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go2JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/ppo/task/allegro_inhand/mujoco.yaml b/conf/ppo/task/allegro_inhand/mujoco.yaml index ea0347243..fa7dee11a 100644 --- a/conf/ppo/task/allegro_inhand/mujoco.yaml +++ b/conf/ppo/task/allegro_inhand/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: AllegroInhandRotation sim_backend: mujoco diff --git a/conf/ppo/task/g1_flip_tracking/motrix.yaml b/conf/ppo/task/g1_flip_tracking/motrix.yaml index 733d98ee3..1d17d213c 100644 --- a/conf/ppo/task/g1_flip_tracking/motrix.yaml +++ b/conf/ppo/task/g1_flip_tracking/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: G1FlipTracking sim_backend: motrix diff --git a/conf/ppo/task/g1_flip_tracking/mujoco.yaml b/conf/ppo/task/g1_flip_tracking/mujoco.yaml index 70cd1265a..21ca2a14d 100644 --- a/conf/ppo/task/g1_flip_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_flip_tracking/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1FlipTracking sim_backend: mujoco diff --git a/conf/ppo/task/g1_joystick/motrix.yaml b/conf/ppo/task/g1_joystick/motrix.yaml index f39268be8..5e6d7b1d5 100644 --- a/conf/ppo/task/g1_joystick/motrix.yaml +++ b/conf/ppo/task/g1_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: G1JoystickFlatTerrain sim_backend: motrix diff --git a/conf/ppo/task/g1_joystick/mujoco.yaml b/conf/ppo/task/g1_joystick/mujoco.yaml index 3e27ae247..901d8cfeb 100644 --- a/conf/ppo/task/g1_joystick/mujoco.yaml +++ b/conf/ppo/task/g1_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/ppo/task/g1_motion_tracking/motrix.yaml b/conf/ppo/task/g1_motion_tracking/motrix.yaml index 40ae40e30..edc4b9a1e 100644 --- a/conf/ppo/task/g1_motion_tracking/motrix.yaml +++ b/conf/ppo/task/g1_motion_tracking/motrix.yaml @@ -1,8 +1,8 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: G1MotionTracking sim_backend: motrix + play_env_num: 128 play_steps: 1000 algo: num_envs: 1024 @@ -13,13 +13,11 @@ algo: - actor algorithm: entropy_coef: 0.005 -motrix_play_only: +play_profile: enabled: true - training_overrides: - play_env_num: 128 - env_cfg_override: + env: render_spacing: 2.5 - scene_override: + scene: enabled: true source_model_file: src/unilab/assets/robots/g1/scene_flat.xml ground_texture_file: src/unilab/assets/robots/g1/floor.png diff --git a/conf/ppo/task/g1_motion_tracking/mujoco.yaml b/conf/ppo/task/g1_motion_tracking/mujoco.yaml index ea89d13b6..f257cd9e6 100644 --- a/conf/ppo/task/g1_motion_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_motion_tracking/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: G1MotionTracking sim_backend: mujoco diff --git a/conf/ppo/task/go1_joystick/motrix.yaml b/conf/ppo/task/go1_joystick/motrix.yaml index 41d118007..8cd8232dd 100644 --- a/conf/ppo/task/go1_joystick/motrix.yaml +++ b/conf/ppo/task/go1_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: Go1JoystickFlatTerrain sim_backend: motrix diff --git a/conf/ppo/task/go1_joystick/mujoco.yaml b/conf/ppo/task/go1_joystick/mujoco.yaml index ae3b409e5..729027109 100644 --- a/conf/ppo/task/go1_joystick/mujoco.yaml +++ b/conf/ppo/task/go1_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go1JoystickFlatTerrain sim_backend: mujoco diff --git a/conf/ppo/task/go2_joystick/motrix.yaml b/conf/ppo/task/go2_joystick/motrix.yaml index 44095ce5c..d888525ed 100644 --- a/conf/ppo/task/go2_joystick/motrix.yaml +++ b/conf/ppo/task/go2_joystick/motrix.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: motrix training: task_name: Go2JoystickFlatTerrain sim_backend: motrix diff --git a/conf/ppo/task/go2_joystick/mujoco.yaml b/conf/ppo/task/go2_joystick/mujoco.yaml index 68b5ae467..4d8990483 100644 --- a/conf/ppo/task/go2_joystick/mujoco.yaml +++ b/conf/ppo/task/go2_joystick/mujoco.yaml @@ -1,5 +1,4 @@ # @package _global_ -_selected_sim_backend: mujoco training: task_name: Go2JoystickFlatTerrain sim_backend: mujoco diff --git a/docs/zh_CN/02-simulation-backends.md b/docs/zh_CN/02-simulation-backends.md index 6218603e0..50af4016a 100644 --- a/docs/zh_CN/02-simulation-backends.md +++ b/docs/zh_CN/02-simulation-backends.md @@ -44,7 +44,7 @@ UniLab 当前支持两个仿真后端: - offpolicy: `conf/offpolicy/task///.yaml` 现在没有单独的 `reward/`、`backend preset`、`sim_backend/` 配置组。`task/` 是唯一 owner 入口，不再是旧的拆分式 task 配置。 -`training.sim_backend` 由 owner YAML 设置，只用于标识最终选择的后端；如果它和 `task` 路径里的 backend 不一致，训练入口会拒绝这种混合配置。 +`training.sim_backend` 由 owner YAML 设置，只用于标识最终选择的后端；不要把它当作独立 backend switch。 ```bash # 默认 MuJoCo diff --git a/src/unilab/training/backend_adapter.py b/src/unilab/training/backend_adapter.py index a2a30db58..18008750f 100644 --- a/src/unilab/training/backend_adapter.py +++ b/src/unilab/training/backend_adapter.py @@ -1,8 +1,7 @@ -"""Backend-specific config adaptation for training entrypoints.""" +"""Resolved config adaptation for training entrypoints.""" from __future__ import annotations -import sys from pathlib import Path from typing import Any, Callable @@ -27,65 +26,36 @@ def __init__( self.root_dir = Path(root_dir) self.algo_name = algo_name self.scene_materializer = scene_materializer - self.explicit_keys = { - arg.split("=", 1)[0].lstrip("+") for arg in sys.argv[1:] if "=" in arg - } - - def _is_motrix(self) -> bool: - return bool(self.cfg.training.sim_backend == "motrix") - - def _assert_task_backend_identity(self) -> None: - selected_backend = getattr(self.cfg, "_selected_sim_backend", None) - if selected_backend is None: - return - resolved_backend = str(self.cfg.training.sim_backend) - if str(selected_backend) != resolved_backend: - raise ValueError( - "Task owner config is inconsistent with training.sim_backend. " - "`task=/` is the backend selection contract; " - "`training.sim_backend` is the selected task owner's identity field, " - "not a standalone backend switch." - ) def build_task_env_cfg_override(self) -> dict[str, Any]: """Build env_cfg_override from the resolved reward + env sections.""" - self._assert_task_backend_identity() env_cfg_override = extract_reward_config(self.cfg) env_cfg_override.update(self._to_plain_dict(getattr(self.cfg, "env", None))) return env_cfg_override def build_play_env_cfg_override(self) -> dict[str, Any]: - """Build play-mode overrides, including Motrix scene customization when configured.""" + """Build play-mode overrides from an optional backend-agnostic play profile.""" env_cfg_override = self.build_task_env_cfg_override() - play_profile = getattr(self.cfg, "motrix_play_only", None) + play_profile = getattr(self.cfg, "play_profile", None) if ( play_profile is None or not getattr(play_profile, "enabled", False) - or not self._is_motrix() or not self.cfg.training.play_only ): return env_cfg_override - training_overrides = getattr(play_profile, "training_overrides", None) - if training_overrides is not None: - self._apply_nested_overrides( - self.cfg.training, - training_overrides, - base_path="training", - ) - - env_profile = getattr(play_profile, "env_cfg_override", None) + env_profile = getattr(play_profile, "env", None) if env_profile is not None: self._apply_env_profile(env_cfg_override, env_profile) - scene_override = getattr(play_profile, "scene_override", None) + scene_override = getattr(play_profile, "scene", None) if scene_override is None or not getattr(scene_override, "enabled", False): return env_cfg_override source_model_file = getattr(scene_override, "source_model_file", None) if not source_model_file: - raise ValueError("motrix_play_only.scene_override.source_model_file must be configured") + raise ValueError("play_profile.scene.source_model_file must be configured") env_cfg_override["model_file"] = self.scene_materializer( self._resolve_root_relative_path(str(source_model_file)), @@ -100,24 +70,6 @@ def build_play_env_cfg_override(self) -> dict[str, Any]: ) return env_cfg_override - def _apply_nested_overrides(self, target: Any, overrides: Any, *, base_path: str) -> None: - override_dict = self._to_plain_dict(overrides) - if not isinstance(override_dict, dict): - return - - for key, value in override_dict.items(): - key_str = str(key) - path = f"{base_path}.{key_str}" - if isinstance(value, dict): - if path in self.explicit_keys: - continue - if key_str not in target or target[key_str] is None: - target[key_str] = {} - self._apply_nested_overrides(target[key_str], value, base_path=path) - continue - if path not in self.explicit_keys: - target[key_str] = value - def _apply_env_profile(self, env_cfg_override: dict[str, Any], env_profile: Any) -> None: env_cfg_override.update(self._to_plain_dict(env_profile)) diff --git a/tests/config/test_config_system.py b/tests/config/test_config_system.py index f1933357a..51517a98a 100644 --- a/tests/config/test_config_system.py +++ b/tests/config/test_config_system.py @@ -10,7 +10,6 @@ from __future__ import annotations from pathlib import Path -from typing import Any, cast import pytest from hydra import compose, initialize_config_dir @@ -144,21 +143,18 @@ def test_legacy_config_groups_removed(): assert not path.exists(), f"legacy config group should be removed: {path}" -def test_task_files_keep_full_identity(): +def test_task_files_keep_full_identity_without_hidden_backend_marker(): for path in sorted(CONF_DIR.glob("*/task/**/*.yaml")): cfg = OmegaConf.load(path) cfg_dict_raw = OmegaConf.to_container(cfg, resolve=True) or {} assert isinstance(cfg_dict_raw, dict) - cfg_dict = cast(dict[str, Any], cfg_dict_raw) - assert "_selected_sim_backend" in cfg_dict, f"task missing backend marker: {path}" - training_raw = cfg_dict.get("training", {}) - assert isinstance(training_raw, dict) - training = cast(dict[str, Any], training_raw) - assert "task_name" in training, f"task missing task_name: {path}" - assert "sim_backend" in training, f"task missing sim_backend: {path}" - assert training["sim_backend"] == cfg_dict["_selected_sim_backend"], ( - f"task backend marker mismatch: {path}" + assert "_selected_sim_backend" not in cfg_dict_raw, ( + f"task has hidden backend marker: {path}" ) + training_raw = cfg_dict_raw.get("training", {}) + assert isinstance(training_raw, dict) + assert "task_name" in training_raw, f"task missing task_name: {path}" + assert "sim_backend" in training_raw, f"task missing sim_backend: {path}" @pytest.mark.parametrize( diff --git a/tests/scripts/test_train_scripts.py b/tests/scripts/test_train_scripts.py index 134aa7be3..32c664dcd 100644 --- a/tests/scripts/test_train_scripts.py +++ b/tests/scripts/test_train_scripts.py @@ -344,6 +344,7 @@ def test_build_ppo_play_env_cfg_override_applies_g1_motion_tracking_play_profile ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=g1_motion_tracking/motrix", "training.play_only=true"]) + assert cfg.training.play_env_num == 128 monkeypatch.setattr( mod, @@ -370,7 +371,7 @@ def test_build_ppo_play_env_cfg_override_respects_cli_play_env_override( "training.play_env_num=32", ] ) - monkeypatch.setattr(sys, "argv", ["train_rsl_rl.py", "training.play_env_num=32"]) + assert cfg.training.play_env_num == 32 monkeypatch.setattr( mod, "materialize_scene_visual_override", @@ -388,9 +389,7 @@ def test_build_ppo_play_env_cfg_override_resolves_relative_ground_texture( ): mod = _train_rsl_rl(monkeypatch) cfg = _ppo_cfg(["task=g1_motion_tracking/motrix", "training.play_only=true"]) - cfg.motrix_play_only.scene_override.ground_texture_file = ( - "src/unilab/assets/robots/g1/floor.png" - ) + cfg.play_profile.scene.ground_texture_file = "src/unilab/assets/robots/g1/floor.png" captured = {} diff --git a/tests/training/test_training_helpers.py b/tests/training/test_training_helpers.py index 465fc80f0..94d5adb30 100644 --- a/tests/training/test_training_helpers.py +++ b/tests/training/test_training_helpers.py @@ -102,8 +102,9 @@ def test_backend_adapter_env_cfg_override_for_motrix_sac_go1(): assert cfg.algo.max_iterations == 2000 -def test_backend_adapter_builds_motrix_play_scene_override(): +def test_backend_adapter_builds_play_scene_override(): cfg = _ppo_cfg(["task=g1_motion_tracking/motrix", "training.play_only=true"]) + assert cfg.training.play_env_num == 128 captured: dict[str, object] = {} def _fake_materializer(source_model_file: str, **kwargs) -> str: @@ -124,15 +125,3 @@ def _fake_materializer(source_model_file: str, **kwargs) -> str: assert captured["ground_texture_file"] == str( _ROOT_DIR / "src/unilab/assets/robots/g1/floor.png" ) - - -def test_backend_adapter_rejects_mixed_task_backend_identity(): - GlobalHydra.instance().clear() - with initialize_config_dir(config_dir=str(_CONF_DIR / "offpolicy"), version_base="1.3"): - cfg = compose( - "config", - overrides=["task=sac/go1_joystick/mujoco", "training.sim_backend=motrix"], - ) - - with pytest.raises(ValueError, match="backend selection contract"): - BackendAdapter(cfg, root_dir=_ROOT_DIR, algo_name="sac").build_task_env_cfg_override()