diff --git a/docs/en/get_started/usage.md b/docs/en/get_started/usage.md index b0ced8e08..bdc10182a 100644 --- a/docs/en/get_started/usage.md +++ b/docs/en/get_started/usage.md @@ -197,6 +197,7 @@ The recommended contract is to put the source identifier in `metadata["source_na - `grpo` ([https://arxiv.org/abs/2402.03300](https://arxiv.org/abs/2402.03300)) - `gspo` ([https://arxiv.org/abs/2507.18071](https://arxiv.org/abs/2507.18071)) - `cispo` ([https://arxiv.org/abs/2506.13585](https://arxiv.org/abs/2506.13585)) + - `dapo` ([https://arxiv.org/abs/2503.14476](https://arxiv.org/abs/2503.14476)) - `reinforce_plus_plus` and `reinforce_plus_plus_baseline` ([https://arxiv.org/abs/2501.03262](https://arxiv.org/abs/2501.03262)) - `ppo` ([https://arxiv.org/abs/1707.06347](https://arxiv.org/abs/1707.06347)) - `--calculate-per-token-loss`: By default, vime calculates loss on a per-sample basis, i.e., `mean(sum(sample_i) / len(sample_i))`. Enable this flag to calculate loss on a per-token basis, i.e., `sum(sum(sample_i)) / sum(len(sample_i))`. @@ -224,6 +225,16 @@ Related parameters: - `--normalize-advantages`: Whether to normalize advantages. - `--eps-clip`: PPO-style clip range. +#### DAPO Algorithm + +DAPO (Decoupled Clip and Dynamic sAmpling Policy Optimization, https://arxiv.org/abs/2503.14476) is exposed as a preset over the existing GRPO implementation. Set: + +```bash +--advantage-estimator dapo +``` + +This selects GRPO with Clip-Higher (`--eps-clip-high 0.28`), token-level loss, the default dynamic-sampling filter, and a Soft Overlong window of one quarter of `--rollout-max-response-len`. Set `--soft-overlong-cache 0` to disable the length penalty. A custom reward post-process replaces the built-in reward shaping. + #### PPO Algorithm PPO (Proximal Policy Optimization) is a classic RL algorithm that uses a critic model to estimate the value function for computing advantages. diff --git a/docs/zh/get_started/usage.md b/docs/zh/get_started/usage.md index 92e95eab7..3801b57cb 100644 --- a/docs/zh/get_started/usage.md +++ b/docs/zh/get_started/usage.md @@ -199,6 +199,7 @@ vLLM 的加载非常简单,只需要: - `grpo`(https://arxiv.org/abs/2402.03300); - `gspo`(https://arxiv.org/abs/2507.18071); - `cispo`(https://arxiv.org/abs/2506.13585); + - `dapo`(https://arxiv.org/abs/2503.14476); - `reinforce_plus_plus` 与 `reinforce_plus_plus_baseline`(https://arxiv.org/abs/2501.03262); - `ppo`(https://arxiv.org/abs/1707.06347)。 - `--calculate-per-token-loss`:vime 中默认的方案是 per sample loss,即 `mean(sum(sample_i) / len(sample_i))`,如果需要计算 per token loss,即 `sum(sum(sample_i)) / sum(len(sample_i))`,可以开启 `--calculate-per-token-loss`; @@ -226,6 +227,16 @@ GRPO 的主要特点: - `--normalize-advantages`:是否对 advantage 进行归一化; - `--eps-clip`:PPO 风格的 clip 范围。 +#### DAPO 算法 + +DAPO(Decoupled Clip and Dynamic sAmpling Policy Optimization,https://arxiv.org/abs/2503.14476)作为现有 GRPO 实现的预设提供。使用: + +```bash +--advantage-estimator dapo +``` + +该预设使用 GRPO,并启用 Clip-Higher(`--eps-clip-high 0.28`)、token-level loss、默认 dynamic-sampling filter,以及长度为 `--rollout-max-response-len` 四分之一的 Soft Overlong 区间。设置 `--soft-overlong-cache 0` 可关闭长度惩罚;自定义 reward post-process 会替代内置 reward shaping。 + #### PPO 算法 PPO(Proximal Policy Optimization)是经典的 RL 算法,使用 critic 模型来估计 value function,从而计算 advantage。 diff --git a/vime/ray/rollout.py b/vime/ray/rollout.py index 17e823536..a2054ae02 100644 --- a/vime/ray/rollout.py +++ b/vime/ray/rollout.py @@ -757,12 +757,20 @@ def _post_process_rewards(self, samples: list[Sample] | list[list[Sample]]): return self.custom_reward_post_process_func(self.args, samples) raw_rewards = [sample.get_reward_value(self.args) for sample in samples] + rewards_for_norm = raw_rewards + if cache := getattr(self.args, "soft_overlong_cache", 0): + max_len = self.args.rollout_max_response_len + rewards_for_norm = [ + float(reward) - min(1.0, max(0.0, (sample.response_length - max_len + cache) / cache)) + for sample, reward in zip(samples, raw_rewards, strict=True) + ] + if ( self.args.advantage_estimator in ["grpo", "gspo", "cispo", "reinforce_plus_plus_baseline"] and self.args.rewards_normalization ): # group norm - rewards = torch.tensor(raw_rewards, dtype=torch.float) + rewards = torch.tensor(rewards_for_norm, dtype=torch.float) if rewards.shape[-1] == self.args.n_samples_per_prompt * self.args.rollout_batch_size: rewards = rewards.reshape(-1, self.args.n_samples_per_prompt) else: @@ -777,7 +785,7 @@ def _post_process_rewards(self, samples: list[Sample] | list[list[Sample]]): return raw_rewards, rewards.flatten().tolist() - return raw_rewards, raw_rewards + return raw_rewards, rewards_for_norm def _convert_samples_to_train_data(self, samples: list[Sample] | list[list[Sample]]): """ diff --git a/vime/utils/arguments.py b/vime/utils/arguments.py index 113ab12a0..661182a7e 100644 --- a/vime/utils/arguments.py +++ b/vime/utils/arguments.py @@ -908,6 +908,12 @@ def add_algo_arguments(parser): parser.add_argument("--eps-clip", type=float, default=0.2, help="PPO clip range") parser.add_argument("--eps-clip-high", type=float, default=None, help="PPO clip upper range") + parser.add_argument( + "--soft-overlong-cache", + type=int, + default=None, + help="DAPO Soft Overlong penalty window; 0 disables it.", + ) parser.add_argument( "--eps-clip-c", type=float, @@ -954,6 +960,7 @@ def add_algo_arguments(parser): "grpo", "gspo", "cispo", + "dapo", "reinforce_plus_plus", "reinforce_plus_plus_baseline", "ppo", @@ -1889,9 +1896,29 @@ def vime_validate_args(args): assert args.use_dynamic_batch_size, "--balance-by-flops requires --use-dynamic-batch-size" args.balance_data = True + if args.advantage_estimator == "dapo": + args.advantage_estimator = "grpo" + if args.eps_clip_high is None: + args.eps_clip_high = 0.28 + args.calculate_per_token_loss = True + if args.dynamic_sampling_filter_path is None: + args.dynamic_sampling_filter_path = ( + "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std" + ) + if args.soft_overlong_cache is None: + if args.rollout_max_response_len is None: + raise ValueError("DAPO requires --rollout-max-response-len or --soft-overlong-cache 0.") + args.soft_overlong_cache = max(1, int(args.rollout_max_response_len) // 4) + if args.eps_clip_high is None: args.eps_clip_high = args.eps_clip + soft_overlong_cache = getattr(args, "soft_overlong_cache", None) + if soft_overlong_cache and ( + args.rollout_max_response_len is None or not 0 < soft_overlong_cache <= args.rollout_max_response_len + ): + raise ValueError("--soft-overlong-cache must be between 0 and --rollout-max-response-len.") + if args.advantage_estimator == "cispo" and args.eps_clip < 1.0: logger.warning( "CISPO is canonically single-sided, but --eps-clip=%s keeps the lower clip bound %s active. "