From a2a332ad5eb9dc8168b9ea4e4454d4fdce588bd5 Mon Sep 17 00:00:00 2001
From: kaiyuan <kyxiezju@163.com>
Date: Wed, 22 Jul 2026 10:28:53 +0800
Subject: [PATCH 1/5] Add DAPO advantage estimator and Soft Overlong support.

Signed-off-by: kaiyuan <kyxiezju@163.com>
---
 docs/en/get_started/usage.md               | 33 +++++++++++++
 docs/zh/get_started/usage.md               | 33 +++++++++++++
 tests/test_megatron_argument_validation.py | 27 +++++++++++
 vime/backends/megatron_utils/loss.py       |  4 +-
 vime/ray/rollout.py                        | 27 +++++++++--
 vime/utils/arguments.py                    | 56 ++++++++++++++++++++++
 6 files changed, 174 insertions(+), 6 deletions(-)

diff --git a/docs/en/get_started/usage.md b/docs/en/get_started/usage.md
index 53f00b8a7..0510d84b0 100644
--- a/docs/en/get_started/usage.md
+++ b/docs/en/get_started/usage.md
@@ -197,6 +197,7 @@ The recommended contract is to put the source identifier in `metadata["source_na
     - `grpo` ([https://arxiv.org/abs/2402.03300](https://arxiv.org/abs/2402.03300))
     - `gspo` ([https://arxiv.org/abs/2507.18071](https://arxiv.org/abs/2507.18071))
     - `cispo` ([https://arxiv.org/abs/2506.13585](https://arxiv.org/abs/2506.13585))
+    - `dapo` ([https://arxiv.org/abs/2503.14476](https://arxiv.org/abs/2503.14476))
     - `reinforce_plus_plus` and `reinforce_plus_plus_baseline` ([https://arxiv.org/abs/2501.03262](https://arxiv.org/abs/2501.03262))
     - `ppo` ([https://arxiv.org/abs/1707.06347](https://arxiv.org/abs/1707.06347))
 - `--calculate-per-token-loss`: By default, vime calculates loss on a per-sample basis, i.e., `mean(sum(sample_i) / len(sample_i))`. Enable this flag to calculate loss on a per-token basis, i.e., `sum(sum(sample_i)) / sum(len(sample_i))`.
@@ -224,6 +225,38 @@ Related parameters:
 - `--normalize-advantages`: Whether to normalize advantages.
 - `--eps-clip`: PPO-style clip range.
 
+#### DAPO Algorithm
+
+DAPO (Decoupled Clip and Dynamic sAmpling Policy Optimization, https://arxiv.org/abs/2503.14476) optimizes the policy with group-relative advantages, Clip-Higher, token-level loss, dynamic sampling, and Soft Overlong. Selecting `dapo` enables these mechanisms with the default settings below.
+
+To use DAPO, set:
+
+```bash
+--advantage-estimator dapo
+--n-samples-per-prompt 8
+--rollout-batch-size 32
+--over-sampling-batch-size 64
+```
+
+When `dapo` is selected and not explicitly overridden, vime automatically sets:
+
+- `--eps-clip-high 0.28` (with default `--eps-clip 0.2` for Clip-Higher);
+- `--calculate-per-token-loss`;
+- `--dynamic-sampling-filter-path vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std`;
+- `--soft-overlong-cache` to `rollout_max_response_len // 4`.
+
+Related parameters:
+
+- `--eps-clip` / `--eps-clip-high`: Asymmetric clip bounds;
+- `--calculate-per-token-loss`: Reduce policy loss at token level instead of per-sample;
+- `--over-sampling-batch-size`: Preferably larger than `--rollout-batch-size` for effective dynamic sampling;
+- `--dynamic-sampling-filter-path`: Drop prompt groups whose reward std is ~0 (all-correct / all-incorrect);
+- `--soft-overlong-cache`: Soft Overlong `L_cache`; set to `0` to disable. The length penalty is added to rewards before group normalization;
+- `--partial-rollout`: Optional; resume aborted generations during dynamic sampling;
+- `--rm-type dapo`: Only the DAPO-style math answer scorer, not the full DAPO recipe.
+
+Note: if `--custom-reward-post-process-path` is set, the built-in Soft Overlong path is skipped; implement the length penalty in your custom function if needed.
+
 #### PPO Algorithm
 
 PPO (Proximal Policy Optimization) is a classic RL algorithm that uses a critic model to estimate the value function for computing advantages.
diff --git a/docs/zh/get_started/usage.md b/docs/zh/get_started/usage.md
index 9c9a4a64e..2f8ea71c1 100644
--- a/docs/zh/get_started/usage.md
+++ b/docs/zh/get_started/usage.md
@@ -199,6 +199,7 @@ vLLM 的加载非常简单，只需要：
   - `grpo`（https://arxiv.org/abs/2402.03300）；
   - `gspo`（https://arxiv.org/abs/2507.18071）；
   - `cispo`（https://arxiv.org/abs/2506.13585）；
+  - `dapo`（https://arxiv.org/abs/2503.14476）；
   - `reinforce_plus_plus` 与 `reinforce_plus_plus_baseline`（https://arxiv.org/abs/2501.03262）；
   - `ppo`（https://arxiv.org/abs/1707.06347）。
 - `--calculate-per-token-loss`：vime 中默认的方案是 per sample loss，即 `mean(sum(sample_i) / len(sample_i))`，如果需要计算 per token loss，即 `sum(sum(sample_i)) / sum(len(sample_i))`，可以开启 `--calculate-per-token-loss`；
@@ -226,6 +227,38 @@ GRPO 的主要特点：
 - `--normalize-advantages`：是否对 advantage 进行归一化；
 - `--eps-clip`：PPO 风格的 clip 范围。
 
+#### DAPO 算法
+
+DAPO（Decoupled Clip and Dynamic sAmpling Policy Optimization，https://arxiv.org/abs/2503.14476）通过组内相对 advantage、Clip-Higher、token-level loss、dynamic sampling 与 Soft Overlong 进行策略优化。选择 `dapo` 后，这些机制会按默认配置启用。
+
+使用 DAPO 时，需要设置：
+
+```bash
+--advantage-estimator dapo
+--n-samples-per-prompt 8
+--rollout-batch-size 32
+--over-sampling-batch-size 64
+```
+
+选择 `dapo` 后，若未显式覆盖，会自动设置：
+
+- `--eps-clip-high 0.28`（与默认 `--eps-clip 0.2` 构成 Clip-Higher）；
+- `--calculate-per-token-loss`；
+- `--dynamic-sampling-filter-path vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std`；
+- `--soft-overlong-cache` 为 `rollout_max_response_len // 4`。
+
+相关参数：
+
+- `--eps-clip` / `--eps-clip-high`：非对称 clip 上下界；
+- `--calculate-per-token-loss`：按 token 而非按 sample 归约 policy loss；
+- `--over-sampling-batch-size`：建议大于 `--rollout-batch-size`，配合 dynamic sampling 过采样；
+- `--dynamic-sampling-filter-path`：过滤组内 reward std≈0（全对/全错）的 prompt 组；
+- `--soft-overlong-cache`：Soft Overlong 的 L_cache；设为 `0` 可关闭。长度惩罚在 group reward 归一化之前叠加到 reward 上；
+- `--partial-rollout`：可选，动态采样 abort 后的续写加速；
+- `--rm-type dapo`：仅表示 DAPO 论文配套的数学答案打分器，不等于完整 DAPO 配方。
+
+注意：若同时设置 `--custom-reward-post-process-path`，内置 Soft Overlong 不会生效，需要在自定义函数中自行实现长度惩罚。
+
 #### PPO 算法
 
 PPO（Proximal Policy Optimization）是经典的 RL 算法，使用 critic 模型来估计 value function，从而计算 advantage。
diff --git a/tests/test_megatron_argument_validation.py b/tests/test_megatron_argument_validation.py
index 0e96b1405..204f366c1 100644
--- a/tests/test_megatron_argument_validation.py
+++ b/tests/test_megatron_argument_validation.py
@@ -212,6 +212,11 @@ def make_vime_validate_args(**overrides):
         balance_data=False,
         eps_clip_high=None,
         eps_clip=0.2,
+        soft_overlong_cache=None,
+        calculate_per_token_loss=False,
+        dynamic_sampling_filter_path=None,
+        custom_reward_post_process_path=None,
+        rollout_max_response_len=1024,
         eval_reward_key=None,
         reward_key="reward",
         dump_details=None,
@@ -306,6 +311,28 @@ def test_vime_validate_args_preserves_zero_rollout_gpus_without_colocate(monkeyp
     assert args.offload_rollout is False
 
 
+@pytest.mark.unit
+def test_vime_validate_args_dapo_recipe_defaults(monkeypatch):
+    module = load_vime_arguments_module(monkeypatch)
+    args = make_vime_validate_args(
+        advantage_estimator="dapo",
+        n_samples_per_prompt=8,
+        rollout_batch_size=32,
+        rollout_max_response_len=1024,
+        over_sampling_batch_size=64,
+    )
+
+    module.vime_validate_args(args)
+
+    assert args.eps_clip_high == 0.28
+    assert args.calculate_per_token_loss is True
+    assert (
+        args.dynamic_sampling_filter_path
+        == "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std"
+    )
+    assert args.soft_overlong_cache == 256
+
+
 @pytest.mark.unit
 def test_update_weight_delta_disabled(monkeypatch):
     module = load_vime_arguments_module(monkeypatch)
diff --git a/vime/backends/megatron_utils/loss.py b/vime/backends/megatron_utils/loss.py
index 2003dd59b..20895711c 100644
--- a/vime/backends/megatron_utils/loss.py
+++ b/vime/backends/megatron_utils/loss.py
@@ -670,7 +670,7 @@ def compute_advantages_and_returns(args: Namespace, rollout_data: RolloutBatch)
 
     This function extracts rewards, log-probs, values, and masks from
     `rollout_data`, computes KL divergences, then applies the chosen advantage
-    estimator. Supported methods: "grpo", "gspo", "cispo", "ppo",
+    estimator. Supported methods: "grpo", "gspo", "cispo", "dapo", "ppo",
     "reinforce_plus_plus", and "reinforce_plus_plus_baseline". When
     `args.normalize_advantages` is True, advantages are whitened across the
     data-parallel group using masked statistics.
@@ -724,7 +724,7 @@ def compute_advantages_and_returns(args: Namespace, rollout_data: RolloutBatch)
         custom_adv_fn(args, rollout_data)
         advantages, returns = rollout_data["advantages"], rollout_data["returns"]
 
-    elif args.advantage_estimator in ["grpo", "gspo", "cispo"]:
+    elif args.advantage_estimator in ["grpo", "gspo", "cispo", "dapo"]:
         rewards = torch.tensor(rewards, dtype=torch.float32, device=kl[0].device)
         returns = get_grpo_returns(rewards, kl)
         # TODO: is the copy necessary?
diff --git a/vime/ray/rollout.py b/vime/ray/rollout.py
index ec88fbfdc..73f82d6c1 100644
--- a/vime/ray/rollout.py
+++ b/vime/ray/rollout.py
@@ -687,12 +687,31 @@ def _post_process_rewards(self, samples: list[Sample] | list[list[Sample]]):
             return self.custom_reward_post_process_func(self.args, samples)
 
         raw_rewards = [sample.get_reward_value(self.args) for sample in samples]
+        rewards_for_norm = raw_rewards
+
+        # Soft Overlong Punishment (DAPO Eq.13), applied before group normalization.
+        soft_overlong_cache = getattr(self.args, "soft_overlong_cache", None)
+        if soft_overlong_cache is not None and soft_overlong_cache > 0:
+            L_max = int(self.args.rollout_max_response_len)
+            L_cache = int(soft_overlong_cache)
+            shaped_rewards = []
+            for sample, reward in zip(samples, raw_rewards, strict=True):
+                response_len = int(sample.response_length)
+                if response_len <= L_max - L_cache:
+                    length_penalty = 0.0
+                elif response_len <= L_max:
+                    length_penalty = ((L_max - L_cache) - response_len) / float(L_cache)
+                else:
+                    length_penalty = -1.0
+                shaped_rewards.append(float(reward) + length_penalty)
+            rewards_for_norm = shaped_rewards
+
         if (
-            self.args.advantage_estimator in ["grpo", "gspo", "cispo", "reinforce_plus_plus_baseline"]
+            self.args.advantage_estimator in ["grpo", "gspo", "cispo", "dapo", "reinforce_plus_plus_baseline"]
             and self.args.rewards_normalization
         ):
             # group norm
-            rewards = torch.tensor(raw_rewards, dtype=torch.float)
+            rewards = torch.tensor(rewards_for_norm, dtype=torch.float)
             if rewards.shape[-1] == self.args.n_samples_per_prompt * self.args.rollout_batch_size:
                 rewards = rewards.reshape(-1, self.args.n_samples_per_prompt)
             else:
@@ -701,13 +720,13 @@ def _post_process_rewards(self, samples: list[Sample] | list[list[Sample]]):
             mean = rewards.mean(dim=-1, keepdim=True)
             rewards = rewards - mean
 
-            if self.args.advantage_estimator in ["grpo", "gspo", "cispo"] and self.args.grpo_std_normalization:
+            if self.args.advantage_estimator in ["grpo", "gspo", "cispo", "dapo"] and self.args.grpo_std_normalization:
                 std = rewards.std(dim=-1, keepdim=True)
                 rewards = rewards / (std + 1e-6)
 
             return raw_rewards, rewards.flatten().tolist()
 
-        return raw_rewards, raw_rewards
+        return raw_rewards, rewards_for_norm
 
     def _convert_samples_to_train_data(self, samples: list[Sample] | list[list[Sample]]):
         """
diff --git a/vime/utils/arguments.py b/vime/utils/arguments.py
index 25fd9cc93..a9ed209d4 100644
--- a/vime/utils/arguments.py
+++ b/vime/utils/arguments.py
@@ -889,6 +889,18 @@ def add_algo_arguments(parser):
 
             parser.add_argument("--eps-clip", type=float, default=0.2, help="PPO clip range")
             parser.add_argument("--eps-clip-high", type=float, default=None, help="PPO clip upper range")
+            parser.add_argument(
+                "--soft-overlong-cache",
+                type=int,
+                default=None,
+                help=(
+                    "Soft Overlong Punishment cache length L_cache from DAPO. "
+                    "When set to a positive value, responses longer than "
+                    "(rollout_max_response_len - soft_overlong_cache) receive a linear length penalty "
+                    "before group reward normalization. Set to 0 to disable. "
+                    "With --advantage-estimator dapo, defaults to rollout_max_response_len // 4."
+                ),
+            )
             parser.add_argument(
                 "--eps-clip-c",
                 type=float,
@@ -935,6 +947,7 @@ def add_algo_arguments(parser):
                     "grpo",
                     "gspo",
                     "cispo",
+                    "dapo",
                     "reinforce_plus_plus",
                     "reinforce_plus_plus_baseline",
                     "ppo",
@@ -1854,9 +1867,40 @@ def vime_validate_args(args):
         assert args.use_dynamic_batch_size, "--balance-by-flops requires --use-dynamic-batch-size"
         args.balance_data = True
 
+    if args.advantage_estimator == "dapo":
+        # DAPO defaults: Clip-Higher + token-level loss + dynamic sampling + Soft Overlong.
+        if args.eps_clip_high is None:
+            args.eps_clip_high = 0.28
+            logger.info("DAPO: setting --eps-clip-high to 0.28 (Clip-Higher).")
+        if not args.calculate_per_token_loss:
+            args.calculate_per_token_loss = True
+            logger.info("DAPO: enabling --calculate-per-token-loss.")
+        if args.dynamic_sampling_filter_path is None:
+            args.dynamic_sampling_filter_path = (
+                "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std"
+            )
+            logger.info(
+                "DAPO: enabling dynamic sampling filter "
+                "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std."
+            )
+        if args.soft_overlong_cache is None:
+            args.soft_overlong_cache = max(1, int(args.rollout_max_response_len) // 4)
+            logger.info(
+                "DAPO: enabling Soft Overlong with --soft-overlong-cache %s.",
+                args.soft_overlong_cache,
+            )
+        if args.custom_reward_post_process_path is not None and args.soft_overlong_cache:
+            logger.warning(
+                "DAPO Soft Overlong (--soft-overlong-cache) is skipped when "
+                "--custom-reward-post-process-path is set; apply length penalty in the custom function if needed."
+            )
+
     if args.eps_clip_high is None:
         args.eps_clip_high = args.eps_clip
 
+    if args.soft_overlong_cache is not None and args.soft_overlong_cache < 0:
+        raise ValueError(f"--soft-overlong-cache must be >= 0, got {args.soft_overlong_cache}.")
+
     if args.advantage_estimator == "cispo" and args.eps_clip < 1.0:
         logger.warning(
             "CISPO is canonically single-sided, but --eps-clip=%s keeps the lower clip bound %s active. "
@@ -1982,6 +2026,18 @@ def vime_validate_args(args):
         f"rollout_batch_size {args.rollout_batch_size}"
     )
 
+    if (
+        args.advantage_estimator == "dapo"
+        and args.dynamic_sampling_filter_path is not None
+        and args.over_sampling_batch_size <= args.rollout_batch_size
+    ):
+        logger.warning(
+            "DAPO dynamic sampling is more effective when "
+            "--over-sampling-batch-size > --rollout-batch-size "
+            f"(got over_sampling_batch_size={args.over_sampling_batch_size}, "
+            f"rollout_batch_size={args.rollout_batch_size})."
+        )
+
     if args.num_epoch is not None:
         if args.num_rollout is not None:
             logger.info("Both num_epoch and num_rollout are set, num_epoch will be ignored.")

From af2cb5ef9b737f60beb6fe1f2854dd9d6f6e4bb5 Mon Sep 17 00:00:00 2001
From: kaiyuan <kyxiezju@163.com>
Date: Wed, 22 Jul 2026 11:10:21 +0800
Subject: [PATCH 2/5] Validate rollout-max-response-len for Soft Overlong.

Signed-off-by: kaiyuan <kyxiezju@163.com>
---
 tests/test_megatron_argument_validation.py | 26 ++++++++++++++++++++++
 vime/utils/arguments.py                    | 12 ++++++++--
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/tests/test_megatron_argument_validation.py b/tests/test_megatron_argument_validation.py
index 204f366c1..d31ac6311 100644
--- a/tests/test_megatron_argument_validation.py
+++ b/tests/test_megatron_argument_validation.py
@@ -333,6 +333,32 @@ def test_vime_validate_args_dapo_recipe_defaults(monkeypatch):
     assert args.soft_overlong_cache == 256
 
 
+@pytest.mark.unit
+def test_vime_validate_args_dapo_requires_rollout_max_response_len(monkeypatch):
+    module = load_vime_arguments_module(monkeypatch)
+    args = make_vime_validate_args(
+        advantage_estimator="dapo",
+        rollout_max_response_len=None,
+        soft_overlong_cache=None,
+    )
+
+    with pytest.raises(ValueError, match="rollout-max-response-len"):
+        module.vime_validate_args(args)
+
+
+@pytest.mark.unit
+def test_vime_validate_args_soft_overlong_requires_rollout_max_response_len(monkeypatch):
+    module = load_vime_arguments_module(monkeypatch)
+    args = make_vime_validate_args(
+        advantage_estimator="grpo",
+        soft_overlong_cache=256,
+        rollout_max_response_len=None,
+    )
+
+    with pytest.raises(ValueError, match="rollout-max-response-len"):
+        module.vime_validate_args(args)
+
+
 @pytest.mark.unit
 def test_update_weight_delta_disabled(monkeypatch):
     module = load_vime_arguments_module(monkeypatch)
diff --git a/vime/utils/arguments.py b/vime/utils/arguments.py
index a9ed209d4..25f1b9d71 100644
--- a/vime/utils/arguments.py
+++ b/vime/utils/arguments.py
@@ -1884,6 +1884,11 @@ def vime_validate_args(args):
                 "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std."
             )
         if args.soft_overlong_cache is None:
+            if args.rollout_max_response_len is None:
+                raise ValueError(
+                    "DAPO Soft Overlong requires --rollout-max-response-len to be set "
+                    "(or pass --soft-overlong-cache 0 to disable Soft Overlong)."
+                )
             args.soft_overlong_cache = max(1, int(args.rollout_max_response_len) // 4)
             logger.info(
                 "DAPO: enabling Soft Overlong with --soft-overlong-cache %s.",
@@ -1898,8 +1903,11 @@ def vime_validate_args(args):
     if args.eps_clip_high is None:
         args.eps_clip_high = args.eps_clip
 
-    if args.soft_overlong_cache is not None and args.soft_overlong_cache < 0:
-        raise ValueError(f"--soft-overlong-cache must be >= 0, got {args.soft_overlong_cache}.")
+    if args.soft_overlong_cache is not None:
+        if args.soft_overlong_cache < 0:
+            raise ValueError(f"--soft-overlong-cache must be >= 0, got {args.soft_overlong_cache}.")
+        if args.soft_overlong_cache > 0 and args.rollout_max_response_len is None:
+            raise ValueError("--rollout-max-response-len must be set when --soft-overlong-cache is enabled.")
 
     if args.advantage_estimator == "cispo" and args.eps_clip < 1.0:
         logger.warning(

From e0c3fb96bd20c823c9966e2cc289344c0afaf78a Mon Sep 17 00:00:00 2001
From: kaiyuan <kyxiezju@163.com>
Date: Wed, 22 Jul 2026 22:06:51 +0800
Subject: [PATCH 3/5] Validate soft-overlong-cache against
 rollout-max-response-len.

Signed-off-by: kaiyuan <kyxiezju@163.com>
---
 tests/test_megatron_argument_validation.py | 13 +++++++++++++
 vime/utils/arguments.py                    | 10 ++++++++++
 2 files changed, 23 insertions(+)

diff --git a/tests/test_megatron_argument_validation.py b/tests/test_megatron_argument_validation.py
index d31ac6311..48f822f2a 100644
--- a/tests/test_megatron_argument_validation.py
+++ b/tests/test_megatron_argument_validation.py
@@ -359,6 +359,19 @@ def test_vime_validate_args_soft_overlong_requires_rollout_max_response_len(monk
         module.vime_validate_args(args)
 
 
+@pytest.mark.unit
+def test_vime_validate_args_soft_overlong_cache_exceeds_max_response_len(monkeypatch):
+    module = load_vime_arguments_module(monkeypatch)
+    args = make_vime_validate_args(
+        advantage_estimator="grpo",
+        soft_overlong_cache=2000,
+        rollout_max_response_len=1024,
+    )
+
+    with pytest.raises(ValueError, match="must be <="):
+        module.vime_validate_args(args)
+
+
 @pytest.mark.unit
 def test_update_weight_delta_disabled(monkeypatch):
     module = load_vime_arguments_module(monkeypatch)
diff --git a/vime/utils/arguments.py b/vime/utils/arguments.py
index 25f1b9d71..d96fb043a 100644
--- a/vime/utils/arguments.py
+++ b/vime/utils/arguments.py
@@ -1908,6 +1908,16 @@ def vime_validate_args(args):
             raise ValueError(f"--soft-overlong-cache must be >= 0, got {args.soft_overlong_cache}.")
         if args.soft_overlong_cache > 0 and args.rollout_max_response_len is None:
             raise ValueError("--rollout-max-response-len must be set when --soft-overlong-cache is enabled.")
+        if (
+            args.soft_overlong_cache > 0
+            and args.rollout_max_response_len is not None
+            and args.soft_overlong_cache > args.rollout_max_response_len
+        ):
+            raise ValueError(
+                f"--soft-overlong-cache ({args.soft_overlong_cache}) must be <= "
+                f"--rollout-max-response-len ({args.rollout_max_response_len}). "
+                "L_cache should not exceed the maximum response length."
+            )
 
     if args.advantage_estimator == "cispo" and args.eps_clip < 1.0:
         logger.warning(

From 98b3b990c507b1065445ef2d520fa16d59dc0035 Mon Sep 17 00:00:00 2001
From: kaiyuan <kyxiezju@163.com>
Date: Thu, 23 Jul 2026 10:12:31 +0800
Subject: [PATCH 4/5] Clarify DAPO token-level loss and Soft Overlong docs.

Signed-off-by: kaiyuan <kyxiezju@163.com>
---
 docs/en/get_started/usage.md | 6 +++---
 docs/zh/get_started/usage.md | 6 +++---
 vime/utils/arguments.py      | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/en/get_started/usage.md b/docs/en/get_started/usage.md
index 0510d84b0..e1f348c7e 100644
--- a/docs/en/get_started/usage.md
+++ b/docs/en/get_started/usage.md
@@ -241,17 +241,17 @@ To use DAPO, set:
 When `dapo` is selected and not explicitly overridden, vime automatically sets:
 
 - `--eps-clip-high 0.28` (with default `--eps-clip 0.2` for Clip-Higher);
-- `--calculate-per-token-loss`;
+- `--calculate-per-token-loss` (mandatory for DAPO; selecting `dapo` always enables it);
 - `--dynamic-sampling-filter-path vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std`;
 - `--soft-overlong-cache` to `rollout_max_response_len // 4`.
 
 Related parameters:
 
 - `--eps-clip` / `--eps-clip-high`: Asymmetric clip bounds;
-- `--calculate-per-token-loss`: Reduce policy loss at token level instead of per-sample;
+- `--calculate-per-token-loss`: Reduce policy loss at token level instead of per-sample; mandatory when using `dapo`;
 - `--over-sampling-batch-size`: Preferably larger than `--rollout-batch-size` for effective dynamic sampling;
 - `--dynamic-sampling-filter-path`: Drop prompt groups whose reward std is ~0 (all-correct / all-incorrect);
-- `--soft-overlong-cache`: Soft Overlong `L_cache`; set to `0` to disable. The length penalty is added to rewards before group normalization;
+- `--soft-overlong-cache`: Soft Overlong `L_cache`; set to `0` to disable. The length penalty is added to rewards before group normalization, so Soft Overlong still applies even when rewards normalization is disabled;
 - `--partial-rollout`: Optional; resume aborted generations during dynamic sampling;
 - `--rm-type dapo`: Only the DAPO-style math answer scorer, not the full DAPO recipe.
 
diff --git a/docs/zh/get_started/usage.md b/docs/zh/get_started/usage.md
index 2f8ea71c1..91d7e517e 100644
--- a/docs/zh/get_started/usage.md
+++ b/docs/zh/get_started/usage.md
@@ -243,17 +243,17 @@ DAPO（Decoupled Clip and Dynamic sAmpling Policy Optimization，https://arxiv.o
 选择 `dapo` 后，若未显式覆盖，会自动设置：
 
 - `--eps-clip-high 0.28`（与默认 `--eps-clip 0.2` 构成 Clip-Higher）；
-- `--calculate-per-token-loss`；
+- `--calculate-per-token-loss`（DAPO 要求 token-level loss，选择 `dapo` 时会强制开启）；
 - `--dynamic-sampling-filter-path vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std`；
 - `--soft-overlong-cache` 为 `rollout_max_response_len // 4`。
 
 相关参数：
 
 - `--eps-clip` / `--eps-clip-high`：非对称 clip 上下界；
-- `--calculate-per-token-loss`：按 token 而非按 sample 归约 policy loss；
+- `--calculate-per-token-loss`：按 token 而非按 sample 归约 policy loss；选择 `dapo` 时为强制项；
 - `--over-sampling-batch-size`：建议大于 `--rollout-batch-size`，配合 dynamic sampling 过采样；
 - `--dynamic-sampling-filter-path`：过滤组内 reward std≈0（全对/全错）的 prompt 组；
-- `--soft-overlong-cache`：Soft Overlong 的 L_cache；设为 `0` 可关闭。长度惩罚在 group reward 归一化之前叠加到 reward 上；
+- `--soft-overlong-cache`：Soft Overlong 的 L_cache；设为 `0` 可关闭。长度惩罚在 group reward 归一化之前叠加到 reward 上，因此无论是否启用 rewards normalization，Soft Overlong 都会生效；
 - `--partial-rollout`：可选，动态采样 abort 后的续写加速；
 - `--rm-type dapo`：仅表示 DAPO 论文配套的数学答案打分器，不等于完整 DAPO 配方。
 
diff --git a/vime/utils/arguments.py b/vime/utils/arguments.py
index d96fb043a..0eaaf34de 100644
--- a/vime/utils/arguments.py
+++ b/vime/utils/arguments.py
@@ -1874,7 +1874,7 @@ def vime_validate_args(args):
             logger.info("DAPO: setting --eps-clip-high to 0.28 (Clip-Higher).")
         if not args.calculate_per_token_loss:
             args.calculate_per_token_loss = True
-            logger.info("DAPO: enabling --calculate-per-token-loss.")
+            logger.info("DAPO: --calculate-per-token-loss is mandatory for DAPO and has been enabled.")
         if args.dynamic_sampling_filter_path is None:
             args.dynamic_sampling_filter_path = (
                 "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std"

From 0851942fc93ebf898a905497b2645f5de9370702 Mon Sep 17 00:00:00 2001
From: aoshen02 <aoshen@inferact.ai>
Date: Fri, 28 Aug 2026 14:13:26 +0000
Subject: [PATCH 5/5] refactor: reduce DAPO to a GRPO preset

Signed-off-by: aoshen02 <aoshen@inferact.ai>
---
 docs/en/get_started/usage.md               | 26 +--------
 docs/zh/get_started/usage.md               | 26 +--------
 tests/test_megatron_argument_validation.py | 66 ----------------------
 vime/backends/megatron_utils/loss.py       |  4 +-
 vime/ray/rollout.py                        | 27 +++------
 vime/utils/arguments.py                    | 65 +++------------------
 6 files changed, 23 insertions(+), 191 deletions(-)

diff --git a/docs/en/get_started/usage.md b/docs/en/get_started/usage.md
index eb51029bc..bdc10182a 100644
--- a/docs/en/get_started/usage.md
+++ b/docs/en/get_started/usage.md
@@ -227,35 +227,13 @@ Related parameters:
 
 #### DAPO Algorithm
 
-DAPO (Decoupled Clip and Dynamic sAmpling Policy Optimization, https://arxiv.org/abs/2503.14476) optimizes the policy with group-relative advantages, Clip-Higher, token-level loss, dynamic sampling, and Soft Overlong. Selecting `dapo` enables these mechanisms with the default settings below.
-
-To use DAPO, set:
+DAPO (Decoupled Clip and Dynamic sAmpling Policy Optimization, https://arxiv.org/abs/2503.14476) is exposed as a preset over the existing GRPO implementation. Set:
 
 ```bash
 --advantage-estimator dapo
---n-samples-per-prompt 8
---rollout-batch-size 32
---over-sampling-batch-size 64
 ```
 
-When `dapo` is selected and not explicitly overridden, vime automatically sets:
-
-- `--eps-clip-high 0.28` (with default `--eps-clip 0.2` for Clip-Higher);
-- `--calculate-per-token-loss` (mandatory for DAPO; selecting `dapo` always enables it);
-- `--dynamic-sampling-filter-path vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std`;
-- `--soft-overlong-cache` to `rollout_max_response_len // 4`.
-
-Related parameters:
-
-- `--eps-clip` / `--eps-clip-high`: Asymmetric clip bounds;
-- `--calculate-per-token-loss`: Reduce policy loss at token level instead of per-sample; mandatory when using `dapo`;
-- `--over-sampling-batch-size`: Preferably larger than `--rollout-batch-size` for effective dynamic sampling;
-- `--dynamic-sampling-filter-path`: Drop prompt groups whose reward std is ~0 (all-correct / all-incorrect);
-- `--soft-overlong-cache`: Soft Overlong `L_cache`; set to `0` to disable. The length penalty is added to rewards before group normalization, so Soft Overlong still applies even when rewards normalization is disabled;
-- `--partial-rollout`: Optional; resume aborted generations during dynamic sampling;
-- `--rm-type dapo`: Only the DAPO-style math answer scorer, not the full DAPO recipe.
-
-Note: if `--custom-reward-post-process-path` is set, the built-in Soft Overlong path is skipped; implement the length penalty in your custom function if needed.
+This selects GRPO with Clip-Higher (`--eps-clip-high 0.28`), token-level loss, the default dynamic-sampling filter, and a Soft Overlong window of one quarter of `--rollout-max-response-len`. Set `--soft-overlong-cache 0` to disable the length penalty. A custom reward post-process replaces the built-in reward shaping.
 
 #### PPO Algorithm
 
diff --git a/docs/zh/get_started/usage.md b/docs/zh/get_started/usage.md
index 1c45d0c56..3801b57cb 100644
--- a/docs/zh/get_started/usage.md
+++ b/docs/zh/get_started/usage.md
@@ -229,35 +229,13 @@ GRPO 的主要特点：
 
 #### DAPO 算法
 
-DAPO（Decoupled Clip and Dynamic sAmpling Policy Optimization，https://arxiv.org/abs/2503.14476）通过组内相对 advantage、Clip-Higher、token-level loss、dynamic sampling 与 Soft Overlong 进行策略优化。选择 `dapo` 后，这些机制会按默认配置启用。
-
-使用 DAPO 时，需要设置：
+DAPO（Decoupled Clip and Dynamic sAmpling Policy Optimization，https://arxiv.org/abs/2503.14476）作为现有 GRPO 实现的预设提供。使用：
 
 ```bash
 --advantage-estimator dapo
---n-samples-per-prompt 8
---rollout-batch-size 32
---over-sampling-batch-size 64
 ```
 
-选择 `dapo` 后，若未显式覆盖，会自动设置：
-
-- `--eps-clip-high 0.28`（与默认 `--eps-clip 0.2` 构成 Clip-Higher）；
-- `--calculate-per-token-loss`（DAPO 要求 token-level loss，选择 `dapo` 时会强制开启）；
-- `--dynamic-sampling-filter-path vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std`；
-- `--soft-overlong-cache` 为 `rollout_max_response_len // 4`。
-
-相关参数：
-
-- `--eps-clip` / `--eps-clip-high`：非对称 clip 上下界；
-- `--calculate-per-token-loss`：按 token 而非按 sample 归约 policy loss；选择 `dapo` 时为强制项；
-- `--over-sampling-batch-size`：建议大于 `--rollout-batch-size`，配合 dynamic sampling 过采样；
-- `--dynamic-sampling-filter-path`：过滤组内 reward std≈0（全对/全错）的 prompt 组；
-- `--soft-overlong-cache`：Soft Overlong 的 L_cache；设为 `0` 可关闭。长度惩罚在 group reward 归一化之前叠加到 reward 上，因此无论是否启用 rewards normalization，Soft Overlong 都会生效；
-- `--partial-rollout`：可选，动态采样 abort 后的续写加速；
-- `--rm-type dapo`：仅表示 DAPO 论文配套的数学答案打分器，不等于完整 DAPO 配方。
-
-注意：若同时设置 `--custom-reward-post-process-path`，内置 Soft Overlong 不会生效，需要在自定义函数中自行实现长度惩罚。
+该预设使用 GRPO，并启用 Clip-Higher（`--eps-clip-high 0.28`）、token-level loss、默认 dynamic-sampling filter，以及长度为 `--rollout-max-response-len` 四分之一的 Soft Overlong 区间。设置 `--soft-overlong-cache 0` 可关闭长度惩罚；自定义 reward post-process 会替代内置 reward shaping。
 
 #### PPO 算法
 
diff --git a/tests/test_megatron_argument_validation.py b/tests/test_megatron_argument_validation.py
index bd00e8e89..05b6e7b2d 100644
--- a/tests/test_megatron_argument_validation.py
+++ b/tests/test_megatron_argument_validation.py
@@ -211,11 +211,6 @@ def make_vime_validate_args(**overrides):
         balance_data=False,
         eps_clip_high=None,
         eps_clip=0.2,
-        soft_overlong_cache=None,
-        calculate_per_token_loss=False,
-        dynamic_sampling_filter_path=None,
-        custom_reward_post_process_path=None,
-        rollout_max_response_len=1024,
         eval_reward_key=None,
         reward_key="reward",
         dump_details=None,
@@ -345,67 +340,6 @@ def test_vime_validate_args_preserves_zero_rollout_gpus_without_colocate(monkeyp
     assert args.offload_rollout is False
 
 
-@pytest.mark.unit
-def test_vime_validate_args_dapo_recipe_defaults(monkeypatch):
-    module = load_vime_arguments_module(monkeypatch)
-    args = make_vime_validate_args(
-        advantage_estimator="dapo",
-        n_samples_per_prompt=8,
-        rollout_batch_size=32,
-        rollout_max_response_len=1024,
-        over_sampling_batch_size=64,
-    )
-
-    module.vime_validate_args(args)
-
-    assert args.eps_clip_high == 0.28
-    assert args.calculate_per_token_loss is True
-    assert (
-        args.dynamic_sampling_filter_path
-        == "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std"
-    )
-    assert args.soft_overlong_cache == 256
-
-
-@pytest.mark.unit
-def test_vime_validate_args_dapo_requires_rollout_max_response_len(monkeypatch):
-    module = load_vime_arguments_module(monkeypatch)
-    args = make_vime_validate_args(
-        advantage_estimator="dapo",
-        rollout_max_response_len=None,
-        soft_overlong_cache=None,
-    )
-
-    with pytest.raises(ValueError, match="rollout-max-response-len"):
-        module.vime_validate_args(args)
-
-
-@pytest.mark.unit
-def test_vime_validate_args_soft_overlong_requires_rollout_max_response_len(monkeypatch):
-    module = load_vime_arguments_module(monkeypatch)
-    args = make_vime_validate_args(
-        advantage_estimator="grpo",
-        soft_overlong_cache=256,
-        rollout_max_response_len=None,
-    )
-
-    with pytest.raises(ValueError, match="rollout-max-response-len"):
-        module.vime_validate_args(args)
-
-
-@pytest.mark.unit
-def test_vime_validate_args_soft_overlong_cache_exceeds_max_response_len(monkeypatch):
-    module = load_vime_arguments_module(monkeypatch)
-    args = make_vime_validate_args(
-        advantage_estimator="grpo",
-        soft_overlong_cache=2000,
-        rollout_max_response_len=1024,
-    )
-
-    with pytest.raises(ValueError, match="must be <="):
-        module.vime_validate_args(args)
-
-
 @pytest.mark.unit
 def test_update_weight_delta_disk_is_valid(monkeypatch):
     module = load_vime_arguments_module(monkeypatch)
diff --git a/vime/backends/megatron_utils/loss.py b/vime/backends/megatron_utils/loss.py
index 7c51cdb32..566be1df7 100644
--- a/vime/backends/megatron_utils/loss.py
+++ b/vime/backends/megatron_utils/loss.py
@@ -706,7 +706,7 @@ def compute_advantages_and_returns(args: Namespace, rollout_data: RolloutBatch)
 
     This function extracts rewards, log-probs, values, and masks from
     `rollout_data`, computes KL divergences, then applies the chosen advantage
-    estimator. Supported methods: "grpo", "gspo", "cispo", "dapo", "ppo",
+    estimator. Supported methods: "grpo", "gspo", "cispo", "ppo",
     "reinforce_plus_plus", and "reinforce_plus_plus_baseline". When
     `args.normalize_advantages` is True, advantages are whitened across the
     data-parallel-with-context-parallel group using masked statistics.
@@ -760,7 +760,7 @@ def compute_advantages_and_returns(args: Namespace, rollout_data: RolloutBatch)
         custom_adv_fn(args, rollout_data)
         advantages, returns = rollout_data["advantages"], rollout_data["returns"]
 
-    elif args.advantage_estimator in ["grpo", "gspo", "cispo", "dapo"]:
+    elif args.advantage_estimator in ["grpo", "gspo", "cispo"]:
         rewards = torch.tensor(rewards, dtype=torch.float32, device=kl[0].device)
         returns = get_grpo_returns(rewards, kl)
         # TODO: is the copy necessary?
diff --git a/vime/ray/rollout.py b/vime/ray/rollout.py
index 45c1ef161..a2054ae02 100644
--- a/vime/ray/rollout.py
+++ b/vime/ray/rollout.py
@@ -758,26 +758,15 @@ def _post_process_rewards(self, samples: list[Sample] | list[list[Sample]]):
 
         raw_rewards = [sample.get_reward_value(self.args) for sample in samples]
         rewards_for_norm = raw_rewards
-
-        # Soft Overlong Punishment (DAPO Eq.13), applied before group normalization.
-        soft_overlong_cache = getattr(self.args, "soft_overlong_cache", None)
-        if soft_overlong_cache is not None and soft_overlong_cache > 0:
-            L_max = int(self.args.rollout_max_response_len)
-            L_cache = int(soft_overlong_cache)
-            shaped_rewards = []
-            for sample, reward in zip(samples, raw_rewards, strict=True):
-                response_len = int(sample.response_length)
-                if response_len <= L_max - L_cache:
-                    length_penalty = 0.0
-                elif response_len <= L_max:
-                    length_penalty = ((L_max - L_cache) - response_len) / float(L_cache)
-                else:
-                    length_penalty = -1.0
-                shaped_rewards.append(float(reward) + length_penalty)
-            rewards_for_norm = shaped_rewards
+        if cache := getattr(self.args, "soft_overlong_cache", 0):
+            max_len = self.args.rollout_max_response_len
+            rewards_for_norm = [
+                float(reward) - min(1.0, max(0.0, (sample.response_length - max_len + cache) / cache))
+                for sample, reward in zip(samples, raw_rewards, strict=True)
+            ]
 
         if (
-            self.args.advantage_estimator in ["grpo", "gspo", "cispo", "dapo", "reinforce_plus_plus_baseline"]
+            self.args.advantage_estimator in ["grpo", "gspo", "cispo", "reinforce_plus_plus_baseline"]
             and self.args.rewards_normalization
         ):
             # group norm
@@ -790,7 +779,7 @@ def _post_process_rewards(self, samples: list[Sample] | list[list[Sample]]):
             mean = rewards.mean(dim=-1, keepdim=True)
             rewards = rewards - mean
 
-            if self.args.advantage_estimator in ["grpo", "gspo", "cispo", "dapo"] and self.args.grpo_std_normalization:
+            if self.args.advantage_estimator in ["grpo", "gspo", "cispo"] and self.args.grpo_std_normalization:
                 std = rewards.std(dim=-1, keepdim=True)
                 rewards = rewards / (std + 1e-6)
 
diff --git a/vime/utils/arguments.py b/vime/utils/arguments.py
index dec8fb69e..661182a7e 100644
--- a/vime/utils/arguments.py
+++ b/vime/utils/arguments.py
@@ -912,13 +912,7 @@ def add_algo_arguments(parser):
                 "--soft-overlong-cache",
                 type=int,
                 default=None,
-                help=(
-                    "Soft Overlong Punishment cache length L_cache from DAPO. "
-                    "When set to a positive value, responses longer than "
-                    "(rollout_max_response_len - soft_overlong_cache) receive a linear length penalty "
-                    "before group reward normalization. Set to 0 to disable. "
-                    "With --advantage-estimator dapo, defaults to rollout_max_response_len // 4."
-                ),
+                help="DAPO Soft Overlong penalty window; 0 disables it.",
             )
             parser.add_argument(
                 "--eps-clip-c",
@@ -1903,56 +1897,27 @@ def vime_validate_args(args):
         args.balance_data = True
 
     if args.advantage_estimator == "dapo":
-        # DAPO defaults: Clip-Higher + token-level loss + dynamic sampling + Soft Overlong.
+        args.advantage_estimator = "grpo"
         if args.eps_clip_high is None:
             args.eps_clip_high = 0.28
-            logger.info("DAPO: setting --eps-clip-high to 0.28 (Clip-Higher).")
-        if not args.calculate_per_token_loss:
-            args.calculate_per_token_loss = True
-            logger.info("DAPO: --calculate-per-token-loss is mandatory for DAPO and has been enabled.")
+        args.calculate_per_token_loss = True
         if args.dynamic_sampling_filter_path is None:
             args.dynamic_sampling_filter_path = (
                 "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std"
             )
-            logger.info(
-                "DAPO: enabling dynamic sampling filter "
-                "vime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std."
-            )
         if args.soft_overlong_cache is None:
             if args.rollout_max_response_len is None:
-                raise ValueError(
-                    "DAPO Soft Overlong requires --rollout-max-response-len to be set "
-                    "(or pass --soft-overlong-cache 0 to disable Soft Overlong)."
-                )
+                raise ValueError("DAPO requires --rollout-max-response-len or --soft-overlong-cache 0.")
             args.soft_overlong_cache = max(1, int(args.rollout_max_response_len) // 4)
-            logger.info(
-                "DAPO: enabling Soft Overlong with --soft-overlong-cache %s.",
-                args.soft_overlong_cache,
-            )
-        if args.custom_reward_post_process_path is not None and args.soft_overlong_cache:
-            logger.warning(
-                "DAPO Soft Overlong (--soft-overlong-cache) is skipped when "
-                "--custom-reward-post-process-path is set; apply length penalty in the custom function if needed."
-            )
 
     if args.eps_clip_high is None:
         args.eps_clip_high = args.eps_clip
 
-    if args.soft_overlong_cache is not None:
-        if args.soft_overlong_cache < 0:
-            raise ValueError(f"--soft-overlong-cache must be >= 0, got {args.soft_overlong_cache}.")
-        if args.soft_overlong_cache > 0 and args.rollout_max_response_len is None:
-            raise ValueError("--rollout-max-response-len must be set when --soft-overlong-cache is enabled.")
-        if (
-            args.soft_overlong_cache > 0
-            and args.rollout_max_response_len is not None
-            and args.soft_overlong_cache > args.rollout_max_response_len
-        ):
-            raise ValueError(
-                f"--soft-overlong-cache ({args.soft_overlong_cache}) must be <= "
-                f"--rollout-max-response-len ({args.rollout_max_response_len}). "
-                "L_cache should not exceed the maximum response length."
-            )
+    soft_overlong_cache = getattr(args, "soft_overlong_cache", None)
+    if soft_overlong_cache and (
+        args.rollout_max_response_len is None or not 0 < soft_overlong_cache <= args.rollout_max_response_len
+    ):
+        raise ValueError("--soft-overlong-cache must be between 0 and --rollout-max-response-len.")
 
     if args.advantage_estimator == "cispo" and args.eps_clip < 1.0:
         logger.warning(
@@ -2081,18 +2046,6 @@ def vime_validate_args(args):
         f"rollout_batch_size {args.rollout_batch_size}"
     )
 
-    if (
-        args.advantage_estimator == "dapo"
-        and args.dynamic_sampling_filter_path is not None
-        and args.over_sampling_batch_size <= args.rollout_batch_size
-    ):
-        logger.warning(
-            "DAPO dynamic sampling is more effective when "
-            "--over-sampling-batch-size > --rollout-batch-size "
-            f"(got over_sampling_batch_size={args.over_sampling_batch_size}, "
-            f"rollout_batch_size={args.rollout_batch_size})."
-        )
-
     if args.num_epoch is not None:
         if args.num_rollout is not None:
             logger.info("Both num_epoch and num_rollout are set, num_epoch will be ignored.")