feature(sunjx): add GSPO and GMPO algorithms support

README.md

@@ -79,7 +79,7 @@ For detailed algorithm descriptions, implementation details, and usage guide, se
 |-----------|------|-----------------|-------|
 | **GRPO** | Policy Optimization | Group normalized advantage estimation |  [arXiv:2402.03300](https://arxiv.org/pdf/2402.03300)  |
 | **GSPO** | Policy Optimization | Group sequence policy optimization | [arXiv:2507.18071](https://arxiv.org/abs/2507.18071) |
-| **GMPO (WIP)** | Policy Optimization | Geometric-mean policy optimization | [arXiv:2507.20673](https://arxiv.org/abs/2507.20673) |
+| **GMPO** | Policy Optimization | Geometric-mean policy optimization | [arXiv:2507.20673](https://arxiv.org/abs/2507.20673) |
 | **Dr.GRPO** | Policy Optimization | Length bias mitigation | [arXiv:2503.20783](https://arxiv.org/abs/2503.20783) |
 | **DAPO** | Policy Optimization | Decoupled clip and dynamic sampling policy optimization | [arXiv:2503.14476](https://arxiv.org/abs/2503.14476) |
 | **REINFORCE++** | Advantage Estimation | Improved baseline estimation | [arXiv:2501.03262](https://arxiv.org/abs/2501.03262) |

README_zh.md

@@ -79,7 +79,7 @@
 |------|------|----------|---------|
 | **GRPO** | Policy Optimization | 组归一化优势估计 |  [arXiv:2402.03300](https://arxiv.org/pdf/2402.03300)  |
 | **GSPO** | Policy Optimization | 组序列策略优化 | [arXiv:2507.18071](https://arxiv.org/abs/2507.18071) |
-| **GMPO (WIP)** | Policy Optimization | 几何平均策略优化 | [arXiv:2507.20673](https://arxiv.org/abs/2507.20673) |
+| **GMPO** | Policy Optimization | 几何平均策略优化 | [arXiv:2507.20673](https://arxiv.org/abs/2507.20673) |
 | **Dr.GRPO** | Policy Optimization | 缓解长度偏差 | [arXiv:2503.20783](https://arxiv.org/abs/2503.20783) |
 | **REINFORCE++** | Advantage Estimation | 改进基线估计 | [arXiv:2501.03262](https://arxiv.org/abs/2501.03262) |
 | **DAPO** | Policy Optimization | 解耦剪裁和动态采样策略优化 | [arXiv:2503.14476](https://arxiv.org/abs/2503.14476) |

examples/grm_vl_rl/train_colocate.py

@@ -292,6 +292,9 @@ def train(args: argparse.Namespace) -> None:
         eps_clip=args.eps_clip,
         loss_agg_mode=args.loss_agg_mode,
         use_gspo=args.use_gspo,
+        use_gmpo=args.use_gmpo,
+        clip_ratio_low=args.clip_ratio_low,
+        clip_ratio_high=args.clip_ratio_high,
         normalize_advantages=args.normalize_advantages,
         use_sequence_rewards=args.use_sequence_rewards,
         gamma=args.gamma,
@@ -384,7 +387,10 @@ def train(args: argparse.Namespace) -> None:
     parser.add_argument("--eps_clip", type=float, default=0.2, help="PPO clip range")
     parser.add_argument("--loss_agg_mode", type=str, default='seq-mean-token-sum',
         help="Loss aggregation mode. Options: ['token-mean', 'seq-mean-token-sum', 'seq-mean-token-mean', 'seq-mean-token-sum-norm']")
-    parser.add_argument("--use_gspo", action="store_true", default=False, help="Enable GSPO (Group Sequence Policy Optimization) mode")
+    parser.add_argument("--use_gspo", action="store_true", default=False, help="Enable GSPO (Group Sequence Policy Optimization) mode. See: https://arxiv.org/abs/2507.18071")
+    parser.add_argument("--use_gmpo", action="store_true", default=False, help="Enable GMPO (Geometric Mean Policy Optimization) mode. See: https://arxiv.org/abs/2502.03950")
+    parser.add_argument("--clip_ratio_low", type=float, default=None, help="Lower clip bound for GSPO asymmetric clipping. Typical value: 0.0003")
+    parser.add_argument("--clip_ratio_high", type=float, default=None, help="Upper clip bound for GSPO asymmetric clipping. Typical value: 0.0004")
     parser.add_argument("--normalize_advantages", action="store_true", default=True, help="Enable advantage normalization in GSPO")
     parser.add_argument("--use_sequence_rewards", action="store_true", default=True, help="Use sequence-level rewards in GSPO")
     parser.add_argument("--value_clip", type=float, default=0.2, help="PPO value clip range")

examples/gsm8k_geo3k/train_colocate.py

@@ -392,6 +392,9 @@ def train(args):
         eps_clip=args.eps_clip,
         loss_agg_mode=args.loss_agg_mode,
         use_gspo=args.use_gspo,
+        use_gmpo=args.use_gmpo,
+        clip_ratio_low=args.clip_ratio_low,
+        clip_ratio_high=args.clip_ratio_high,
         normalize_advantages=args.normalize_advantages,
         use_sequence_rewards=args.use_sequence_rewards,
         gamma=args.gamma,
@@ -486,7 +489,10 @@ def train(args):
     parser.add_argument("--eps_clip", type=float, default=0.2, help="PPO clip range")
     parser.add_argument("--loss_agg_mode", type=str, default='seq-mean-token-mean',
         help="Loss aggregation mode. Options: ['token-mean', 'seq-mean-token-sum', 'seq-mean-token-mean', 'seq-mean-token-sum-norm']")
-    parser.add_argument("--use_gspo", action="store_true", default=False, help="Enable GSPO (Group Sequence Policy Optimization) mode")
+    parser.add_argument("--use_gspo", action="store_true", default=False, help="Enable GSPO (Group Sequence Policy Optimization) mode. See: https://arxiv.org/abs/2507.18071")
+    parser.add_argument("--use_gmpo", action="store_true", default=False, help="Enable GMPO (Geometric Mean Policy Optimization) mode. See: https://arxiv.org/abs/2502.03950")
+    parser.add_argument("--clip_ratio_low", type=float, default=None, help="Lower clip bound for GSPO asymmetric clipping. Typical value: 0.0003")
+    parser.add_argument("--clip_ratio_high", type=float, default=None, help="Upper clip bound for GSPO asymmetric clipping. Typical value: 0.0004")
     parser.add_argument("--normalize_advantages", action="store_true", default=True, help="Enable advantage normalization in GSPO")
     parser.add_argument("--use_sequence_rewards", action="store_true", default=True, help="Use sequence-level rewards in GSPO")
     parser.add_argument("--value_clip", type=float, default=0.2, help="PPO value clip range")

examples/r1_aqa/train_colocate.py

@@ -304,6 +304,9 @@ def train(args):
         eps_clip=args.eps_clip,
         loss_agg_mode=args.loss_agg_mode,
         use_gspo=args.use_gspo,
+        use_gmpo=args.use_gmpo,
+        clip_ratio_low=args.clip_ratio_low,
+        clip_ratio_high=args.clip_ratio_high,
         normalize_advantages=args.normalize_advantages,
         use_sequence_rewards=args.use_sequence_rewards,
         gamma=args.gamma,
@@ -410,7 +413,10 @@ def train(args):
     parser.add_argument("--ptx_coef", type=float, default=0.05)
     parser.add_argument("--eps_clip", type=float, default=0.2)
     parser.add_argument("--loss_agg_mode", type=str, default="seq-mean-token-mean")
-    parser.add_argument("--use_gspo", action="store_true", default=False)
+    parser.add_argument("--use_gspo", action="store_true", default=False, help="Enable GSPO (Group Sequence Policy Optimization) mode. See: https://arxiv.org/abs/2507.18071")
+    parser.add_argument("--use_gmpo", action="store_true", default=False, help="Enable GMPO (Geometric Mean Policy Optimization) mode. See: https://arxiv.org/abs/2502.03950")
+    parser.add_argument("--clip_ratio_low", type=float, default=None, help="Lower clip bound for GSPO asymmetric clipping. Typical value: 0.0003")
+    parser.add_argument("--clip_ratio_high", type=float, default=None, help="Upper clip bound for GSPO asymmetric clipping. Typical value: 0.0004")
     parser.add_argument("--normalize_advantages", action="store_true", default=True)
     parser.add_argument("--use_sequence_rewards", action="store_true", default=True)
     parser.add_argument("--value_clip", type=float, default=0.2)