Repo cleanup

CITATION.cff

@@ -0,0 +1,29 @@
+cff-version: 1.2.0
+message: "If you use this code, please cite the paper below."
+preferred-citation:
+  type: article
+  title: "Policy-Driven World Model Adaptation for Robust Offline Model-based Reinforcement Learning"
+  authors:
+    - family-names: Chen
+      given-names: Jiayu
+    - family-names: Xu
+      given-names: Le
+    - family-names: Venugopal
+      given-names: Aravind
+    - family-names: Schneider
+      given-names: Jeff
+  year: 2025
+  journal: "arXiv preprint arXiv:2505.13709"
+  url: "https://arxiv.org/abs/2505.13709"
+title: "ROMBRL: Policy-Driven World Model Adaptation for Robust Offline Model-based Reinforcement Learning"
+authors:
+  - family-names: Chen
+    given-names: Jiayu
+  - family-names: Xu
+    given-names: Le
+  - family-names: Venugopal
+    given-names: Aravind
+  - family-names: Schneider
+    given-names: Jeff
+url: "https://github.com/Agentic-Intelligence-Lab/ROMBRL"
+license: MIT

D4RL/.dockerignore

@@ -0,0 +1,28 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+*.egg-info/
+log/
+logs/
+runs/
+wandb/
+tensorboard/
+*.log
+data/
+datasets/
+checkpoints/
+checkpoint/
+models/
+*.h5
+*.hdf5
+*.npy
+*.npz
+*.pkl
+*.pt
+*.pth
+*.ckpt
+build/
+dist/
+*.so
+.git/

D4RL/Dockerfile

@@ -0,0 +1,47 @@
+# ROMBRL — D4RL MuJoCo experiments (Tables 1, 2, 4)
+#
+# NOTE: this Dockerfile has been written to match D4RL/requirements.txt and
+# D4RL/README.md but has not been build-tested in this environment (no local
+# Docker daemon available). Build and smoke-test before relying on it:
+#   docker build -t rombrl-d4rl -f D4RL/Dockerfile D4RL/  # context must be D4RL/, not repo root
+#   docker run --rm -it rombrl-d4rl python run_rombrl2.py --help
+
+FROM pytorch/pytorch:2.0.1-cuda11.7-cudnn8-devel
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    MUJOCO_PY_MUJOCO_PATH=/root/.mujoco/mujoco210 \
+    LD_LIBRARY_PATH=/root/.mujoco/mujoco210/bin:${LD_LIBRARY_PATH}
+
+# System dependencies for mujoco-py / gym mujoco rendering + Cython build tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        git \
+        wget \
+        unzip \
+        patchelf \
+        libosmesa6-dev \
+        libgl1-mesa-glx \
+        libgl1-mesa-dev \
+        libglfw3 \
+        libglew-dev \
+        libglu1-mesa \
+        libglu1-mesa-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# MuJoCo 2.1.0 binary (required by mujoco-py)
+RUN mkdir -p /root/.mujoco && \
+    wget -q https://github.com/google-deepmind/mujoco/releases/download/2.1.0/mujoco210-linux-x86_64.tar.gz -O /tmp/mujoco210.tar.gz && \
+    tar -xzf /tmp/mujoco210.tar.gz -C /root/.mujoco && \
+    rm /tmp/mujoco210.tar.gz
+
+WORKDIR /workspace/D4RL
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+# Build the ctree Cython/C++ extension used by the search-based baselines
+RUN cd offlinerlkit/utils/ctree && python setup.py build_ext --inplace && cd ../../..
+
+CMD ["/bin/bash"]

D4RL/README.md

@@ -1,8 +1,17 @@
 # Policy-Driven World Model Adaptation for Robust Offline Model-based Reinforcement Learning
 
-- Please download the dynamics/reward models and hyperparameter files from [d4rl_data](https://drive.google.com/drive/folders/1FiJbpAJvul629u4VjgOyugHwcBPJyc7u?usp=sharing) to the folder `data`.
+## Setup
 
-- Please set up a virtual environment based on the instructions from [OfflineRLKit](https://github.com/yihaosun1124/OfflineRL-Kit).
+- Download the mujoco210 binary from the [MuJoCo 2.1.0 release](https://github.com/google-deepmind/mujoco/releases/tag/2.1.0) and place it at `~/.mujoco/mujoco210` (required by `mujoco-py`/`gym.envs.mujoco`).
+
+- Create a Python virtual environment (Python 3.8–3.9 recommended) and install dependencies:
+    ```bash
+    pip install -r requirements.txt
+    cd offlinerlkit/utils/ctree && python setup.py build_ext --inplace && cd ../../..
+    ```
+    This codebase builds on [OfflineRL-Kit](https://github.com/yihaosun1124/OfflineRL-Kit); consult it if you hit environment issues not covered here.
+
+- Download the dynamics/reward models and hyperparameter files from [d4rl_data](https://drive.google.com/drive/folders/1FiJbpAJvul629u4VjgOyugHwcBPJyc7u?usp=sharing) to the folder `data`.
 
 ## Table 1: Noisy D4RL MuJoCo
 
@@ -18,7 +27,7 @@ python run_XXX.py
 rombrl2, cql, edac, combo, rambo, mobile, rorl, tracer, rfqi
 ```
 
-These correspond to ROMBRL and the baselines reported in Table 1. The `run_bambrl.py` script is also kept in the repository for additional/legacy comparisons.
+These correspond to ROMBRL and the baselines reported in Table 1. The `run_bamcts.py` script is also kept in the repository for additional/legacy comparisons.
 
 To specify the task and random seed for each run, change `load_path_id` at the bottom of each `run_XXX.py` script. The default D4RL evaluation uses measurement noise controlled by `--noise_scale`; the Table 1 noisy setting uses `--noise_scale 0.05`.
 

D4RL/offlinerlkit/policy/__init__.py

@@ -16,7 +16,7 @@
 from offlinerlkit.policy.model_based.mobile import MOBILEPolicy
 from offlinerlkit.policy.model_based.rambo import RAMBOPolicy
 from offlinerlkit.policy.model_based.combo import COMBOPolicy
-from offlinerlkit.policy.model_based.bambrl import BAMBRLPolicy
+from offlinerlkit.policy.model_based.bamcts import BAMCTSPolicy
 #from offlinerlkit.policy.model_based.rombrl import ROMBRLPolicy
 from offlinerlkit.policy.model_based.rombrl2 import ROMBRL2Policy
 #from offlinerlkit.policy.model_based.rombrl3 import ROMBRL3Policy
@@ -37,7 +37,7 @@
     "MOBILEPolicy",
     "RAMBOPolicy",
     "COMBOPolicy",
-    "BAMBRLPolicy",
+    "BAMCTSPolicy",
     #"ROMBRLPolicy",
     "ROMBRL2Policy",
     #"ROMBRL3Policy"

D4RL/offlinerlkit/policy/model_based/bambrl.py → D4RL/offlinerlkit/policy/model_based/bamcts.py

@@ -11,7 +11,7 @@
 from offlinerlkit.buffer import SLReplayBuffer, SL_Transition
 from torch.distributions import Normal, Independent
 
-class BAMBRLPolicy(MOBILEPolicy):
+class BAMCTSPolicy(MOBILEPolicy):
 
     def __init__(
         self,