diff --git a/habitat-lab/habitat/core/env.py b/habitat-lab/habitat/core/env.py
index d0fe621a0a..cb09436fb6 100644
--- a/habitat-lab/habitat/core/env.py
+++ b/habitat-lab/habitat/core/env.py
@@ -102,9 +102,10 @@ def __init__(
             self._setup_episode_iterator()
             self.current_episode = next(self.episode_iterator)
             with read_write(self._config):
-                self._config.simulator.scene_dataset = (
-                    self.current_episode.scene_dataset_config
-                )
+                if self._config.simulator.scene_dataset == "default":
+                    self._config.simulator.scene_dataset = (
+                        self.current_episode.scene_dataset_config
+                    )
                 self._config.simulator.scene = self.current_episode.scene_id
 
             self.number_of_episodes = len(self.episodes)