diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6a5c82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +*.pyo +AWAC/ diff --git a/AWAC b/AWAC new file mode 160000 index 0000000..3ad931e --- /dev/null +++ b/AWAC @@ -0,0 +1 @@ +Subproject commit 3ad931ec73101798ffe82c62b19313a8607e4f1e diff --git a/alg/__pycache__/__init__.cpython-38.pyc b/alg/__pycache__/__init__.cpython-38.pyc index 3e5eda4..cf30c95 100644 Binary files a/alg/__pycache__/__init__.cpython-38.pyc and b/alg/__pycache__/__init__.cpython-38.pyc differ diff --git a/alg/awac.py b/alg/awac.py new file mode 100644 index 0000000..91476a1 --- /dev/null +++ b/alg/awac.py @@ -0,0 +1,470 @@ +from copy import deepcopy +import itertools +import numpy as np +import torch +from torch.optim import Adam + +# import d4rl +import gym +import time +from . import core + +# from alg.utils.logx import EpochLogger +import torch.nn.functional as F +import os +import warnings +from termcolor import colored + +device = torch.device("cpu") + + +class ReplayBuffer: + """ + A simple FIFO experience replay buffer for SAC agents. + """ + + def __init__(self, obs_dim, act_dim, size): + self.obs_buf = np.zeros(core.combined_shape(size, obs_dim), dtype=np.float32) + self.obs2_buf = np.zeros(core.combined_shape(size, obs_dim), dtype=np.float32) + self.act_buf = np.zeros(core.combined_shape(size, act_dim), dtype=np.float32) + self.rew_buf = np.zeros(size, dtype=np.float32) + self.done_buf = np.zeros(size, dtype=np.float32) + self.ptr, self.size, self.max_size = 0, 0, size + + def store(self, obs, act, rew, next_obs, done): + self.obs_buf[self.ptr] = obs + self.obs2_buf[self.ptr] = next_obs + self.act_buf[self.ptr] = act + self.rew_buf[self.ptr] = rew + self.done_buf[self.ptr] = done + self.ptr = (self.ptr + 1) % self.max_size + self.size = min(self.size + 1, self.max_size) + + def sample_batch(self, batch_size=32, idxs=None): + if idxs is None: + idxs = np.random.randint(0, self.size, size=batch_size) + + batch = dict( + obs=self.obs_buf[idxs], + obs2=self.obs2_buf[idxs], + act=self.act_buf[idxs], + rew=self.rew_buf[idxs], + done=self.done_buf[idxs], + ) + return {k: torch.as_tensor(v, dtype=torch.float32) for k, v in batch.items()} + + +class AWAC: + + def __init__( + self, + env_fn, + actor_critic=core.MLPActorCritic, + ac_kwargs=dict(), + seed=0, + steps_per_epoch=100, + epochs=10000, + replay_size=int(2000000), + gamma=0.99, + polyak=0.995, + lr=3e-4, + p_lr=3e-4, + alpha=0.0, + batch_size=1024, + start_steps=10000, + update_after=0, + update_every=50, + num_test_episodes=10, + max_ep_len=1000, + logger_kwargs=dict(), + save_freq=1, + algo="SAC", + ): + """ + Soft Actor-Critic (SAC) + + + Args: + env_fn : A function which creates a copy of the environment. + The environment must satisfy the OpenAI Gym API. + + actor_critic: The constructor method for a PyTorch Module with an ``act`` + method, a ``pi`` module, a ``q1`` module, and a ``q2`` module. + The ``act`` method and ``pi`` module should accept batches of + observations as inputs, and ``q1`` and ``q2`` should accept a batch + of observations and a batch of actions as inputs. When called, + ``act``, ``q1``, and ``q2`` should return: + + =========== ================ ====================================== + Call Output Shape Description + =========== ================ ====================================== + ``act`` (batch, act_dim) | Numpy array of actions for each + | observation. + ``q1`` (batch,) | Tensor containing one current estimate + | of Q* for the provided observations + | and actions. (Critical: make sure to + | flatten this!) + ``q2`` (batch,) | Tensor containing the other current + | estimate of Q* for the provided observations + | and actions. (Critical: make sure to + | flatten this!) + =========== ================ ====================================== + + Calling ``pi`` should return: + + =========== ================ ====================================== + Symbol Shape Description + =========== ================ ====================================== + ``a`` (batch, act_dim) | Tensor containing actions from policy + | given observations. + ``logp_pi`` (batch,) | Tensor containing log probabilities of + | actions in ``a``. Importantly: gradients + | should be able to flow back into ``a``. + =========== ================ ====================================== + + ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object + you provided to SAC. + + seed (int): Seed for random number generators. + + steps_per_epoch (int): Number of steps of interaction (state-action pairs) + for the agent and the environment in each epoch. + + epochs (int): Number of epochs to run and train agent. + + replay_size (int): Maximum length of replay buffer. + + gamma (float): Discount factor. (Always between 0 and 1.) + + polyak (float): Interpolation factor in polyak averaging for target + networks. Target networks are updated towards main networks + according to: + + .. math:: \\theta_{\\text{targ}} \\leftarrow + \\rho \\theta_{\\text{targ}} + (1-\\rho) \\theta + + where :math:`\\rho` is polyak. (Always between 0 and 1, usually + close to 1.) + + lr (float): Learning rate (used for both policy and value learning). + + alpha (float): Entropy regularization coefficient. (Equivalent to + inverse of reward scale in the original SAC paper.) + + batch_size (int): Minibatch size for SGD. + + start_steps (int): Number of steps for uniform-random action selection, + before running real policy. Helps exploration. + + update_after (int): Number of env interactions to collect before + starting to do gradient descent updates. Ensures replay buffer + is full enough for useful updates. + + update_every (int): Number of env interactions that should elapse + between gradient descent updates. Note: Regardless of how long + you wait between updates, the ratio of env steps to gradient steps + is locked to 1. + + num_test_episodes (int): Number of episodes to test the deterministic + policy at the end of each epoch. + + max_ep_len (int): Maximum length of trajectory / episode / rollout. + + logger_kwargs (dict): Keyword args for EpochLogger. + + save_freq (int): How often (in terms of gap between epochs) to save + the current policy and value function. + + """ + + # self.logger = EpochLogger(**logger_kwargs) + # self.logger.save_config(locals()) + + torch.manual_seed(seed) + np.random.seed(seed) + + self.env, self.test_env = env_fn(), env_fn() + self.obs_dim = self.env.observation_space.shape + self.act_dim = self.env.action_space.shape[0] + + # Action limit for clamping: critically, assumes all dimensions share the same bound! + self.act_limit = self.env.action_space.high[0] + + # Create actor-critic module and target networks + self.ac = actor_critic( + self.env.observation_space, + self.env.action_space, + special_policy="awac", + **ac_kwargs + ) + self.ac_targ = actor_critic( + self.env.observation_space, + self.env.action_space, + special_policy="awac", + **ac_kwargs + ) + self.ac_targ.load_state_dict(self.ac.state_dict()) + self.gamma = gamma + + # Freeze target networks with respect to optimizers (only update via polyak averaging) + for p in self.ac_targ.parameters(): + p.requires_grad = False + + # List of parameters for both Q-networks (save this for convenience) + self.q_params = itertools.chain( + self.ac.q1.parameters(), self.ac.q2.parameters() + ) + + # Experience buffer + self.replay_buffer = ReplayBuffer( + obs_dim=self.obs_dim, act_dim=self.act_dim, size=replay_size + ) + + # Count variables (protip: try to get a feel for how different size networks behave!) + var_counts = tuple( + core.count_vars(module) for module in [self.ac.pi, self.ac.q1, self.ac.q2] + ) + # + self.algo = algo + + self.p_lr = p_lr + self.lr = lr + self.alpha = 0 + # # Algorithm specific hyperparams + + # Set up optimizers for policy and q-function + self.pi_optimizer = Adam( + self.ac.pi.parameters(), lr=self.p_lr, weight_decay=1e-4 + ) + self.q_optimizer = Adam(self.q_params, lr=self.lr) + self.num_test_episodes = num_test_episodes + self.max_ep_len = max_ep_len + self.epochs = epochs + self.steps_per_epoch = steps_per_epoch + self.update_after = update_after + self.update_every = update_every + self.batch_size = batch_size + self.save_freq = save_freq + self.polyak = polyak + # Set up model saving + # self.logger.setup_pytorch_saver(self.ac) + print("Running Offline RL algorithm: {}".format(self.algo)) + + def populate_replay_buffer(self, env_name): + data_envs = { + "HalfCheetah-v2": ( + "awac_data/hc_action_noise_15.npy", + "awac_data/hc_off_policy_15_demos_100.npy", + ), + "Ant-v2": ( + "awac_data/ant_action_noise_15.npy", + "awac_data/ant_off_policy_15_demos_100.npy", + ), + "Walker2d-v2": ( + "awac_data/walker_action_noise_15.npy", + "awac_data/walker_off_policy_15_demos_100.npy", + ), + } + if env_name in data_envs: + print("Loading saved data") + for file in data_envs[env_name]: + if not os.path.exists(file): + warnings.warn( + colored( + "Offline data not found. Follow awac_data/instructions.txt to download. Running without offline data.", + "red", + ) + ) + break + data = np.load(file, allow_pickle=True) + for demo in data: + for transition in list( + zip( + demo["observations"], + demo["actions"], + demo["rewards"], + demo["next_observations"], + demo["terminals"], + ) + ): + self.replay_buffer.store(*transition) + else: + dataset = d4rl.qlearning_dataset(self.env) + N = dataset["rewards"].shape[0] + for i in range(N): + self.replay_buffer.store( + dataset["observations"][i], + dataset["actions"][i], + dataset["rewards"][i], + dataset["next_observations"][i], + float(dataset["terminals"][i]), + ) + print("Loaded dataset") + + # Set up function for computing SAC Q-losses + def compute_loss_q(self, data): + o, a, r, o2, d = ( + data["obs"], + data["act"], + data["rew"], + data["obs2"], + data["done"], + ) + + q1 = self.ac.q1(o, a) + q2 = self.ac.q2(o, a) + + # Bellman backup for Q functions + with torch.no_grad(): + # Target actions come from *current* policy + a2, logp_a2 = self.ac.pi(o2) + + # Target Q-values + q1_pi_targ = self.ac_targ.q1(o2, a2) + q2_pi_targ = self.ac_targ.q2(o2, a2) + q_pi_targ = torch.min(q1_pi_targ, q2_pi_targ) + backup = r + self.gamma * (1 - d) * (q_pi_targ - self.alpha * logp_a2) + + # MSE loss against Bellman backup + loss_q1 = ((q1 - backup) ** 2).mean() + loss_q2 = ((q2 - backup) ** 2).mean() + loss_q = loss_q1 + loss_q2 + + # Useful info for logging + q_info = dict(Q1Vals=q1.detach().numpy(), Q2Vals=q2.detach().numpy()) + + return loss_q, q_info + + # Set up function for computing SAC pi loss + def compute_loss_pi(self, data): + o = data["obs"] + + pi, logp_pi = self.ac.pi(o) + q1_pi = self.ac.q1(o, pi) + q2_pi = self.ac.q2(o, pi) + v_pi = torch.min(q1_pi, q2_pi) + + beta = 2 + q1_old_actions = self.ac.q1(o, data["act"]) + q2_old_actions = self.ac.q2(o, data["act"]) + q_old_actions = torch.min(q1_old_actions, q2_old_actions) + + adv_pi = q_old_actions - v_pi + weights = F.softmax(adv_pi / beta, dim=0) + policy_logpp = self.ac.pi.get_logprob(o, data["act"]) + loss_pi = (-policy_logpp * len(weights) * weights.detach()).mean() + + # Useful info for logging + pi_info = dict(LogPi=policy_logpp.detach().numpy()) + + return loss_pi, pi_info + + def update(self, data, update_timestep): + # First run one gradient descent step for Q1 and Q2 + self.q_optimizer.zero_grad() + loss_q, q_info = self.compute_loss_q(data) + loss_q.backward() + self.q_optimizer.step() + + # Record things + # self.logger.store(LossQ=loss_q.item(), **q_info) + # Freeze Q-networks so you don't waste computational effort + # computing gradients for them during the policy learning step. + for p in self.q_params: + p.requires_grad = False + + # Next run one gradient descent step for pi. + self.pi_optimizer.zero_grad() + loss_pi, pi_info = self.compute_loss_pi(data) + loss_pi.backward() + self.pi_optimizer.step() + + # Unfreeze Q-networks so you can optimize it at next DDPG step. + for p in self.q_params: + p.requires_grad = True + + # Record things + # self.logger.store(LossPi=loss_pi.item(), **pi_info) + + # Finally, update target networks by polyak averaging. + with torch.no_grad(): + for p, p_targ in zip(self.ac.parameters(), self.ac_targ.parameters()): + # NB: We use an in-place operations "mul_", "add_" to update target + # params, as opposed to "mul" and "add", which would make new tensors. + p_targ.data.mul_(self.polyak) + p_targ.data.add_((1 - self.polyak) * p.data) + + def get_action(self, o, deterministic=False): + return self.ac.act(torch.as_tensor(o, dtype=torch.float32), deterministic) + + def test_agent(self): + for j in range(self.num_test_episodes): + o, d, ep_ret, ep_len = self.test_env.reset(), False, 0, 0 + while not (d or (ep_len == self.max_ep_len)): + # Take deterministic actions at test time + o, r, d, _ = self.test_env.step(self.get_action(o, True)) + ep_ret += r + ep_len += 1 + # self.logger.store( + TestEpRet = ep_ret, TestEpLen = ep_len + # Get unnormalized score + + # # self.logger.store(TestEpRet=100*self.test_env.get_normalized_score(ep_ret), TestEpLen=ep_len) # Get normalized score + + def run(self): + # Prepare for interaction with environment + total_steps = self.epochs * self.steps_per_epoch + start_time = time.time() + obs, ep_ret, ep_len = self.env.reset(), 0, 0 + done = True + num_train_episodes = 0 + + # Main loop: collect experience in env and update/log each epoch + for t in range(total_steps): + + # Reset stuff if necessary + if done and t > 0: + # self.logger.store(ExplEpRet=ep_ret, ExplEpLen=ep_len) + + obs, ep_ret, ep_len = self.env.reset(), 0, 0 + num_train_episodes += 1 + + # Collect experience + act = self.get_action(obs, deterministic=False) + next_obs, rew, done, info = self.env.step(act) + + self.replay_buffer.store(obs, act, rew, next_obs, done) + obs = next_obs + + # Update handling + if t > self.update_after and t % self.update_every == 0: + for _ in range(self.update_every): + batch = self.replay_buffer.sample_batch(self.batch_size) + self.update(data=batch, update_timestep=t) + + # End of epoch handling + if (t + 1) % self.steps_per_epoch == 0: + epoch = (t + 1) // self.steps_per_epoch + + # Save model + if (epoch % self.save_freq == 0) or (epoch == self.epochs): + # self.logger.save_state({"env": self.env}, None) + pass + + # Test the performance of the deterministic version of the agent. + self.test_agent() + + # Log info about epoch + + +# self.logger.log_tabular("Epoch", epoch) +# self.logger.log_tabular("TestEpRet", with_min_and_max=True) +# self.logger.log_tabular("TestEpLen", average_only=True) +# self.logger.log_tabular("TotalUpdates", t) +# self.logger.log_tabular("Q1Vals", with_min_and_max=True) +# self.logger.log_tabular("Q2Vals", with_min_and_max=True) +# self.logger.log_tabular("LogPi", with_min_and_max=True) +# self.logger.log_tabular("LossPi", average_only=True) +# self.logger.log_tabular("LossQ", average_only=True) +# self.logger.log_tabular("Time", time.time() - start_time) +# self.logger.dump_tabular() diff --git a/alg/banana.py b/alg/banana.py index 2adb558..ab8dc58 100644 --- a/alg/banana.py +++ b/alg/banana.py @@ -9,9 +9,17 @@ from envs.task_envs import PnPNewRobotEnv from utils.demos import prepare_demo_pool -from utils.env_wrappers import ActionNormalizer, ResetWrapper, TimeLimitWrapper, TrajectoryRecord +from utils.env_wrappers import ( + ActionNormalizer, + ResetWrapper, + TimeLimitWrapper, + TrajectoryRecord, +) -def feature_function(traj_pairs: List[Tuple[Dict[str, np.ndarray], np.ndarray]]) -> np.ndarray: + +def feature_function( + traj_pairs: List[Tuple[Dict[str, np.ndarray], np.ndarray]], +) -> np.ndarray: if len(traj_pairs) == 0: return np.zeros(8, dtype=np.float32) @@ -118,6 +126,7 @@ def feature_function(traj_pairs: List[Tuple[Dict[str, np.ndarray], np.ndarray]]) # # return features + def capture_frame(env: Any, width: int = 320, height: int = 240) -> np.ndarray: """Render the current simulation state to an image via PyBullet offscreen rendering. @@ -165,6 +174,7 @@ def capture_frame(env: Any, width: int = 320, height: int = 240) -> np.ndarray: except Exception: return np.zeros((height, width, 3), dtype=np.uint8) + def setup_environment(*, render: bool = False) -> Any: """Construct and initialise the Pick-and-Place environment with standard wrappers. @@ -188,6 +198,7 @@ def setup_environment(*, render: bool = False) -> Any: return env + def rollout( env: Any, action_seq: np.ndarray, @@ -273,6 +284,7 @@ def random_rollout( return traj_pairs, frames + def main() -> None: """generate expert and random trajectory clips, then serialise records. @@ -283,7 +295,7 @@ def main() -> None: 4. Serialise all TrajectoryRecord objects to ``/saved/trajectory_records.json``. """ - env = setup_environment(render=True) # CHECK + env = setup_environment(render=True) # CHECK repo_root = Path(__file__).resolve().parents[1] demo_dir = repo_root / "demo_data" / "PickAndPlace" @@ -300,7 +312,6 @@ def main() -> None: feature_rows = [] - fps = 30 writer_kwargs: Dict[str, Any] = dict( fps=fps, @@ -311,7 +322,7 @@ def main() -> None: print(f"\nGenerating {len(demos)} expert clips") for i, demo in enumerate(demos): - action_seq = demo['action_trajectory'] + action_seq = demo["action_trajectory"] traj_pairs, frames = rollout(env, action_seq) clip_path = clips_dir / f"expert_{i}.mp4" @@ -320,11 +331,13 @@ def main() -> None: features = feature_function(traj_pairs) - feature_rows.append({ - "type": "expert", - "clip": str(clip_path), - **{f"f{j}": float(features[j]) for j in range(len(features))} - }) + feature_rows.append( + { + "type": "expert", + "clip": str(clip_path), + **{f"f{j}": float(features[j]) for j in range(len(features))}, + } + ) record = TrajectoryRecord( clip_path=str(clip_path), @@ -333,7 +346,6 @@ def main() -> None: saved_records.append(record) - print(f"\nGenerating 10 random clips") for i in range(10): traj_pairs, frames = random_rollout(env) @@ -344,11 +356,13 @@ def main() -> None: features = feature_function(traj_pairs) - feature_rows.append({ - "type": "random", - "clip": str(clip_path), - **{f"f{j}": float(features[j]) for j in range(len(features))} - }) + feature_rows.append( + { + "type": "random", + "clip": str(clip_path), + **{f"f{j}": float(features[j]) for j in range(len(features))}, + } + ) record = TrajectoryRecord( clip_path=str(clip_path), @@ -357,7 +371,6 @@ def main() -> None: saved_records.append(record) - env.close() out_path = saved_dir / "trajectory_records.json" @@ -377,4 +390,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/alg/core.py b/alg/core.py new file mode 100644 index 0000000..6c08b22 --- /dev/null +++ b/alg/core.py @@ -0,0 +1,328 @@ +import numpy as np +import scipy.signal + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributions.normal import Normal +import torch.distributions as D +import os +import math +import argparse +import pprint +import copy + + +device = torch.device("cpu") +def combined_shape(length, shape=None): + if shape is None: + return (length,) + return (length, shape) if np.isscalar(shape) else (length, *shape) + +def mlp(sizes, activation, output_activation=nn.Identity): + layers = [] + for j in range(len(sizes)-1): + act = activation if j < len(sizes)-2 else output_activation + layers += [nn.Linear(sizes[j], sizes[j+1]), act()] + return nn.Sequential(*layers) + +def count_vars(module): + return sum([np.prod(p.shape) for p in module.parameters()]) + + +LOG_STD_MAX = 2 +LOG_STD_MIN = -20 + +class SquashedGaussianMLPActor(nn.Module): + + def __init__(self, obs_dim, act_dim, hidden_sizes, activation, act_limit): + super().__init__() + self.net = mlp([obs_dim] + list(hidden_sizes), activation, activation) + self.mu_layer = nn.Linear(hidden_sizes[-1], act_dim) + self.log_std_layer = nn.Linear(hidden_sizes[-1], act_dim) + self.act_limit = act_limit + + def forward(self, obs, deterministic=False, with_logprob=True): + net_out = self.net(obs) + mu = self.mu_layer(net_out) + log_std = self.log_std_layer(net_out) + log_std = torch.clamp(log_std, LOG_STD_MIN, LOG_STD_MAX) + std = torch.exp(log_std) + + # Pre-squash distribution and sample + pi_distribution = Normal(mu, std) + if deterministic: + # Only used for evaluating policy at test time. + pi_action = mu + else: + pi_action = pi_distribution.rsample() + + if with_logprob: + # Compute logprob from Gaussian, and then apply correction for Tanh squashing. + # NOTE: The correction formula is a little bit magic. To get an understanding + # of where it comes from, check out the original SAC paper (arXiv 1801.01290) + # and look in appendix C. This is a more numerically-stable equivalent to Eq 21. + # Try deriving it yourself as a (very difficult) exercise. :) + logp_pi = pi_distribution.log_prob(pi_action).sum(axis=-1) + logp_pi -= (2*(np.log(2) - pi_action - F.softplus(-2*pi_action))).sum(axis=1) + else: + logp_pi = None + + pi_action = torch.tanh(pi_action) + pi_action = self.act_limit * pi_action + + return pi_action, logp_pi + + def get_logprob(self,obs, actions): + net_out = self.net(obs) + mu = self.mu_layer(net_out) + log_std = self.log_std_layer(net_out) + log_std = torch.clamp(log_std, LOG_STD_MIN, LOG_STD_MAX) + std = torch.exp(log_std) + pi_distribution = Normal(mu, std) + logp_pi = pi_distribution.log_prob(actions).sum(axis=-1) + logp_pi -= (2*(np.log(2) - actions - F.softplus(-2*actions))).sum(axis=1) + + return logp_pi + + + + +class awacMLPActor(nn.Module): + + def __init__(self, obs_dim, act_dim, hidden_sizes, activation, act_limit): + super().__init__() + self.net = mlp([obs_dim] + list(hidden_sizes), activation, activation) + self.mu_layer = nn.Linear(hidden_sizes[-1], act_dim) + + self.log_std_logits = nn.Parameter( + torch.zeros(act_dim, requires_grad=True)) + self.min_log_std = -6 + self.max_log_std = 0 + # self.log_std_layer = nn.Linear(hidden_sizes[-1], act_dim) + self.act_limit = act_limit + + def forward(self, obs, deterministic=False, with_logprob=True): + # print("Using the special policy") + net_out = self.net(obs) + mu = self.mu_layer(net_out) + mu = torch.tanh(mu) * self.act_limit + + log_std = torch.sigmoid(self.log_std_logits) + + log_std = self.min_log_std + log_std * ( + self.max_log_std - self.min_log_std) + std = torch.exp(log_std) + # print("Std: {}".format(std)) + + # Pre-squash distribution and sample + pi_distribution = Normal(mu, std) + if deterministic: + # Only used for evaluating policy at test time. + pi_action = mu + else: + pi_action = pi_distribution.rsample() + + if with_logprob: + # Compute logprob from Gaussian, and then apply correction for Tanh squashing. + # NOTE: The correction formula is a little bit magic. To get an understanding + # of where it comes from, check out the original SAC paper (arXiv 1801.01290) + # and look in appendix C. This is a more numerically-stable equivalent to Eq 21. + # Try deriving it yourself as a (very difficult) exercise. :) + logp_pi = pi_distribution.log_prob(pi_action).sum(axis=-1) + # logp_pi -= (2*(np.log(2) - pi_action - F.softplus(-2*pi_action))).sum(axis=1) + else: + logp_pi = None + + + return pi_action, logp_pi + + def get_logprob(self,obs, actions): + net_out = self.net(obs) + mu = self.mu_layer(net_out) + mu = torch.tanh(mu) * self.act_limit + log_std = torch.sigmoid(self.log_std_logits) + # log_std = self.log_std_layer(net_out) + log_std = self.min_log_std + log_std * ( + self.max_log_std - self.min_log_std) + std = torch.exp(log_std) + pi_distribution = Normal(mu, std) + logp_pi = pi_distribution.log_prob(actions).sum(axis=-1) + + return logp_pi + + + + +class MLPVFunction(nn.Module): + + def __init__(self, obs_dim, act_dim, hidden_sizes, activation): + super().__init__() + self.v = mlp([obs_dim] + list(hidden_sizes) + [1], activation) + + def forward(self, obs): + v = self.v(obs) + return torch.squeeze(v, -1) # Critical to ensure q has right shape. + +class MLPQFunction(nn.Module): + + def __init__(self, obs_dim, act_dim, hidden_sizes, activation): + super().__init__() + self.q = mlp([obs_dim + act_dim] + list(hidden_sizes) + [1], activation) + + def forward(self, obs, act): + q = self.q(torch.cat([obs, act], dim=-1)) + return torch.squeeze(q, -1) # Critical to ensure q has right shape. + +class MLPActorCritic(nn.Module): + + def __init__(self, observation_space, action_space, hidden_sizes=(256,256), + activation=nn.ReLU, special_policy=None): + super().__init__() + + obs_dim = observation_space.shape[0] + act_dim = action_space.shape[0] + act_limit = action_space.high[0] + # build policy and value functions + if special_policy is 'awac': + self.pi = awacMLPActor(obs_dim, act_dim, (256,256,256,256), activation, act_limit).to(device) + else: + self.pi = SquashedGaussianMLPActor(obs_dim, act_dim, hidden_sizes, activation, act_limit).to(device) + self.q1 = MLPQFunction(obs_dim, act_dim, hidden_sizes, activation).to(device) + self.q2 = MLPQFunction(obs_dim, act_dim, hidden_sizes, activation).to(device) + self.v = MLPVFunction(obs_dim, act_dim, hidden_sizes, activation).to(device) + + + def act_batch(self, obs, deterministic=False): + with torch.no_grad(): + a, _ = self.pi(obs, deterministic, False) + return a + + def act(self, obs, deterministic=False): + with torch.no_grad(): + a, _ = self.pi(obs, deterministic, False) + return a.cpu().data.numpy().flatten() + + + + + +# -------------------- +# Density estimator +# Model layers and helpers +# -------------------- + +def create_masks(input_size, hidden_size, n_hidden, input_order='sequential', input_degrees=None): + # MADE paper sec 4: + # degrees of connections between layers -- ensure at most in_degree - 1 connections + degrees = [] + + # set input degrees to what is provided in args (the flipped order of the previous layer in a stack of mades); + # else init input degrees based on strategy in input_order (sequential or random) + if input_order == 'sequential': + degrees += [torch.arange(input_size)] if input_degrees is None else [input_degrees] + for _ in range(n_hidden + 1): + degrees += [torch.arange(hidden_size) % (input_size - 1)] + degrees += [torch.arange(input_size) % input_size - 1] if input_degrees is None else [input_degrees % input_size - 1] + + elif input_order == 'random': + degrees += [torch.randperm(input_size)] if input_degrees is None else [input_degrees] + for _ in range(n_hidden + 1): + min_prev_degree = min(degrees[-1].min().item(), input_size - 1) + degrees += [torch.randint(min_prev_degree, input_size, (hidden_size,))] + min_prev_degree = min(degrees[-1].min().item(), input_size - 1) + degrees += [torch.randint(min_prev_degree, input_size, (input_size,)) - 1] if input_degrees is None else [input_degrees - 1] + + # construct masks + masks = [] + for (d0, d1) in zip(degrees[:-1], degrees[1:]): + masks += [(d1.unsqueeze(-1) >= d0.unsqueeze(0)).float()] + + return masks, degrees[0] + + +class MaskedLinear(nn.Linear): + """ MADE building block layer """ + def __init__(self, input_size, n_outputs, mask, cond_label_size=None): + super().__init__(input_size, n_outputs) + + self.register_buffer('mask', mask) + + self.cond_label_size = cond_label_size + if cond_label_size is not None: + self.cond_weight = nn.Parameter(torch.rand(n_outputs, cond_label_size) / math.sqrt(cond_label_size)) + + def forward(self, x, y=None): + out = F.linear(x, self.weight * self.mask, self.bias) + if y is not None: + out = out + F.linear(y, self.cond_weight) + return out + + def extra_repr(self): + return 'in_features={}, out_features={}, bias={}'.format( + self.in_features, self.out_features, self.bias is not None + ) + (self.cond_label_size != None) * ', cond_features={}'.format(self.cond_label_size) + + +class MADE(nn.Module): + def __init__(self, input_size, hidden_size, n_hidden, cond_label_size=None, activation='relu', input_order='sequential', input_degrees=None): + """ + Args: + input_size -- scalar; dim of inputs + hidden_size -- scalar; dim of hidden layers + n_hidden -- scalar; number of hidden layers + activation -- str; activation function to use + input_order -- str or tensor; variable order for creating the autoregressive masks (sequential|random) + or the order flipped from the previous layer in a stack of mades + conditional -- bool; whether model is conditional + """ + super().__init__() + # base distribution for calculation of log prob under the model + self.register_buffer('base_dist_mean', torch.zeros(input_size)) + self.register_buffer('base_dist_var', torch.ones(input_size)) + + # create masks + masks, self.input_degrees = create_masks(input_size, hidden_size, n_hidden, input_order, input_degrees) + + # setup activation + if activation == 'relu': + activation_fn = nn.ReLU() + elif activation == 'tanh': + activation_fn = nn.Tanh() + else: + raise ValueError('Check activation function.') + + # construct model + self.net_input = MaskedLinear(input_size, hidden_size, masks[0], cond_label_size) + self.net = [] + for m in masks[1:-1]: + self.net += [activation_fn, MaskedLinear(hidden_size, hidden_size, m)] + self.net += [activation_fn, MaskedLinear(hidden_size, 2 * input_size, masks[-1].repeat(2,1))] + self.net = nn.Sequential(*self.net) + + @property + def base_dist(self): + return D.Normal(self.base_dist_mean, self.base_dist_var) + + def forward(self, x, y=None): + # MAF eq 4 -- return mean and log std + m, loga = self.net(self.net_input(x, y)).chunk(chunks=2, dim=1) + u = (x - m) * torch.exp(-loga) + # MAF eq 5 + log_abs_det_jacobian = - loga + return u, log_abs_det_jacobian + + def inverse(self, u, y=None, sum_log_abs_det_jacobians=None): + # MAF eq 3 + D = u.shape[1] + x = torch.zeros_like(u) + # run through reverse model + for i in self.input_degrees: + m, loga = self.net(self.net_input(x, y)).chunk(chunks=2, dim=1) + x[:,i] = u[:,i] * torch.exp(loga[:,i]) + m[:,i] + log_abs_det_jacobian = loga + return x, log_abs_det_jacobian + + def log_prob(self, x, y=None): + u, log_abs_det_jacobian = self.forward(x, y) + return torch.sum(self.base_dist.log_prob(u) + log_abs_det_jacobian, dim=1) \ No newline at end of file diff --git a/alg/policy_learn.py b/alg/policy_learn.py new file mode 100644 index 0000000..dd01da0 --- /dev/null +++ b/alg/policy_learn.py @@ -0,0 +1,312 @@ +import os +from time import sleep +from Project_HIAL_Group1.alg.banana import feature_function, rollout +from envs.task_envs import PnPNewRobotEnv +from utils.env_wrappers import ActionNormalizer, ResetWrapper, TimeLimitWrapper +import csv +from pathlib import Path +import numpy as np +import sys +import gymnasium as gym +from utils.env_wrappers import ( + ActionNormalizer, + ResetWrapper, + TimeLimitWrapper, + reconstruct_state, +) +from utils.demos import prepare_demo_pool +import torch +from alg.awac import AWAC, ReplayBuffer +import matplotlib.pyplot as plt + +# @software{Sikchi_pytorch-AWAC, +# author = {Sikchi, Harshit and Wilcox, Albert}, +# doi = {10.5281/zenodo.5121023}, +# title = {{pytorch-AWAC}}, +# url = {https://github.com/hari-sikchi/AWAC} +# } + +# 1. Load feature weights from CSV +# 2. Create the environment +# 3. Create the AWAC agent +# 4. Load expert demos into replay buffer +# 5. Training loop (500k steps): +# a. Roll out current policy for one episode +# b. Compute rewards using comp_reward +# c. Store transitions in replay buffer +# d. Update the agent +# e. Every 1k steps: evaluate and save +# 6. Plot learning curve + + +# load weights +repo_root = Path(__file__).resolve().parents[1] +weights_path = repo_root / "saved" / "feature_weights_volume_removal.csv" +weights = [] +with open(weights_path) as fw: + reader = csv.DictReader(fw) + for row in reader: + weights.append(float(row["weight"])) +weights = np.array(weights) +# print(weights) + + +# reward function +def comp_reward(flat_states, T, weights, episode_success): + goal_start = flat_states[0].shape[0] - 3 + + obj_goal_dists = np.array( + [ + np.linalg.norm( + flat_states[t][7:10] - flat_states[t][goal_start : goal_start + 3] + ) + for t in range(T) + ] + ) + gripper_obj_dists = np.array( + [np.linalg.norm(flat_states[t][0:3] - flat_states[t][7:10]) for t in range(T)] + ) + gripper_goal_dists = np.array( + [ + np.linalg.norm( + flat_states[t][0:3] - flat_states[t][goal_start : goal_start + 3] + ) + for t in range(T) + ] + ) + success_steps = np.array([float(obj_goal_dists[t] < 0.17) for t in range(T)]) + + features = np.array( + [ + obj_goal_dists.mean(), # 0: avg object → goal distance + obj_goal_dists[-1], # 1: final object → goal distance + obj_goal_dists.min(), # 2: closest object → goal + gripper_obj_dists.mean(), # 3: avg gripper → object distance + gripper_obj_dists[-1], # 4: final gripper → object distance + gripper_goal_dists[-1], # 5: final gripper → goal distance + float(T), # 6: trajectory length + success_steps.mean(), # 7: fraction of successful steps + ], + dtype=np.float32, + ) + + total_reward = np.dot(weights, features) + return total_reward / T + + +# create awac agent + + +# create the environment +def make_env(): + env = PnPNewRobotEnv(render=False) + env = ResetWrapper(env) + env = ActionNormalizer(env) + env = TimeLimitWrapper(env, max_steps=150) + env.observation_space = gym.spaces.Box( + low=-np.inf, high=np.inf, shape=(22,), dtype=np.float32 + ) + + return env + + +# [0:3] ee_pos +# [3:6] ee_vel +# [6] finger_width +# [7:10] banana_pos +# [10:14] banana_quat +# [14:17] banana_vel +# [17:20] banana_ang_vel +# [20:23] desired_goal (plate center) +def compute_distances(flat_states, T): + goal_start = flat_states[0].shape[0] - 3 + + dists = np.array( + [ + np.linalg.norm( + flat_states[t][7:10] - flat_states[t][goal_start : goal_start + 3] + ) + for t in range(T) + ] + ) + arm_dists = np.array( + [np.linalg.norm(flat_states[t][0:3] - flat_states[t][7:10]) for t in range(T)] + ) + return dists, arm_dists + + +def main() -> None: + os.makedirs("saved/policy_learning_curve_steps", exist_ok=True) + # create the AWAC agent + agent = AWAC(env_fn=make_env) + + # load expert demos into replay buffer and store + demo_dir = repo_root / "demo_data" / "PickAndPlace" + demos = prepare_demo_pool(demo_dir, verbose=True) + + rollout_env = setup_environment(render=False) + + for demo in demos: + states = demo["state_trajectory"] + actions = demo["action_trajectory"] + next_states = demo["next_state_trajectory"] + rewards = demo["reward_trajectory"] + dones = demo["done_trajectory"] + T = len(actions) + + traj_pairs, _ = rollout(rollout_env, actions) + features = feature_function(traj_pairs) + total_reward = np.dot(weights, features) + per_step_reward = total_reward / T + + episode_success = bool(np.squeeze(dones[-1])) + + for t in range(T): + agent.replay_buffer.store( + states[t], + actions[t], + per_step_reward, + next_states[t], + float(np.squeeze(dones[t])), + ) + + # check demo state + print(f"Replay buffer size after demos: {agent.replay_buffer.size}") + # print("demo state shape:", states[0].shape) + # print("demo state:", states[0]) + # check live env state + + # untested from here on + # training loop + # 5. Training loop (500k steps): + # a. Roll out current policy for one episode + # b. Compute rewards using comp_reward + # c. Store transitions in replay buffer + # d. Update the agent + # e. Every 1k steps: evaluate and save + # 6. Plot learning curve + + # state 7:10 = banana position + # state 19:22 = goal position, subtract these + + # print("agent.env", agent.env) + # print("demo state, ", states) + # print("demo state[7], ", states[7]) + # print(states[0][19:]) + total_steps = 0 + max_steps = 10000 + last_save = 0 + steps = [] + success_rates = [] + eval_env = make_env() + + while total_steps < max_steps: + obs_dict, info = agent.env.reset() + obs = reconstruct_state(obs_dict) if isinstance(obs_dict, dict) else obs_dict + done = False + currEpisode_states = [obs] + currEpisode_actions = [] + currEpisode_info = [] + + while not done: + action = agent.get_action(obs) + + next_obs_dict, _, terminated, truncated, info = agent.env.step(action) + next_obs = ( + reconstruct_state(next_obs_dict) + if isinstance(next_obs_dict, dict) + else next_obs_dict + ) + done = terminated or truncated + + currEpisode_states.append(next_obs) + currEpisode_actions.append(action) + currEpisode_info.append(info) + obs = next_obs + total_steps += 1 + # print(f"total_steps: {total_steps}") + + T = len(currEpisode_actions) + # print("currEpisode_states", currEpisode_states) + + # test + episode_success = bool(currEpisode_info[-1].get("is_success", False)) + + per_step_reward = comp_reward(currEpisode_states, T, weights, episode_success) + print( + f"Step {total_steps} | Train ep success: {episode_success} | final info: {currEpisode_info[-1] } | Reward {per_step_reward}" + ) + + for t in range(T): + # store transitions in replay buffer + agent.replay_buffer.store( + currEpisode_states[t], + currEpisode_actions[t], + per_step_reward, + currEpisode_states[t + 1], + float(t == T - 1), + ) + + # update agent — multiple updates per episode (1:1 update-to-data ratio) + if agent.replay_buffer.size > agent.batch_size: + for _ in range(T): + batch = agent.replay_buffer.sample_batch(agent.batch_size) + agent.update(data=batch, update_timestep=total_steps) + + # print(f"Loss Q: {agent.compute_loss_q(data=batch)}, Loss Pi: {agent.compute_loss_pi(data=batch)}") + + # every 1k eval and save + # had to do this way because it was skipping over 1000 with the 150 episode so total_steps % 1000 was never hitting + if total_steps - last_save >= 1000: + # agent.ac.state_dict() returns all the NN weights & biases as a dictionary + torch.save( + agent.ac.state_dict(), + f"saved/policy_learning_curve_steps/policy_{total_steps}.pt", + ) + last_save = total_steps + print(f"Saved policy at step {total_steps}") + + successes = 0 + for _ in range(10): + obs_dict, _ = eval_env.reset() + obs_eval = reconstruct_state(obs_dict) + terminated = truncated = False + + while not (terminated or truncated): + action = agent.get_action(obs_eval, deterministic=True) + next_obs_dict, _, terminated, truncated, eval_info = eval_env.step( + action + ) + obs_eval = reconstruct_state(next_obs_dict) + + # Call is_success manually to see what it should return + achieved_goal = eval_env.unwrapped.task.get_achieved_goal() + # Object position + desired_goal = eval_env.unwrapped.task.get_goal() # Target position + dist = np.linalg.norm(achieved_goal - desired_goal) + print( + f"Step {total_steps} | Eval ep dist: {dist:.4f} | threshold: {eval_env.unwrapped.task.distance_threshold}" + ) + # print(f"Manual is_success call: {manual_success}") + # print(f"Distance: {np.linalg.norm(achieved_goal - desired_goal)}") + # print(f"Threshold: {eval_env.unwrapped.task.distance_threshold}") + + if eval_info.get("is_success", False): + successes += 1 + + success_rate = successes / 10 + steps.append(total_steps) + success_rates.append(success_rate) + + if steps and success_rates: + fix, ax = plt.subplots() + ax.plot(steps, success_rates) + ax.set_xlabel("Environment Steps", fontsize=12) + ax.set_ylabel("Average Success Rate", fontsize=12) + plt.savefig("saved/learning_curve.png") + + # need to fix success rate rollouts because its not working + + +if __name__ == "__main__": + main() diff --git a/alg/policy_test.py b/alg/policy_test.py new file mode 100644 index 0000000..0a8f5d6 --- /dev/null +++ b/alg/policy_test.py @@ -0,0 +1,44 @@ +import torch +import numpy as np +from pathlib import Path +from envs.task_envs import PnPNewRobotEnv +from utils.env_wrappers import ( + ActionNormalizer, + ResetWrapper, + TimeLimitWrapper, + reconstruct_state, +) +from alg.awac import AWAC, ReplayBuffer +from alg.policy_learn import make_env, FlattenObsWrapper + + +def load_final_policy(path_to_saved_policy): + """ + load your final trained policy + + Args: + path_to_saved_policy (str): the path to your saved policy model + + Returns: + your saved policy model under the corresponding path + """ + agent = AWAC(env_fn=make_env) + agent.ac.load_state_dict(torch.load(path_to_saved_policy)) + agent.ac.eval() # set to evaluation mode + return agent + + +def get_policy_action(state, saved_policy_model): + """ + get the action that the policy decides to take for the given environment state + + Args: + state (dict): the state of the environment returned by the env.step() or env.reset(), which is a dictionary including keys of "observation", "achieved_goal", and "desired_goal" + saved_policy_model: a saved model in the same format as the one returned by your load_final_policy() function + + Returns: + action (np.array): the action that the saved policy model decides to take under the given state + """ + flat_state = reconstruct_state(state) + action = saved_policy_model.get_action(flat_state, deterministic=True) + return action diff --git a/alg/pref_learn.py b/alg/pref_learn.py index 7cb93da..0e86969 100644 --- a/alg/pref_learn.py +++ b/alg/pref_learn.py @@ -47,12 +47,13 @@ def setup_environment(*, render: bool = False) -> Any: return env + def learn_weights( traj_set: TrajectorySet, *, num_queries: int = 10, seed: int = 0, - acquisition_function: str + acquisition_function: str, ) -> np.ndarray: """Run an active preference learning loop to infer reward feature weights. # @@ -75,11 +76,10 @@ def learn_weights( # Returns: # A float32 array of shape (feature_dim,) containing the posterior # mean reward weights after all queries have been collected. - # """ + #""" # # https://github.com/Stanford-ILIAD/APReL # # use this for the APReL's query optimizer together with a SamplingBasedBelief. - valid_acquisition_functions = { "disagreement", "mutual_information", @@ -99,9 +99,7 @@ def learn_weights( print("Feature dimension:", feature_dim) query_optimizer = QueryOptimizerDiscreteTrajectorySet(traj_set) - params = { - "weights": util_funs.get_random_normalized_vector(feature_dim) - } + params = {"weights": util_funs.get_random_normalized_vector(feature_dim)} user_model = SoftmaxUser(params) belief = SamplingBasedBelief(user_model, [], params) @@ -110,9 +108,7 @@ def learn_weights( for i in range(num_queries): queries, objective_values = query_optimizer.optimize( - acquisition_function, - belief, - query + acquisition_function, belief, query ) best_query = queries[0] @@ -127,7 +123,8 @@ def learn_weights( print("Estimated user parameters:", belief.mean) - return belief.mean['weights'].astype(np.float32) + return belief.mean["weights"].astype(np.float32) + def save_weights(weights: np.ndarray, out_path: Path) -> None: """Serialise learned feature weights to a two-column CSV file. @@ -186,7 +183,12 @@ def main() -> None: out_path = saved_dir / f"feature_weights_{acquisition_function}.csv" try: - weights = learn_weights(trajectories, num_queries=10, seed=0, acquisition_function = acquisition_function) + weights = learn_weights( + trajectories, + num_queries=10, + seed=0, + acquisition_function=acquisition_function, + ) except Exception as e: raise finally: @@ -197,4 +199,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/alg/utils/logx.py b/alg/utils/logx.py new file mode 100644 index 0000000..e16896e --- /dev/null +++ b/alg/utils/logx.py @@ -0,0 +1,415 @@ +""" + +Some simple logging functionality, inspired by rllab's logging. + +Logs to a tab-separated-values file (path/to/output_directory/progress.txt) + +""" + +import json +import joblib +import shutil +import numpy as np +import tensorflow as tf +import torch +import os.path as osp, time, atexit, os +import warnings +from alg.utils.mpi_tools import proc_id, mpi_statistics_scalar +from utils.serialization_utils import convert_json + +color2num = dict( + gray=30, + red=31, + green=32, + yellow=33, + blue=34, + magenta=35, + cyan=36, + white=37, + crimson=38, +) + + +def colorize(string, color, bold=False, highlight=False): + """ + Colorize a string. + + This function was originally written by John Schulman. + """ + attr = [] + num = color2num[color] + if highlight: + num += 10 + attr.append(str(num)) + if bold: + attr.append("1") + return "\x1b[%sm%s\x1b[0m" % (";".join(attr), string) + + +def restore_tf_graph(sess, fpath): + """ + Loads graphs saved by Logger. + + Will output a dictionary whose keys and values are from the 'inputs' + and 'outputs' dict you specified with logger.setup_tf_saver(). + + Args: + sess: A Tensorflow session. + fpath: Filepath to save directory. + + Returns: + A dictionary mapping from keys to tensors in the computation graph + loaded from ``fpath``. + """ + tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], fpath) + model_info = joblib.load(osp.join(fpath, "model_info.pkl")) + graph = tf.get_default_graph() + model = dict() + model.update( + {k: graph.get_tensor_by_name(v) for k, v in model_info["inputs"].items()} + ) + model.update( + {k: graph.get_tensor_by_name(v) for k, v in model_info["outputs"].items()} + ) + return model + + +class Logger: + """ + A general-purpose logger. + + Makes it easy to save diagnostics, hyperparameter configurations, the + state of a training run, and the trained model. + """ + + def __init__(self, output_dir=None, output_fname="progress.txt", exp_name=None): + """ + Initialize a Logger. + + Args: + output_dir (string): A directory for saving results to. If + ``None``, defaults to a temp directory of the form + ``/tmp/experiments/somerandomnumber``. + + output_fname (string): Name for the tab-separated-value file + containing metrics logged throughout a training run. + Defaults to ``progress.txt``. + + exp_name (string): Experiment name. If you run multiple training + runs and give them all the same ``exp_name``, the plotter + will know to group them. (Use case: if you run the same + hyperparameter configuration with multiple random seeds, you + should give them all the same ``exp_name``.) + """ + if proc_id() == 0: + self.output_dir = output_dir or "/tmp/experiments/%i" % int(time.time()) + if osp.exists(self.output_dir): + print( + "Warning: Log dir %s already exists! Storing info there anyway." + % self.output_dir + ) + else: + os.makedirs(self.output_dir) + self.output_file = open(osp.join(self.output_dir, output_fname), "w") + atexit.register(self.output_file.close) + print( + colorize( + "Logging data to %s" % self.output_file.name, "green", bold=True + ) + ) + else: + self.output_dir = None + self.output_file = None + self.first_row = True + self.log_headers = [] + self.log_current_row = {} + self.exp_name = exp_name + + def log(self, msg, color="green"): + """Print a colorized message to stdout.""" + if proc_id() == 0: + print(colorize(msg, color, bold=True)) + + def log_tabular(self, key, val): + """ + Log a value of some diagnostic. + + Call this only once for each diagnostic quantity, each iteration. + After using ``log_tabular`` to store values for each diagnostic, + make sure to call ``dump_tabular`` to write them out to file and + stdout (otherwise they will not get saved anywhere). + """ + if self.first_row: + self.log_headers.append(key) + else: + assert key in self.log_headers, ( + "Trying to introduce a new key %s that you didn't include in the first iteration" + % key + ) + assert key not in self.log_current_row, ( + "You already set %s this iteration. Maybe you forgot to call dump_tabular()" + % key + ) + self.log_current_row[key] = val + + def save_config(self, config): + """ + Log an experiment configuration. + + Call this once at the top of your experiment, passing in all important + config vars as a dict. This will serialize the config to JSON, while + handling anything which can't be serialized in a graceful way (writing + as informative a string as possible). + + Example use: + + .. code-block:: python + + logger = EpochLogger(**logger_kwargs) + logger.save_config(locals()) + """ + config_json = convert_json(config) + if self.exp_name is not None: + config_json["exp_name"] = self.exp_name + if proc_id() == 0: + output = json.dumps( + config_json, separators=(",", ":\t"), indent=4, sort_keys=True + ) + print(colorize("Saving config:\n", color="cyan", bold=True)) + print(output) + with open(osp.join(self.output_dir, "config.json"), "w") as out: + out.write(output) + + def save_state(self, state_dict, itr=None): + """ + Saves the state of an experiment. + + To be clear: this is about saving *state*, not logging diagnostics. + All diagnostic logging is separate from this function. This function + will save whatever is in ``state_dict``---usually just a copy of the + environment---and the most recent parameters for the model you + previously set up saving for with ``setup_tf_saver``. + + Call with any frequency you prefer. If you only want to maintain a + single state and overwrite it at each call with the most recent + version, leave ``itr=None``. If you want to keep all of the states you + save, provide unique (increasing) values for 'itr'. + + Args: + state_dict (dict): Dictionary containing essential elements to + describe the current state of training. + + itr: An int, or None. Current iteration of training. + """ + if proc_id() == 0: + fname = "vars.pkl" if itr is None else "vars%d.pkl" % itr + try: + joblib.dump(state_dict, osp.join(self.output_dir, fname)) + except: + self.log("Warning: could not pickle state_dict.", color="red") + if hasattr(self, "tf_saver_elements"): + self._tf_simple_save(itr) + if hasattr(self, "pytorch_saver_elements"): + self._pytorch_simple_save(itr) + + def setup_tf_saver(self, sess, inputs, outputs): + """ + Set up easy model saving for tensorflow. + + Call once, after defining your computation graph but before training. + + Args: + sess: The Tensorflow session in which you train your computation + graph. + + inputs (dict): A dictionary that maps from keys of your choice + to the tensorflow placeholders that serve as inputs to the + computation graph. Make sure that *all* of the placeholders + needed for your outputs are included! + + outputs (dict): A dictionary that maps from keys of your choice + to the outputs from your computation graph. + """ + self.tf_saver_elements = dict(session=sess, inputs=inputs, outputs=outputs) + self.tf_saver_info = { + "inputs": {k: v.name for k, v in inputs.items()}, + "outputs": {k: v.name for k, v in outputs.items()}, + } + + def _tf_simple_save(self, itr=None): + """ + Uses simple_save to save a trained model, plus info to make it easy + to associated tensors to variables after restore. + """ + if proc_id() == 0: + assert hasattr( + self, "tf_saver_elements" + ), "First have to setup saving with self.setup_tf_saver" + fpath = "tf1_save" + ("%d" % itr if itr is not None else "") + fpath = osp.join(self.output_dir, fpath) + if osp.exists(fpath): + # simple_save refuses to be useful if fpath already exists, + # so just delete fpath if it's there. + shutil.rmtree(fpath) + tf.saved_model.simple_save(export_dir=fpath, **self.tf_saver_elements) + joblib.dump(self.tf_saver_info, osp.join(fpath, "model_info.pkl")) + + def setup_pytorch_saver(self, what_to_save): + """ + Set up easy model saving for a single PyTorch model. + + Because PyTorch saving and loading is especially painless, this is + very minimal; we just need references to whatever we would like to + pickle. This is integrated into the logger because the logger + knows where the user would like to save information about this + training run. + + Args: + what_to_save: Any PyTorch model or serializable object containing + PyTorch models. + """ + self.pytorch_saver_elements = what_to_save + + def _pytorch_simple_save(self, itr=None): + """ + Saves the PyTorch model (or models). + """ + if proc_id() == 0: + assert hasattr( + self, "pytorch_saver_elements" + ), "First have to setup saving with self.setup_pytorch_saver" + fpath = "pyt_save" + fpath = osp.join(self.output_dir, fpath) + fname = "model" + ("%d" % itr if itr is not None else "") + ".pt" + fname = osp.join(fpath, fname) + os.makedirs(fpath, exist_ok=True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + # We are using a non-recommended way of saving PyTorch models, + # by pickling whole objects (which are dependent on the exact + # directory structure at the time of saving) as opposed to + # just saving network weights. This works sufficiently well + # for the purposes of Spinning Up, but you may want to do + # something different for your personal PyTorch project. + # We use a catch_warnings() context to avoid the warnings about + # not being able to save the source code. + torch.save(self.pytorch_saver_elements, fname) + + def dump_tabular(self): + """ + Write all of the diagnostics from the current iteration. + + Writes both to stdout, and to the output file. + """ + if proc_id() == 0: + vals = [] + key_lens = [len(key) for key in self.log_headers] + max_key_len = max(15, max(key_lens)) + keystr = "%" + "%d" % max_key_len + fmt = "| " + keystr + "s | %15s |" + n_slashes = 22 + max_key_len + print("-" * n_slashes) + for key in self.log_headers: + val = self.log_current_row.get(key, "") + valstr = "%8.3g" % val if hasattr(val, "__float__") else val + print(fmt % (key, valstr)) + vals.append(val) + print("-" * n_slashes, flush=True) + if self.output_file is not None: + if self.first_row: + self.output_file.write("\t".join(self.log_headers) + "\n") + self.output_file.write("\t".join(map(str, vals)) + "\n") + self.output_file.flush() + self.log_current_row.clear() + self.first_row = False + + +class EpochLogger(Logger): + """ + A variant of Logger tailored for tracking average values over epochs. + + Typical use case: there is some quantity which is calculated many times + throughout an epoch, and at the end of the epoch, you would like to + report the average / std / min / max value of that quantity. + + With an EpochLogger, each time the quantity is calculated, you would + use + + .. code-block:: python + + epoch_logger.store(NameOfQuantity=quantity_value) + + to load it into the EpochLogger's state. Then at the end of the epoch, you + would use + + .. code-block:: python + + epoch_logger.log_tabular(NameOfQuantity, **options) + + to record the desired values. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.epoch_dict = dict() + + def store(self, **kwargs): + """ + Save something into the epoch_logger's current state. + + Provide an arbitrary number of keyword arguments with numerical + values. + """ + for k, v in kwargs.items(): + if not (k in self.epoch_dict.keys()): + self.epoch_dict[k] = [] + self.epoch_dict[k].append(v) + + def log_tabular(self, key, val=None, with_min_and_max=False, average_only=False): + """ + Log a value or possibly the mean/std/min/max values of a diagnostic. + + Args: + key (string): The name of the diagnostic. If you are logging a + diagnostic whose state has previously been saved with + ``store``, the key here has to match the key you used there. + + val: A value for the diagnostic. If you have previously saved + values for this key via ``store``, do *not* provide a ``val`` + here. + + with_min_and_max (bool): If true, log min and max values of the + diagnostic over the epoch. + + average_only (bool): If true, do not log the standard deviation + of the diagnostic over the epoch. + """ + if val is not None: + super().log_tabular(key, val) + else: + v = self.epoch_dict[key] + vals = ( + np.concatenate(v) + if isinstance(v[0], np.ndarray) and len(v[0].shape) > 0 + else v + ) + stats = mpi_statistics_scalar(vals, with_min_and_max=with_min_and_max) + super().log_tabular(key if average_only else "Average" + key, stats[0]) + if not (average_only): + super().log_tabular("Std" + key, stats[1]) + if with_min_and_max: + super().log_tabular("Max" + key, stats[3]) + super().log_tabular("Min" + key, stats[2]) + self.epoch_dict[key] = [] + + def get_stats(self, key): + """ + Lets an algorithm ask the logger for mean/std/min/max of a diagnostic. + """ + v = self.epoch_dict[key] + vals = ( + np.concatenate(v) + if isinstance(v[0], np.ndarray) and len(v[0].shape) > 0 + else v + ) + return mpi_statistics_scalar(vals) diff --git a/alg/utils/mpi_tools.py b/alg/utils/mpi_tools.py new file mode 100644 index 0000000..0d03212 --- /dev/null +++ b/alg/utils/mpi_tools.py @@ -0,0 +1,92 @@ +from mpi4py import MPI +import os, subprocess, sys +import numpy as np + + +def mpi_fork(n, bind_to_core=False): + """ + Re-launches the current script with workers linked by MPI. + + Also, terminates the original process that launched it. + + Taken almost without modification from the Baselines function of the + `same name`_. + + .. _`same name`: https://github.com/openai/baselines/blob/master/baselines/common/mpi_fork.py + + Args: + n (int): Number of process to split into. + + bind_to_core (bool): Bind each MPI process to a core. + """ + if n<=1: + return + if os.getenv("IN_MPI") is None: + env = os.environ.copy() + env.update( + MKL_NUM_THREADS="1", + OMP_NUM_THREADS="1", + IN_MPI="1" + ) + args = ["mpirun", "-np", str(n)] + if bind_to_core: + args += ["-bind-to", "core"] + args += [sys.executable] + sys.argv + subprocess.check_call(args, env=env) + sys.exit() + + +def msg(m, string=''): + print(('Message from %d: %s \t '%(MPI.COMM_WORLD.Get_rank(), string))+str(m)) + +def proc_id(): + """Get rank of calling process.""" + return MPI.COMM_WORLD.Get_rank() + +def allreduce(*args, **kwargs): + return MPI.COMM_WORLD.Allreduce(*args, **kwargs) + +def num_procs(): + """Count active MPI processes.""" + return MPI.COMM_WORLD.Get_size() + +def broadcast(x, root=0): + MPI.COMM_WORLD.Bcast(x, root=root) + +def mpi_op(x, op): + x, scalar = ([x], True) if np.isscalar(x) else (x, False) + x = np.asarray(x, dtype=np.float32) + buff = np.zeros_like(x, dtype=np.float32) + allreduce(x, buff, op=op) + return buff[0] if scalar else buff + +def mpi_sum(x): + return mpi_op(x, MPI.SUM) + +def mpi_avg(x): + """Average a scalar or vector over MPI processes.""" + return mpi_sum(x) / num_procs() + +def mpi_statistics_scalar(x, with_min_and_max=False): + """ + Get mean/std and optional min/max of scalar x across MPI processes. + + Args: + x: An array containing samples of the scalar to produce statistics + for. + + with_min_and_max (bool): If true, return min and max of x in + addition to mean and std. + """ + x = np.array(x, dtype=np.float32) + global_sum, global_n = mpi_sum([np.sum(x), len(x)]) + mean = global_sum / global_n + + global_sum_sq = mpi_sum(np.sum((x - mean)**2)) + std = np.sqrt(global_sum_sq / global_n) # compute global std + + if with_min_and_max: + global_min = mpi_op(np.min(x) if len(x) > 0 else np.inf, op=MPI.MIN) + global_max = mpi_op(np.max(x) if len(x) > 0 else -np.inf, op=MPI.MAX) + return mean, std, global_min, global_max + return mean, std \ No newline at end of file diff --git a/alg/utils/serialization_utils.py b/alg/utils/serialization_utils.py new file mode 100644 index 0000000..a21e94a --- /dev/null +++ b/alg/utils/serialization_utils.py @@ -0,0 +1,33 @@ +import json + +def convert_json(obj): + """ Convert obj to a version which can be serialized with JSON. """ + if is_json_serializable(obj): + return obj + else: + if isinstance(obj, dict): + return {convert_json(k): convert_json(v) + for k,v in obj.items()} + + elif isinstance(obj, tuple): + return (convert_json(x) for x in obj) + + elif isinstance(obj, list): + return [convert_json(x) for x in obj] + + elif hasattr(obj,'__name__') and not('lambda' in obj.__name__): + return convert_json(obj.__name__) + + elif hasattr(obj,'__dict__') and obj.__dict__: + obj_dict = {convert_json(k): convert_json(v) + for k,v in obj.__dict__.items()} + return {str(obj): obj_dict} + + return str(obj) + +def is_json_serializable(v): + try: + json.dumps(v) + return True + except: + return False \ No newline at end of file diff --git a/envs/__pycache__/__init__.cpython-38.pyc b/envs/__pycache__/__init__.cpython-38.pyc index ccaf83e..cfa6c30 100644 Binary files a/envs/__pycache__/__init__.cpython-38.pyc and b/envs/__pycache__/__init__.cpython-38.pyc differ diff --git a/envs/__pycache__/assets.cpython-38.pyc b/envs/__pycache__/assets.cpython-38.pyc index 918a716..7cda889 100644 Binary files a/envs/__pycache__/assets.cpython-38.pyc and b/envs/__pycache__/assets.cpython-38.pyc differ diff --git a/envs/__pycache__/task_envs.cpython-38.pyc b/envs/__pycache__/task_envs.cpython-38.pyc index c38be5a..487856e 100644 Binary files a/envs/__pycache__/task_envs.cpython-38.pyc and b/envs/__pycache__/task_envs.cpython-38.pyc differ diff --git a/envs/tasks/__pycache__/__init__.cpython-38.pyc b/envs/tasks/__pycache__/__init__.cpython-38.pyc index 557dc8a..cb18f0f 100644 Binary files a/envs/tasks/__pycache__/__init__.cpython-38.pyc and b/envs/tasks/__pycache__/__init__.cpython-38.pyc differ diff --git a/envs/tasks/__pycache__/pick_and_place.cpython-38.pyc b/envs/tasks/__pycache__/pick_and_place.cpython-38.pyc index f62b68c..fb5e881 100644 Binary files a/envs/tasks/__pycache__/pick_and_place.cpython-38.pyc and b/envs/tasks/__pycache__/pick_and_place.cpython-38.pyc differ diff --git a/envs/tasks/__pycache__/ur_robot.cpython-38.pyc b/envs/tasks/__pycache__/ur_robot.cpython-38.pyc index dc8bfd1..97341fe 100644 Binary files a/envs/tasks/__pycache__/ur_robot.cpython-38.pyc and b/envs/tasks/__pycache__/ur_robot.cpython-38.pyc differ diff --git a/saved/feature_weights_volume_removal.csv b/saved/feature_weights_volume_removal.csv index e4e52b2..24f7b4c 100644 --- a/saved/feature_weights_volume_removal.csv +++ b/saved/feature_weights_volume_removal.csv @@ -1,8 +1,9 @@ -feature_index,weight -0,-0.29096025228500366 -1,-0.1516142040491104 -2,-0.07159309089183807 -3,-0.21884848177433014 -4,0.11546217650175095 -5,0.9051831960678101 -6,0.08152846246957779 +feature_index,weight +0,-0.16435712575912476 +1,-0.2010500431060791 +2,0.3091944754123688 +3,0.39045679569244385 +4,-0.49459245800971985 +5,0.17358191311359406 +6,0.615068793296814 +7,-0.17733001708984375 diff --git a/saved/learning_curve.png b/saved/learning_curve.png new file mode 100644 index 0000000..f129773 Binary files /dev/null and b/saved/learning_curve.png differ diff --git a/saved/policy_learning_curve_steps/policy_10497.pt b/saved/policy_learning_curve_steps/policy_10497.pt new file mode 100644 index 0000000..9ff9785 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_10497.pt differ diff --git a/saved/policy_learning_curve_steps/policy_1050.pt b/saved/policy_learning_curve_steps/policy_1050.pt new file mode 100644 index 0000000..65f0457 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_1050.pt differ diff --git a/saved/policy_learning_curve_steps/policy_1143.pt b/saved/policy_learning_curve_steps/policy_1143.pt new file mode 100644 index 0000000..9c6dd6e Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_1143.pt differ diff --git a/saved/policy_learning_curve_steps/policy_11530.pt b/saved/policy_learning_curve_steps/policy_11530.pt new file mode 100644 index 0000000..d8f5696 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_11530.pt differ diff --git a/saved/policy_learning_curve_steps/policy_12579.pt b/saved/policy_learning_curve_steps/policy_12579.pt new file mode 100644 index 0000000..44e8544 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_12579.pt differ diff --git a/saved/policy_learning_curve_steps/policy_13629.pt b/saved/policy_learning_curve_steps/policy_13629.pt new file mode 100644 index 0000000..f9edadc Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_13629.pt differ diff --git a/saved/policy_learning_curve_steps/policy_14765.pt b/saved/policy_learning_curve_steps/policy_14765.pt new file mode 100644 index 0000000..4d1712c Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_14765.pt differ diff --git a/saved/policy_learning_curve_steps/policy_15869.pt b/saved/policy_learning_curve_steps/policy_15869.pt new file mode 100644 index 0000000..b9b10f2 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_15869.pt differ diff --git a/saved/policy_learning_curve_steps/policy_16919.pt b/saved/policy_learning_curve_steps/policy_16919.pt new file mode 100644 index 0000000..b416f6e Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_16919.pt differ diff --git a/saved/policy_learning_curve_steps/policy_17927.pt b/saved/policy_learning_curve_steps/policy_17927.pt new file mode 100644 index 0000000..fd9281b Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_17927.pt differ diff --git a/saved/policy_learning_curve_steps/policy_18977.pt b/saved/policy_learning_curve_steps/policy_18977.pt new file mode 100644 index 0000000..0f50602 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_18977.pt differ diff --git a/saved/policy_learning_curve_steps/policy_20027.pt b/saved/policy_learning_curve_steps/policy_20027.pt new file mode 100644 index 0000000..ea174bb Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_20027.pt differ diff --git a/saved/policy_learning_curve_steps/policy_2100.pt b/saved/policy_learning_curve_steps/policy_2100.pt new file mode 100644 index 0000000..04d8cc0 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_2100.pt differ diff --git a/saved/policy_learning_curve_steps/policy_21143.pt b/saved/policy_learning_curve_steps/policy_21143.pt new file mode 100644 index 0000000..4489d83 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_21143.pt differ diff --git a/saved/policy_learning_curve_steps/policy_2193.pt b/saved/policy_learning_curve_steps/policy_2193.pt new file mode 100644 index 0000000..b96939f Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_2193.pt differ diff --git a/saved/policy_learning_curve_steps/policy_22193.pt b/saved/policy_learning_curve_steps/policy_22193.pt new file mode 100644 index 0000000..7115193 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_22193.pt differ diff --git a/saved/policy_learning_curve_steps/policy_23243.pt b/saved/policy_learning_curve_steps/policy_23243.pt new file mode 100644 index 0000000..793d007 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_23243.pt differ diff --git a/saved/policy_learning_curve_steps/policy_24293.pt b/saved/policy_learning_curve_steps/policy_24293.pt new file mode 100644 index 0000000..c47d5ef Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_24293.pt differ diff --git a/saved/policy_learning_curve_steps/policy_25343.pt b/saved/policy_learning_curve_steps/policy_25343.pt new file mode 100644 index 0000000..b50bb5f Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_25343.pt differ diff --git a/saved/policy_learning_curve_steps/policy_26393.pt b/saved/policy_learning_curve_steps/policy_26393.pt new file mode 100644 index 0000000..044c9f2 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_26393.pt differ diff --git a/saved/policy_learning_curve_steps/policy_27443.pt b/saved/policy_learning_curve_steps/policy_27443.pt new file mode 100644 index 0000000..a919aa0 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_27443.pt differ diff --git a/saved/policy_learning_curve_steps/policy_3216.pt b/saved/policy_learning_curve_steps/policy_3216.pt new file mode 100644 index 0000000..d67e20a Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_3216.pt differ diff --git a/saved/policy_learning_curve_steps/policy_3339.pt b/saved/policy_learning_curve_steps/policy_3339.pt new file mode 100644 index 0000000..d4b07ed Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_3339.pt differ diff --git a/saved/policy_learning_curve_steps/policy_4266.pt b/saved/policy_learning_curve_steps/policy_4266.pt new file mode 100644 index 0000000..db570d3 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_4266.pt differ diff --git a/saved/policy_learning_curve_steps/policy_4389.pt b/saved/policy_learning_curve_steps/policy_4389.pt new file mode 100644 index 0000000..988a7d7 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_4389.pt differ diff --git a/saved/policy_learning_curve_steps/policy_5271.pt b/saved/policy_learning_curve_steps/policy_5271.pt new file mode 100644 index 0000000..741471e Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_5271.pt differ diff --git a/saved/policy_learning_curve_steps/policy_5478.pt b/saved/policy_learning_curve_steps/policy_5478.pt new file mode 100644 index 0000000..cce8e7e Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_5478.pt differ diff --git a/saved/policy_learning_curve_steps/policy_6314.pt b/saved/policy_learning_curve_steps/policy_6314.pt new file mode 100644 index 0000000..2c5350b Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_6314.pt differ diff --git a/saved/policy_learning_curve_steps/policy_6528.pt b/saved/policy_learning_curve_steps/policy_6528.pt new file mode 100644 index 0000000..a59a69f Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_6528.pt differ diff --git a/saved/policy_learning_curve_steps/policy_7330.pt b/saved/policy_learning_curve_steps/policy_7330.pt new file mode 100644 index 0000000..f2be614 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_7330.pt differ diff --git a/saved/policy_learning_curve_steps/policy_7542.pt b/saved/policy_learning_curve_steps/policy_7542.pt new file mode 100644 index 0000000..f0c0dc2 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_7542.pt differ diff --git a/saved/policy_learning_curve_steps/policy_8422.pt b/saved/policy_learning_curve_steps/policy_8422.pt new file mode 100644 index 0000000..1e01e93 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_8422.pt differ diff --git a/saved/policy_learning_curve_steps/policy_8638.pt b/saved/policy_learning_curve_steps/policy_8638.pt new file mode 100644 index 0000000..84db028 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_8638.pt differ diff --git a/saved/policy_learning_curve_steps/policy_9447.pt b/saved/policy_learning_curve_steps/policy_9447.pt new file mode 100644 index 0000000..e11c528 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_9447.pt differ diff --git a/saved/policy_learning_curve_steps/policy_9777.pt b/saved/policy_learning_curve_steps/policy_9777.pt new file mode 100644 index 0000000..6bed819 Binary files /dev/null and b/saved/policy_learning_curve_steps/policy_9777.pt differ diff --git a/scripts/__pycache__/test_env.cpython-38.pyc b/scripts/__pycache__/test_env.cpython-38.pyc index 93a60f5..2806f11 100644 Binary files a/scripts/__pycache__/test_env.cpython-38.pyc and b/scripts/__pycache__/test_env.cpython-38.pyc differ diff --git a/utils/__pycache__/__init__.cpython-38.pyc b/utils/__pycache__/__init__.cpython-38.pyc index 258baa7..4a1aa36 100644 Binary files a/utils/__pycache__/__init__.cpython-38.pyc and b/utils/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/__pycache__/demos.cpython-38.pyc b/utils/__pycache__/demos.cpython-38.pyc index 233a333..7ec5964 100644 Binary files a/utils/__pycache__/demos.cpython-38.pyc and b/utils/__pycache__/demos.cpython-38.pyc differ diff --git a/utils/__pycache__/env_wrappers.cpython-38.pyc b/utils/__pycache__/env_wrappers.cpython-38.pyc index 9f5b4b5..4ab56ca 100644 Binary files a/utils/__pycache__/env_wrappers.cpython-38.pyc and b/utils/__pycache__/env_wrappers.cpython-38.pyc differ