diff --git a/README.md b/README.md index 3a8cecb..d6ebd3d 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,8 @@ Below is a table of the gym ids of the actions available in this environment (le | 8 | Pick Up | N/A | | N/A | View inventory | 'i' | | N/A | View shopping list | 'l' | +| N/A | Pause | 'p' | +| N/A | Reverse/Revert | 'z' | Actions are formatted as tuples. In the pick-up action an agent can specify an index that corresponds with a food item. @@ -204,6 +206,8 @@ While the best way to get a sense for how the game works is to try running it in * Toggle shopping cart: if the user is currently holding a shopping cart, this lets go of the cart. Otherwise, picks up the cart. Note that to pick up a cart, the user must be behind the shopping cart (adjacent to the handle). * Display inventory: displays a list of the food (a) that the player is holding, or (b) that is in the player’s cart, and the quantities of each food. Only available in keyboard mode; not necessary in gym/socket mode (since this information is already available in the observation). * Display shopping list: displays a list of the food on the player’s shopping list, and the quantities of each food. Only available in keyboard mode; not necessary in gym/socket mode (since this information is already available in the observation). +* Pause: Pauses the game play. Generally only useful in history playback +* Reverse/revert: Actions are reversed to be played back again if the game play is playing back history sent from replay_agent_actions.py. The same commands will be played back again after the reverse. Otherwise, this reverts the player actions and allows different commands to be sent. If the revert command is called when the user is in an action recording game state, the action history will also be altered so that the reverted actions do not show up in history. ### General facts diff --git a/env.py b/env.py index da30484..cbff8fa 100755 --- a/env.py +++ b/env.py @@ -78,7 +78,8 @@ def reset(self,seed = None, options = None, obs=None): sprite_paths=self.player_sprites, record_path=self.record_path, stay_alive=self.stay_alive, - record_actions=self.record_actions) + record_actions=self.record_actions + ) self.unwrapped.game.set_up() if obs is not None: self.unwrapped.game.set_observation(obs) diff --git a/game.py b/game.py index 3f34e04..45eed72 100755 --- a/game.py +++ b/game.py @@ -1,3 +1,4 @@ +import copy from random import uniform, choice import pygame @@ -113,6 +114,9 @@ def __init__(self, num_players=1, player_speed=0.07, keyboard_input=False, rende self.record_actions = record_actions self.action_history = [] + self.is_playback = False + self.game_state_observations = dict() # Used to reverse back to a certain state in the game + if not headless: if follow_player == -1: config.SCALE = 32 @@ -331,10 +335,14 @@ def randomize_position(self, player): y = uniform(0, 25) player.position = [x, y] - def save_state(self, filename): - with open(filename, "w") as f: - f.write(str(self.observation(True))) - # f.write(str(self.observation(False))) + def save_state(self, filename=None, obs=None): + if obs: + self.game_state_observations[obs[0]] = copy.deepcopy(obs[1]) + + elif filename: + with open(filename, "w") as f: + f.write(str(self.observation(True))) + # f.write(str(self.observation(False))) def current_player(self): if self.curr_player == -1: @@ -884,6 +892,6 @@ def check_register_zones(self, register): def write_action_history(self, filename): with open(filename, "w") as f: history = "" - for action in self.action_history: - history += action + '\n' + for row in self.action_history: + history += row[0] + " " + row[1] + " " + row[3] + "\n" f.write(history) diff --git a/replay_agent_actions.py b/replay_agent_actions.py index 51c3af2..46d569b 100644 --- a/replay_agent_actions.py +++ b/replay_agent_actions.py @@ -28,6 +28,14 @@ sock_game = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock_game.connect((HOST, PORT)) + print("Sending playback signal") + sock_game.send(str.encode("Playback")) # send action to env + + output = recv_socket_data(sock_game) # get observation from env + output = json.loads(output) + + + actions = [] with open(args.filename, "r") as file: for line in file: @@ -38,7 +46,8 @@ print("Sending action: ", action) sock_game.send(str.encode(action)) # send action to env - output = recv_socket_data(sock_game) # get observation from env - output = json.loads(output) + output = recv_socket_data(sock_game) # get observation from env (wait for response before sending next command) + # output = json.loads(output) # Put back if you want to inspect received message - print("JSON: ", output) + # print("JSON: ", output) + exit() diff --git a/socket_env.py b/socket_env.py index 0f835aa..5349dde 100755 --- a/socket_env.py +++ b/socket_env.py @@ -2,6 +2,7 @@ import argparse import datetime import json +from pprint import pprint import selectors import socket import types @@ -33,17 +34,104 @@ def __init__(self, env, keyboard_input=False): self.running = True def single_player_action(self, action, arg=0): - if self.env.unwrapped.game.record_actions: - self.env.unwrapped.game.action_history.append( - str(self.curr_player) + " " + str(action).split(".")[1] + " " + str(datetime.datetime.now().timestamp())) return self.curr_player, action, arg + def record_action_and_obs(self, action, obs, arg=0): + # Record action history + timestamp = datetime.datetime.now().timestamp() + self.env.unwrapped.game.action_history.append( + (str(self.curr_player), + str(action).split(".")[-1], + str(arg), str(timestamp))) + # print("appending action: ", (str(self.curr_player), + # str(action).split(".")[-1], + # str(arg), str(timestamp))) + + # Record observation + self.env.unwrapped.game.save_state(obs=(str(timestamp), obs)) + + return str(timestamp) + def handle_events(self): if self.env.unwrapped.game.players[self.curr_player].interacting: self.handle_interactive_events() else: self.handle_exploratory_events() self.env.render(mode='violations') + + def reverse(self, replay_index=None): + + history_len = len(self.env.unwrapped.game.action_history) + + num_steps = int(input("How many commands would you like to reverse/revert? ")) + if num_steps < 0: + print("Error: Cannot reverse by negative number of steps!") + exit(1) + + if replay_index == None: + replay_index = 0 if history_len < num_steps else history_len - num_steps + else: + replay_index = 0 if (replay_index - num_steps) < 0 else replay_index - num_steps + + print("replay_index: ", replay_index) + timestamp = self.env.unwrapped.game.action_history[replay_index][-1] + obs = self.env.unwrapped.game.game_state_observations[timestamp] + + self.env.unwrapped.game.set_observation(obs) + env.render() + self.env.unwrapped.game.update() + # obs = self.env.unwrapped.game.observation(True) + + step_count = self.env.unwrapped.step_count # to preserve original step count + + if self.env.unwrapped.game.is_playback: + for action_tuple in self.env.unwrapped.game.action_history[replay_index + 1:]: + for event in pygame.event.get(): + if event.type == pygame.KEYDOWN and event.key == pygame.K_z: + self.reverse(replay_index=replay_index) + return + elif event.type == pygame.KEYDOWN and event.key == pygame.K_p: + if (self.pause_game(replay_index)): + return + print("action_tuple[:-1]: ", action_tuple[:-1]) + action_list = list(action_tuple[:-1]) + action_list[0] = int(action_list[0]) + action_list[1] = ACTION_COMMANDS.index(action_list[1]) + self.env.step(action_list) + + env.render() + pygame.time.delay(20) + replay_index += 1 + else: + self.env.unwrapped.game.action_history = self.env.unwrapped.game.action_history[:replay_index + 1] + obs_list = self.env.unwrapped.game.game_state_observations.items() + self.env.unwrapped.game.game_state_observations = dict(filter(lambda item: (item[0] <= timestamp), obs_list)) + + self.env.unwrapped.step_count = step_count + + ''' + Purpose: Pauses game until key p is hit again + Note: Could be given a starting point of which index in history to + start reverse from if user requests for a reverse or revert in + this function + ''' + def pause_game(self, reverse_index=None): + waiting = True + while waiting: + for event in pygame.event.get(): + if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE): + self.env.unwrapped.game.running = False + elif event.type == pygame.KEYDOWN and event.key == pygame.K_p: + waiting = False + elif event.type == pygame.KEYDOWN and event.key == pygame.K_z: + # need to start reverse from potentially middle of the game history + if reverse_index: + self.reverse(reverse_index) + return True + else: + self.reverse() + return False + return False def handle_exploratory_events(self): player = self.env.unwrapped.game.players[self.curr_player] @@ -52,14 +140,20 @@ def handle_exploratory_events(self): self.env.unwrapped.game.running = False elif event.type == pygame.KEYDOWN and event.key == pygame.K_s: filename = input("Please enter a filename for saving the state.\n>>> ") - self.env.unwrapped.game.save_state(filename) + self.env.unwrapped.game.save_state(filename=filename) print("State saved to {filename}.".format(filename=filename)) + elif event.type == pygame.KEYDOWN and event.key == pygame.K_p: + self.pause_game() + elif event.type == pygame.KEYDOWN and event.key == pygame.K_z: + print("z pressed without pause") + self.reverse() elif event.type == pygame.KEYDOWN and event.key == pygame.K_r: self.env.unwrapped.game.toggle_record() elif self.keyboard_input: if event.type == pygame.KEYDOWN: if event.key == pygame.K_RETURN: - self.env.step(self.single_player_action(PlayerAction.INTERACT)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.INTERACT)) + self.record_action_and_obs(PlayerAction.INTERACT, obs) # i key shows inventory elif event.key == pygame.K_i: player.render_shopping_list = False @@ -72,8 +166,8 @@ def handle_exploratory_events(self): player.interacting = True elif event.key == pygame.K_c: - self.env.step(self.single_player_action(PlayerAction.TOGGLE)) - + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.TOGGLE)) + self.record_action_and_obs(PlayerAction.TOGGLE, obs) # switch players (up to 9 players) else: for i in range(1, len(self.env.unwrapped.game.players) + 1): @@ -86,20 +180,26 @@ def handle_exploratory_events(self): # player stands still if not moving elif event.type == pygame.KEYUP: - self.env.step(self.single_player_action(PlayerAction.NOP)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.NOP)) + self.record_action_and_obs(PlayerAction.NOP, obs) if self.keyboard_input: keys = pygame.key.get_pressed() if keys[pygame.K_UP]: # up - self.env.step(self.single_player_action(PlayerAction.NORTH)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.NORTH)) + self.record_action_and_obs(PlayerAction.NORTH, obs) + elif keys[pygame.K_DOWN]: # down - self.env.step(self.single_player_action(PlayerAction.SOUTH)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.SOUTH)) + self.record_action_and_obs(PlayerAction.SOUTH, obs) elif keys[pygame.K_LEFT]: # left - self.env.step(self.single_player_action(PlayerAction.WEST)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.WEST)) + self.record_action_and_obs(PlayerAction.WEST, obs) elif keys[pygame.K_RIGHT]: # right - self.env.step(self.single_player_action(PlayerAction.EAST)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.EAST)) + self.record_action_and_obs(PlayerAction.EAST, obs) self.running = self.env.unwrapped.game.running @@ -112,11 +212,13 @@ def handle_interactive_events(self): if event.type == pygame.KEYDOWN and self.keyboard_input: # b key cancels interaction if event.key == pygame.K_b: - self.env.step(self.single_player_action(PlayerAction.CANCEL)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.CANCEL)) + self.record_action_and_obs(PlayerAction.CANCEL, obs) # return key continues interaction elif event.key == pygame.K_RETURN: - self.env.step(self.single_player_action(PlayerAction.INTERACT)) + obs, _, _, _, _ = self.env.step(self.single_player_action(PlayerAction.INTERACT)) + self.record_action_and_obs(PlayerAction.INTERACT, obs) # i key turns off inventory rendering elif event.key == pygame.K_i: if player.render_inventory: @@ -163,6 +265,8 @@ def get_action_json(action, env_, obs, reward, done, info_=None, violations=''): def is_single_player(command_): return ',' not in command_ +def is_playback_mode(command_): + return command == "Playback" def get_player_and_command(command_): split_command = command_.split(' ') @@ -287,7 +391,7 @@ def accept_wrapper(sock): record_path=args.record_path, stay_alive=args.stay_alive, record_actions=args.record_actions - ) + ) norms = [CartTheftNorm(), BasketTheftNorm(), @@ -345,7 +449,7 @@ def accept_wrapper(sock): curr_action = [(0,0)] * env.unwrapped.num_players e = [] if not args.headless: - handler.handle_events() + handler.handle_events() # TIA TODO: INJECTION OF KEYBOARD COMMANDS env.render() for key, mask in events: if key.data is None: @@ -365,10 +469,11 @@ def accept_wrapper(sock): obs = command[4:] from json import loads obs_to_return = env.reset(obs=loads(obs)) - print(obs_to_return) json_to_send = get_action_json("SET", env, obs_to_return, 0., False, None) data = key.data data.outb = str.encode(json.dumps(json_to_send,default=lambda o: o.__dict__) + "\n") + if is_playback_mode(command): + env.unwrapped.game.is_playback = True if is_single_player(command): player, command, arg = get_player_and_command(command) e.append((key, mask, command)) @@ -390,12 +495,15 @@ def accept_wrapper(sock): sent = sock.send(data.outb) # Should be ready to write data.outb = data.outb[sent:] if should_perform_action: - for index, player in enumerate(curr_action): - env.unwrapped.game.action_history.append( - str(index) + " " + ACTION_COMMANDS[player[0]] + " " + - str(datetime.datetime.now().timestamp()) - ) + handler.curr_player = player + handler.env.curr_player = player + handler.env.unwrapped.game.curr_player = player + obs, reward, done, info, violations = env.step(tuple(curr_action)) + + for index, player_action in enumerate(curr_action): + handler.record_action_and_obs(ACTION_COMMANDS[player_action[0]], obs) + for key, mask, command in e: json_to_send = get_action_json(command, env, obs, reward, done, info, violations) @@ -410,3 +518,5 @@ def accept_wrapper(sock): if env.unwrapped.game.record_actions: filename = input("Please enter a filename for saving the action history.\n>>> ") env.unwrapped.game.write_action_history(filename) + +