reach_circle_explore_exploit.py

#!/usr/bin/python
from tkinter import *
from tkinter.ttk import * 
import random
import time
import numpy as np

# Set global values
alpha = 0.9
gamma = 1.0
freq = 10 # Set the time gap of canvas update
initial_pos_x1 = 50
initial_pos_y1 = 80
mov_list = [-20,-10,10,20]


fout = open("logfile.txt","w")

#Set up Q table
state_space = {} # Set of all possible states
temp_list = []
for n in range(0,500,10):
	for m in range(0,360,10):
		temp_list.append((n,m))
for k in range(1800):
	state_space[k] = temp_list[k]

action_space = {} # Set of all possible actions
temp_list = []
for n in mov_list:
	for m in mov_list:
		temp_list.append((m,n))
for k in range(16):
	action_space[k] = temp_list[k]

q_table = np.zeros([len(state_space),len(action_space)])
#q_table = np.loadtxt("qtable.csv",delimiter=',')

#Set up Environment and Simulation
class GFG: 
	
	def __init__(self, master = None): 
		# Set up local variables
		self.steps = 0
		self.checkpoint = 0
		self.pos_x1 = initial_pos_x1
		self.pos_y1 = initial_pos_y1
		self.x1 = 10
		self.y1 = 10

		self.game = 0
		self.reward = 0

		self.reset = 0
  
		# canvas object to create shape 
		self.canvas = Canvas(master,width=500, height=360, bg="black") 
		# creating circle 
		self.circle = self.canvas.create_oval( 42, 72, 60, 90, fill = "green") 
		self.circle4 = self.canvas.create_oval( 230, 160, 270, 200, outline = "yellow")
		self.canvas.pack() 
		# Set purely exploration mode
		self.epsilon = 1.0
		# Switch between explore and exploit
		self.exploit_flag = 0

		self.movement() 


	def movement(self): # Movement of Green Ball

		if self.reset == 1:
			new_x1 = (initial_pos_x1 - self.pos_x1)
			new_y1 = (initial_pos_y1 - self.pos_y1)
			self.canvas.move(self.circle,new_x1,new_y1)
			self.pos_x1 += new_x1
			self.pos_y1 += new_y1
			self.reset = 0
			time.sleep(1)

		action = 0
		self.reward = 0
		state = list(state_space.keys())[list(state_space.values()).index((self.pos_x1, self.pos_y1))]
		self.steps += 1
		if self.game % 50 == 0 and self.exploit_flag == 1:	
			remainder = self.game / 50	
			self.epsilon = 1 - 0.2*(remainder-1) # Slowly exlore less and exploit more
				
		# If max steps is reached
		if self.steps % 200 == 0:
			self.checkpoint = self.steps
			self.game += 1
			print("Game : "+str(self.game) + " lost")
			fout.write("game "+str(self.game) + " lost\n")
			fout.flush()
			new_x1 = (initial_pos_x1 - self.pos_x1)
			new_y1 = (initial_pos_y1 - self.pos_y1)
			self.canvas.move(self.circle, new_x1, new_y1)
			self.pos_x1 += new_x1
			self.pos_y1 += new_y1		

		if self.game < 300: # Purely explore till these many games
			self.x1 = random.choice(mov_list)
			self.y1 = random.choice(mov_list)
			action = list(action_space.keys())[list(action_space.values()).index((self.x1, self.y1))]
		else:
			# Start progressively exploiting more
			self.exploit_flag = 1
			if random.random() < self.epsilon:
				self.x1 = random.choice(mov_list)
				self.y1 = random.choice(mov_list)
				action = list(action_space.keys())[list(action_space.values()).index((self.x1, self.y1))]
			else:
				action = np.argmax(q_table[state])
				self.x1, self.y1 = action_space[action]

		# Bounce back from boundaries
		if self.pos_x1 > 450:
			self.x1 = -10
		if self.pos_x1 < 25:
			self.x1 = 10
		if self.pos_y1 > 300:
			self.y1 = -10
		if self.pos_y1 < 25:
			self.y1 = 10  

		# Update position
		self.canvas.move(self.circle, self.x1, self.y1)	
		self.pos_x1 += self.x1
		self.pos_y1 += self.y1

		# If target obtained
		if self.pos_x1 == 250 and self.pos_y1 == 180:
			self.reward = 100
			self.game += 1
			print("Game : "+str(self.game) + " won")
			fout.write("game "+str(self.game) + " won\n")
			fout.flush()
			self.steps = self.checkpoint # Reset steps to start of the run for 200
			self.reset = 1

		# Update the Q table
		old_q_value = q_table[state, action]	
		next_state = list(state_space.keys())[list(state_space.values()).index((self.pos_x1, self.pos_y1))]
		next_max = np.max(q_table[next_state])	
		q_target = self.reward + gamma * next_max
		q_delta = q_target - old_q_value
		q_table[state, action] = old_q_value + alpha * q_delta


		self.canvas.after(freq, self.movement)
	  
  
if __name__ == "__main__": 
  
	# object of class Tk, resposible for creating 
	# a tkinter toplevel window 
	master = Tk() 
	gfg = GFG(master)
	mainloop()