Added simple plotting and pickling

Author: seanstappas

actions.py

@@ -61,7 +61,7 @@
 
 
 def get_valid_action_numbers_from_state(s, state_repr='simple'):
-    if state_repr is 'simple' or state_repr is 'adjacent':
+    if state_repr is 'simple' or state_repr is 'adjacent' or state_repr is 'adjacent_conservative':
         actions = []
         top_left, top_right, bot_left, bot_right = s
         if top_left is not None:

agent.py

@@ -1,12 +1,9 @@
 import logging
 from abc import ABCMeta, abstractmethod
 
-from actions import action_number_to_name
 from learner import QLearner
 from world import QbertWorld
 
-import matplotlib.pyplot as plt
-
 
 class Agent:
     __metaclass__ = ABCMeta
@@ -18,8 +15,9 @@ def action(self):
 
 class QbertAgent(Agent):
     def __init__(self, agent_type='subsumption', random_seed=123, frame_skip=4, repeat_action_probability=0,
-                 sound=True, display_screen=False, state_repr='adjacent', alpha=0.1, gamma=0.95, epsilon=0.2,
-                 unexplored_threshold=1, unexplored_reward=100, exploration='combined', distance_metric=None):
+                 sound=True, display_screen=False, state_repr='adjacent_conservative', alpha=0.1, gamma=0.95,
+                 epsilon=0.2, unexplored_threshold=1, unexplored_reward=100, exploration='combined',
+                 distance_metric=None):
         if agent_type is 'block':
             self.agent = QbertBlockAgent(random_seed, frame_skip, repeat_action_probability, sound, display_screen,
                                          state_repr, alpha, gamma, epsilon, unexplored_threshold, unexplored_reward,
@@ -41,6 +39,15 @@ def __init__(self, agent_type='subsumption', random_seed=123, frame_skip=4, repe
     def action(self):
         return self.agent.action()
 
+    def q_size(self):
+        return self.agent.q_size()
+
+    def save(self, filename):
+        self.agent.save(filename)
+
+    def load(self, filename):
+        self.agent.load(filename)
+
 
 class QbertBlockAgent(Agent):
     def __init__(self, random_seed, frame_skip, repeat_action_probability, sound, display_screen, state_repr, alpha,
@@ -57,6 +64,15 @@ def action(self):
         self.block_learner.update(s, a, s_next, block_score)
         return block_score + friendly_score + enemy_score
 
+    def q_size(self):
+        return len(self.block_learner.Q)
+
+    def save(self, filename):
+        self.block_learner.save(filename)
+
+    def load(self, filename):
+        self.block_learner.load(filename)
+
 
 class QbertEnemyAgent(Agent):
     def __init__(self, random_seed, frame_skip, repeat_action_probability, sound, display_screen, state_repr, alpha,
@@ -73,6 +89,15 @@ def action(self):
         self.enemy_learner.update(s, a, s_next, enemy_score + enemy_penalty)
         return block_score + friendly_score + enemy_score
 
+    def q_size(self):
+        return len(self.enemy_learner.Q)
+
+    def save(self, filename):
+        self.enemy_learner.save(filename)
+
+    def load(self, filename):
+        self.enemy_learner.load(filename)
+
 
 class QbertFriendlyAgent(Agent):
     def __init__(self, random_seed, frame_skip, repeat_action_probability, sound, display_screen, state_repr, alpha,
@@ -89,6 +114,15 @@ def action(self):
         self.friendly_learner.update(s, a, s_next, friendly_score)
         return block_score + friendly_score + enemy_score
 
+    def q_size(self):
+        return len(self.friendly_learner.Q)
+
+    def save(self, filename):
+        self.friendly_learner.save(filename)
+
+    def load(self, filename):
+        self.friendly_learner.load(filename)
+
 
 class QbertSubsumptionAgent(Agent):
     def __init__(self, random_seed, frame_skip, repeat_action_probability, sound, display_screen, state_repr, alpha,
@@ -130,14 +164,6 @@ def action(self):
         else:
             logging.debug('Chose block action!')
             chosen_action = a
-        if chosen_action is None:
-            logging.info('None action!')
-            logging.info('Current row/col : {}/{}'.format(self.world.current_row, self.world.current_col))
-            logging.info('Prev block state: {}'.format(s))
-            logging.info('Prev enemy state: {}'.format(s_enemies))
-            logging.info('Prev friendly state: {}'.format(s_friendlies))
-            plt.imshow(self.world.rgb_screen)
-            plt.show()
         block_score, friendly_score, enemy_score, enemy_penalty = self.world.perform_action(chosen_action)
         if enemy_present:
             s_next_enemies = self.world.to_state_enemies()
@@ -154,4 +180,18 @@ def action(self):
         self.block_learner.update(s, a, s_next, block_score)
         return block_score + friendly_score + enemy_score
 
+    def q_size(self):
+        return len(self.block_learner.Q) + \
+               len(self.friendly_learner.Q) + \
+               len(self.enemy_learner.Q)
+
+    def save(self, filename):
+        self.block_learner.save('{}_{}'.format(filename, 'block'))
+        self.friendly_learner.save('{}_{}'.format(filename, 'friendly'))
+        self.enemy_learner.save('{}_{}'.format(filename, 'enemy'))
+
+    def load(self, filename):
+        self.block_learner.load('{}_{}'.format(filename, 'block'))
+        self.friendly_learner.load('{}_{}'.format(filename, 'friendly'))
+        self.enemy_learner.load('{}_{}'.format(filename, 'enemy'))
         # Human high scores: 15825, 27000

learner.py

@@ -3,6 +3,7 @@
 from abc import ABCMeta, abstractmethod
 
 from actions import action_number_to_name, get_valid_action_numbers_from_state
+from pickler import save_to_pickle, load_from_pickle
 
 
 class Learner:
@@ -19,16 +20,16 @@ def update(self, s, a, s_next, reward):
 
 class QLearner(Learner):
     def __init__(self, world, alpha, gamma, epsilon, unexplored_threshold, unexplored_reward, exploration,
-                 distance_metric, state_repr):
+                 distance_metric, state_repr, initial_q=None, initial_n=None):
         self.alpha = alpha
         self.gamma = gamma
         self.epsilon = epsilon
         self.unexplored_threshold = unexplored_threshold
         self.unexplored_reward = unexplored_reward
         self.exploration = exploration
         self.distance_metric = distance_metric
-        self.Q = {}
-        self.N = {}
+        self.Q = initial_q if initial_q is not None else {}
+        self.N = initial_n if initial_n is not None else {}
         self.world = world
         self.state_repr = state_repr
 
@@ -46,6 +47,8 @@ def update(self, s, a, s_next, reward):
         self.q_update(s, a, s_next, reward)
 
     def q_update(self, s, a, s_next, reward):
+        if self.exploration is 'combined':
+            self.N[s, a] = self.N.get((s, a), 0) + 1
         old_q = self.get_q(s, a)
         new_q = old_q + self.alpha * (reward + self.gamma * self.get_max_q(s_next) - old_q)
         self.Q[s, a] = new_q
@@ -112,3 +115,13 @@ def update_close(self, a, new_q):
         for s_close, a_close in zip(states_close, actions_close):
             self.Q[s_close, a_close] = new_q
 
+    def save(self, filename):
+        save_to_pickle(self.Q, '{}_{}'.format(filename, 'Q'))
+        save_to_pickle(self.N, '{}_{}'.format(filename, 'N'))
+
+    def load(self, filename):
+        self.Q = load_from_pickle('{}_{}'.format(filename, 'Q'))
+        self.N = load_from_pickle('{}_{}'.format(filename, 'N'))
+        logging.debug('Loaded Q: {}'.format(self.Q))
+        logging.debug('Loaded N: {}'.format(self.N))
+

main.py

@@ -5,6 +5,7 @@
 from argparse import ArgumentParser
 
 from agent import QbertAgent
+from plotter import plot_scores
 
 LOGGING_LEVELS = {
     'info': logging.INFO,
@@ -15,11 +16,14 @@
 }
 
 
-def play_learning_agent(num_episodes=1000, show_image=False):
-
+def play_learning_agent(num_episodes=1000, show_image=False, load_learning_filename='test_pickle',
+                        save_learning_filename='test_pickle', plot_filename='adjacent_conservative_sub_combined'):
     agent = QbertAgent()
     world = agent.world
     max_score = 0
+    scores = []
+    if load_learning_filename is not None:
+        agent.load(load_learning_filename)
     for episode in range(num_episodes):
         total_reward = 0
         world.reset()
@@ -28,12 +32,16 @@ def play_learning_agent(num_episodes=1000, show_image=False):
         if show_image:
             plt.imshow(world.rgb_screen)
             plt.show()
+        scores.append(total_reward)
         logging.info('Episode {} ended with score: {}'.format(episode + 1, total_reward))
         max_score = max(max_score, total_reward)
         world.ale.reset_game()
+    if plot_filename is not None:
+        plot_scores(scores, plot_filename)
+    if save_learning_filename is not None:
+        agent.save(save_learning_filename)
     logging.info('Maximum reward: {}'.format(max_score))
-    # TODO: plot results here
-
+    logging.info('Total Q size: {}'.format(agent.q_size()))
     # TODO: Exploration very key... getting very high scores early on because of unexplored weighting...
 
 

misc_test.py

@@ -1,5 +1,7 @@
 import numpy as np
 
+from pickler import save_to_pickle, load_from_pickle
+
 INITIAL_PARAMETERS1 = [
     [0],
     [0, 0],
@@ -50,5 +52,13 @@ def test_return_none(param):
         return None
 
 
+def test_pickle():
+    q = {(1, 2): 5, (5, 6): 10}
+    print(q)
+    save_to_pickle(q, 'test')
+    q2 = load_from_pickle('test')
+    print(q2)
+
+
 if __name__ == '__main__':
-    print(test_return_none(5))
+    test_pickle()

pickler.py

@@ -0,0 +1,12 @@
+import pickle
+
+
+def save_to_pickle(data, filename):
+    with open('pickle/{}.pkl'.format(filename), 'wb') as f:
+        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+def load_from_pickle(filename):
+    with open('pickle/{}.pkl'.format(filename), 'rb') as f:
+        data = pickle.load(f)
+    return data

plotter.py

@@ -1 +1,11 @@
 import matplotlib.pyplot as plt
+from matplotlib.ticker import MaxNLocator
+
+
+def plot_scores(scores, filename):
+    f = plt.figure()
+    ax = f.gca()
+    ax.xaxis.set_major_locator(MaxNLocator(integer=True))
+    plt.plot(scores, label='Score')
+    plt.legend()
+    f.savefig('report/plots/{}.pdf'.format(filename), bbox_inches='tight')

world.py

@@ -99,7 +99,8 @@ def __init__(self, random_seed, frame_skip, repeat_action_probability, sound, di
         ale = ALEInterface()
 
         # Get & Set the desired settings
-        ale.setInt('random_seed', random_seed)
+        if random_seed is not None:
+            ale.setInt('random_seed', random_seed)
         ale.setInt('frame_skip', frame_skip)
         ale.setFloat('repeat_action_probability', repeat_action_probability)
 
@@ -148,7 +149,7 @@ def __init__(self, random_seed, frame_skip, repeat_action_probability, sound, di
     def to_state_blocks(self):
         if self.state_repr is 'simple':
             return self.to_state_blocks_simple()
-        elif self.state_repr is 'adjacent':
+        elif self.state_repr is 'adjacent' or self.state_repr is 'adjacent_conservative':
             return self.to_state_blocks_adjacent()
         elif self.state_repr is 'verbose':
             return self.to_state_blocks_verbose()
@@ -158,13 +159,15 @@ def to_state_enemies(self):
             return self.to_state_enemies_simple()
         elif self.state_repr is 'adjacent':
             return self.to_state_enemies_adjacent()
+        elif self.state_repr is 'adjacent_conservative':
+            return self.to_state_enemies_adjacent_conservative()
         elif self.state_repr is 'verbose':
             return self.to_state_enemies_verbose()
 
     def to_state_friendlies(self):
         if self.state_repr is 'simple':
             return self.to_state_friendlies_simple()
-        elif self.state_repr is 'adjacent':
+        elif self.state_repr is 'adjacent' or self.state_repr is 'adjacent_conservative':
             return self.to_state_friendlies_simple()  # TODO: Make adjacent version of friendlies
         elif self.state_repr is 'verbose':
             return self.to_state_friendlies_verbose()
@@ -346,7 +349,7 @@ def to_state_enemies_adjacent(self):
                 bot_right = 0
         return top_left, top_right, bot_left, bot_right
 
-    def to_state_enemies_adjacent_old(self):
+    def to_state_enemies_adjacent_conservative(self):
         """
         Adjacent state representation for enemies around Qbert.
 
@@ -493,7 +496,7 @@ def update_rgb(self):
         if self.screen_not_flashing() \
                 and not np.array_equal(score_color, COLOR_BLACK) \
                 and not np.array_equal(score_color, self.desired_color):
-            logging.info('Identified {} as new desired color'.format(score_color))
+            logging.debug('Identified {} as new desired color'.format(score_color))
             self.desired_color = score_color
 
         self.enemy_present = False
@@ -553,8 +556,6 @@ def reset_position(self):
             reward += self.ale.act(NO_OP)
             self.ale.getRAM(self.ram)
         self.update_rgb()
-        if reward > 0:
-            logging.info('Nonzero reward of {} when resetting position.'.format(reward))
         return reward
 
     def reset(self):