Added OpenAI gym environment for a planar crane with continuous inputs.

DocVaughan · DocVaughan · commit d166c4faa09e · 2017-07-13T15:32:34.000-05:00
diff --git a/OpenAI Gym/openAI_planarCraneContinuous_episodeDataProcessing.py b/OpenAI Gym/openAI_planarCraneContinuous_episodeDataProcessing.py
@@ -0,0 +1,169 @@
+#! /usr/bin/env python
+
+###############################################################################
+# openAI_planarCrane_episodeDataProcessing.py
+#
+# script to process the episode data saved in the CRAWLAB planar_crane
+# OpenAI gym environment
+#
+# NOTE: Any plotting is set up for output, not viewing on screen.
+#       So, it will likely be ugly on screen. The saved PDFs should look
+#       better.
+#
+# Created: 07/13/17
+#   - Joshua Vaughan
+#   - joshua.vaughan@louisiana.edu
+#   - http://www.ucs.louisiana.edu/~jev9637
+#
+# Modified:
+#   * 
+#
+# TODO:
+#   * 
+###############################################################################
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+FILENAME = 'example_data/EpisodeData_2017-07-13_132633.csv'
+CABLE_LENGTH = 2.0
+
+# Files have data saved as:
+# Time (s), Angle (rad), Angle (rad/s), Trolley Pos (m), Trolly Vel (m/s), Trolley Accel (m/s^2), Reward
+# 
+# We'll unpack that data inline with opening the data file
+t, theta, theta_dot, x, x_dot, x_ddot, reward = np.loadtxt(FILENAME, delimiter=',', unpack=True)
+
+
+# ---- Plot the payload angle -------------------------------------------------
+# Set the plot size - 3x2 aspect ratio is best
+fig = plt.figure(figsize=(6,4))
+ax = plt.gca()
+plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
+
+# Change the axis units font
+plt.setp(ax.get_ymajorticklabels(),fontsize=18)
+plt.setp(ax.get_xmajorticklabels(),fontsize=18)
+
+ax.spines['right'].set_color('none')
+ax.spines['top'].set_color('none')
+
+ax.xaxis.set_ticks_position('bottom')
+ax.yaxis.set_ticks_position('left')
+
+# Turn on the plot grid and set appropriate linestyle and color
+ax.grid(True,linestyle=':', color='0.75')
+ax.set_axisbelow(True)
+
+# Define the X and Y axis labels
+plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
+plt.ylabel('Angle (deg)', fontsize=22, weight='bold', labelpad=10)
+ 
+plt.plot(t, theta*180/np.pi, linewidth=2, linestyle='-', label=r'Data 1')
+# plt.plot(t, y2, linewidth=2, linestyle='--', label=r'Data 2')
+
+# uncomment below and set limits if needed
+# plt.xlim(0,5)
+# plt.ylim(0,10)
+
+# Create the legend, then fix the fontsize
+# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
+# ltext  = leg.get_texts()
+# plt.setp(ltext,fontsize=18)
+
+# Adjust the page layout filling the page using the new tight_layout command
+plt.tight_layout(pad=0.5)
+
+# save the figure as a high-res pdf in the current folder
+# plt.savefig('OpenAI_planarCrane_angle.pdf')
+
+
+# ----- Plot the position of the payload --------------------------------------
+# Set the plot size - 3x2 aspect ratio is best
+fig = plt.figure(figsize=(6,4))
+ax = plt.gca()
+plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
+
+# Change the axis units font
+plt.setp(ax.get_ymajorticklabels(),fontsize=18)
+plt.setp(ax.get_xmajorticklabels(),fontsize=18)
+
+ax.spines['right'].set_color('none')
+ax.spines['top'].set_color('none')
+
+ax.xaxis.set_ticks_position('bottom')
+ax.yaxis.set_ticks_position('left')
+
+# Turn on the plot grid and set appropriate linestyle and color
+ax.grid(True,linestyle=':', color='0.75')
+ax.set_axisbelow(True)
+
+# Define the X and Y axis labels
+plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
+plt.ylabel('Position (m)', fontsize=22, weight='bold', labelpad=10)
+
+plt.plot(t, x, linewidth=2, linestyle='--', label=r'Trolley') 
+plt.plot(t, x - CABLE_LENGTH * np.sin(theta), linewidth=2, linestyle='-', label=r'Payload')
+
+# uncomment below and set limits if needed
+# plt.xlim(0,5)
+# plt.ylim(0,10)
+
+# Create the legend, then fix the fontsize
+leg = plt.legend(ncol = 1, fancybox=True) #, loc='upper right')
+ltext  = leg.get_texts()
+plt.setp(ltext,fontsize=18)
+
+# Adjust the page layout filling the page using the new tight_layout command
+plt.tight_layout(pad=0.5)
+
+# save the figure as a high-res pdf in the current folder
+# plt.savefig('OpenAI_planarCrane_position.pdf')
+
+
+
+
+
+# ----- Plot the acceleration input -------------------------------------------
+# Set the plot size - 3x2 aspect ratio is best
+fig = plt.figure(figsize=(6,4))
+ax = plt.gca()
+plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
+
+# Change the axis units font
+plt.setp(ax.get_ymajorticklabels(),fontsize=18)
+plt.setp(ax.get_xmajorticklabels(),fontsize=18)
+
+ax.spines['right'].set_color('none')
+ax.spines['top'].set_color('none')
+
+ax.xaxis.set_ticks_position('bottom')
+ax.yaxis.set_ticks_position('left')
+
+# Turn on the plot grid and set appropriate linestyle and color
+ax.grid(True,linestyle=':', color='0.75')
+ax.set_axisbelow(True)
+
+# Define the X and Y axis labels
+plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
+plt.ylabel('Accel. (m/s$^2$)', fontsize=22, weight='bold', labelpad=10)
+ 
+plt.plot(t, x_ddot, linewidth=2, linestyle='-', label=r'Accel. Input')
+
+# uncomment below and set limits if needed
+# plt.xlim(0,5)
+# plt.ylim(0,10)
+
+# Create the legend, then fix the fontsize
+# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
+# ltext  = leg.get_texts()
+# plt.setp(ltext,fontsize=18)
+
+# Adjust the page layout filling the page using the new tight_layout command
+plt.tight_layout(pad=0.5)
+
+# save the figure as a high-res pdf in the current folder
+# plt.savefig('OpenAI_planarCrane_Accelcommand.pdf')
+
+# show the figure
+plt.show()
diff --git a/OpenAI Gym/openAI_planarCraneContinuous_randomAction.py b/OpenAI Gym/openAI_planarCraneContinuous_randomAction.py
@@ -0,0 +1,58 @@
+#! /usr/bin/env python
+
+###############################################################################
+# openAI_variableLengthPendulum.py
+#
+# script to run the variable lenght pendulum OpenAI environment
+#
+# NOTE: Any plotting is set up for output, not viewing on screen.
+#       So, it will likely be ugly on screen. The saved PDFs should look
+#       better.
+#
+# Created: 07/07/17
+#   - Joshua Vaughan
+#   - joshua.vaughan@louisiana.edu
+#   - http://www.ucs.louisiana.edu/~jev9637
+#
+# Modified:
+#   * 
+#
+# TODO:
+#   * 
+###############################################################################
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+import gym
+import time
+import planar_crane_continuous
+
+env = gym.make('planar_crane_continuous-v0')
+
+# run 5 episodes of 1000 timesteps, taking random actions at each step
+for i_episode in range(5):
+    observation = env.reset()
+    for t in range(1000):
+        env.render()
+        
+        # just randomly choose an action
+        action = env.action_space.sample() 
+        observation, reward, done, info = env.step(action)
+        
+        # Finally, print the updated state of the system
+        print("\033[2J\033[;H") # Clear the terminal each time
+        j = 40  # padding
+        d = '.' # what to fill with
+        print("Theta (deg).:".ljust(j,d), '{:+8.3f}'.format(observation[0]*180/np.pi))
+        print("Theta_dot (deg/s):".ljust(j,d), '{:+8.3f}'.format(observation[1]*180/np.pi))
+        print("x (m):".ljust(j,d), '{:+8.3f}'.format(observation[2]))
+        print("x_dot (m/s):".ljust(j,d), '{:+8.3f}'.format(observation[3]))
+        print("x_ddot (m/s^2) Input:".ljust(j,d), '{:+8.3f}'.format(action[0]))
+        print("Reward:".ljust(j,d), '{:+8.3f}'.format(reward))
+
+        # if episode finishes before full time range, notify
+        if done:
+            print("\r\nEpisode finished after {} timesteps".format(t+1))
+            time.sleep(1)
+            break
diff --git a/OpenAI Gym/openAI_planarCraneContinuous_test.py b/OpenAI Gym/openAI_planarCraneContinuous_test.py
@@ -0,0 +1,113 @@
+#! /usr/bin/env python
+
+###############################################################################
+# openAI_planarCrane_test.py
+#
+# File to test on the CRAWLAB custom OpenAI planar crane environment 
+#
+# Requires:
+#  * CRAWLAB planar_crane Open_AI environment folder to be in the same as this file
+#  * keras, openAI gym, keras-rl packages (all are pip or conda installable)
+#
+# NOTE: Any plotting is set up for output, not viewing on screen.
+#       So, it will likely be ugly on screen. The saved PDFs should look
+#       better.
+#
+# Created: 07/09/17
+#   - Joshua Vaughan
+#   - joshua.vaughan@louisiana.edu
+#   - http://www.ucs.louisiana.edu/~jev9637
+#
+# Modified:
+#   * 
+#
+# TODO:
+#   * 
+###############################################################################
+
+import numpy as np
+import datetime     # used to generate unique filenames
+
+import gym
+import planar_crane_continuous
+
+from keras.models import Sequential, Model
+from keras.layers import Dense, Activation, Flatten, Input, merge
+from keras.optimizers import Adam
+
+from rl.agents import DDPGAgent
+from rl.memory import SequentialMemory
+from rl.random import OrnsteinUhlenbeckProcess
+
+
+ENV_NAME = 'planar_crane_continuous-v0'
+
+LAYER_SIZE = 2048
+NUM_HIDDEN_LAYERS = 3
+NUM_STEPS = 50000
+TRIAL_ID = datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')
+
+# TODO: Add file picker GUI - For now, look for files with the format below
+# Remove the _actor or _critic from the filename. The load method automatically
+# appends these.
+FILENAME = 'weights/ddpg_planar_crane_continuous-v0_weights_2048_3_50000_2017-07-13_144811.h5f'
+
+# Get the environment and extract the number of actions.
+env = gym.make(ENV_NAME)
+nb_actions = env.action_space.shape[0]
+
+# Record episode data?
+env.SAVE_DATA = True
+
+
+
+# Next, we build a very simple actor model.
+actor = Sequential()
+
+# Input Layer
+actor.add(Flatten(input_shape=(1,) + env.observation_space.shape))
+
+# Hidden layers
+for _ in range(NUM_HIDDEN_LAYERS):
+    actor.add(Dense(LAYER_SIZE))
+    actor.add(Activation('relu'))
+
+# Output layer
+actor.add(Dense(nb_actions))
+actor.add(Activation('linear'))
+print(actor.summary())
+
+
+# critic model
+action_input = Input(shape=(nb_actions,), name='action_input')
+observation_input = Input(shape=(1,) + env.observation_space.shape, name='observation_input')
+flattened_observation = Flatten()(observation_input)
+x = merge([action_input, flattened_observation], mode='concat')
+
+# Hidden layers
+for _ in range(NUM_HIDDEN_LAYERS):
+    x = (Dense(LAYER_SIZE))(x)
+    x = Activation('relu')(x)
+
+# Output Layer
+x = Dense(1)(x)
+x = Activation('linear')(x)
+critic = Model(input=[action_input, observation_input], output=x)
+print(critic.summary())
+
+# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
+# even the metrics!
+memory = SequentialMemory(limit=2*NUM_STEPS, window_length=1)
+random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=.15, mu=0., sigma=.3)
+agent = DDPGAgent(nb_actions=nb_actions, actor=actor, critic=critic, critic_action_input=action_input,
+                  memory=memory, nb_steps_warmup_critic=100, nb_steps_warmup_actor=100,
+                  random_process=random_process, gamma=.99, target_model_update=1e-3)
+agent.compile(Adam(lr=.001, clipnorm=1.), metrics=['mae'])
+
+# Load the model weights - this method will automatically load the weights for
+# both the actor and critic
+agent.load_weights(FILENAME)
+
+
+# Finally, evaluate our algorithm for 5 episodes.
+agent.test(env, nb_episodes=5, nb_max_episode_steps=500, visualize=True)
diff --git a/OpenAI Gym/openAI_planarCraneContinuous_train.py b/OpenAI Gym/openAI_planarCraneContinuous_train.py
