Skip to content

Commit 95df230

Browse files
DocVaughanDocVaughan
committed
Updates to OpenAI gym environments. Improved reward functions. Added data processing scripts.
1 parent 746d127 commit 95df230

17 files changed

+392
-73
lines changed

‎OpenAI Gym/openAI_montior_processing.py

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#! /usr/bin/env python
2+
3+
###############################################################################
4+
# openAI_montior_processing.py
5+
#
6+
# Script to process the json data generated by the monitoring wrapper on
7+
# OpenAI gym environments.
8+
#
9+
# NOTE: Any plotting is set up for output, not viewing on screen.
10+
# So, it will likely be ugly on screen. The saved PDFs should look
11+
# better.
12+
#
13+
# Created: 07/12/17
14+
# - Joshua Vaughan
15+
# - [email protected]
16+
# - http://www.ucs.louisiana.edu/~jev9637
17+
#
18+
# Modified:
19+
# *
20+
#
21+
# TODO:
22+
# *
23+
###############################################################################
24+
25+
import numpy as np
26+
import matplotlib.pyplot as plt
27+
28+
import json # the data files generated are json
29+
30+
# TODO: 07/12/17 - JEV - Add GUI, argparse, or CLI for selecting file
31+
FILENAME = "example_data/duel_dqn_planar_crane-v0_monitor_2048_4_100000_2017-07-12_185754/openaigym.episode_batch.0.37814.stats.json"
32+
33+
34+
35+
with open(FILENAME) as data_file:
36+
data = json.load(data_file)
37+
38+
# This is the key data that we're interested in plotting. You can use the
39+
# method data.keys() to see others
40+
time = np.array(data['timestamps']) - data['timestamps'][0]
41+
rewards = np.array(data['episode_rewards'])
42+
episode_lengts = np.array(data['episode_lengths'])
43+
44+
45+
46+
47+
#----- Plot reward vs computation time -----
48+
# Set the plot size - 3x2 aspect ratio is best
49+
fig = plt.figure(figsize=(6,4))
50+
ax = plt.gca()
51+
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
52+
53+
# Change the axis units font
54+
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
55+
plt.setp(ax.get_xmajorticklabels(),fontsize=18)
56+
57+
ax.spines['right'].set_color('none')
58+
ax.spines['top'].set_color('none')
59+
60+
ax.xaxis.set_ticks_position('bottom')
61+
ax.yaxis.set_ticks_position('left')
62+
63+
# Turn on the plot grid and set appropriate linestyle and color
64+
ax.grid(True,linestyle=':', color='0.75')
65+
ax.set_axisbelow(True)
66+
67+
# Define the X and Y axis labels
68+
plt.xlabel('Computation Time (s)', fontsize=22, weight='bold', labelpad=5)
69+
plt.ylabel('Reward', fontsize=22, weight='bold', labelpad=10)
70+
71+
plt.plot(time, rewards, linewidth=2, linestyle='-', label=r'Data 1')
72+
73+
# uncomment below and set limits if needed
74+
# plt.xlim(0,5)
75+
# plt.ylim(0,10)
76+
77+
# Create the legend, then fix the fontsize
78+
# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
79+
# ltext = leg.get_texts()
80+
# plt.setp(ltext,fontsize=18)
81+
82+
# Adjust the page layout filling the page using the new tight_layout command
83+
plt.tight_layout(pad=0.5)
84+
85+
# save the figure as a high-res pdf in the current folder
86+
# plt.savefig('plot_filename.pdf')
87+
88+
89+
#----- Plot reward vs episode number -----
90+
# Set the plot size - 3x2 aspect ratio is best
91+
fig = plt.figure(figsize=(6,4))
92+
ax = plt.gca()
93+
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
94+
95+
# Change the axis units font
96+
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
97+
plt.setp(ax.get_xmajorticklabels(),fontsize=18)
98+
99+
ax.spines['right'].set_color('none')
100+
ax.spines['top'].set_color('none')
101+
102+
ax.xaxis.set_ticks_position('bottom')
103+
ax.yaxis.set_ticks_position('left')
104+
105+
# Turn on the plot grid and set appropriate linestyle and color
106+
ax.grid(True,linestyle=':', color='0.75')
107+
ax.set_axisbelow(True)
108+
109+
# Define the X and Y axis labels
110+
plt.xlabel('Episode', fontsize=22, weight='bold', labelpad=5)
111+
plt.ylabel('Reward', fontsize=22, weight='bold', labelpad=10)
112+
113+
plt.plot(rewards, linewidth=2, linestyle='-', label=r'Data 1')
114+
115+
# uncomment below and set limits if needed
116+
# plt.xlim(0,5)
117+
# plt.ylim(0,10)
118+
119+
# Create the legend, then fix the fontsize
120+
# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
121+
# ltext = leg.get_texts()
122+
# plt.setp(ltext,fontsize=18)
123+
124+
# Adjust the page layout filling the page using the new tight_layout command
125+
plt.tight_layout(pad=0.5)
126+
127+
# save the figure as a high-res pdf in the current folder
128+
# plt.savefig('plot_filename.pdf')
129+
130+
# show the figure
131+
plt.show()

‎OpenAI Gym/openAI_planarCrane_episodeDataProcessing.py

+169
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#! /usr/bin/env python
2+
3+
###############################################################################
4+
# openAI_planarCrane_episodeDataProcessing.py
5+
#
6+
# script to process the episode data saved in the CRAWLAB planar_crane
7+
# OpenAI gym environment
8+
#
9+
# NOTE: Any plotting is set up for output, not viewing on screen.
10+
# So, it will likely be ugly on screen. The saved PDFs should look
11+
# better.
12+
#
13+
# Created: 07/12/17
14+
# - Joshua Vaughan
15+
# - [email protected]
16+
# - http://www.ucs.louisiana.edu/~jev9637
17+
#
18+
# Modified:
19+
# *
20+
#
21+
# TODO:
22+
# *
23+
###############################################################################
24+
25+
import numpy as np
26+
import matplotlib.pyplot as plt
27+
28+
FILENAME = 'example_data/EpisodeData_2017-07-12_142149.csv'
29+
CABLE_LENGTH = 2.0
30+
31+
# Files have data saved as:
32+
# Time (s), Angle (rad), Angle (rad/s), Trolley Pos (m), Trolly Vel (m/s), Trolley Accel (m/s^2), Reward
33+
#
34+
# We'll unpack that data inline with opening the data file
35+
t, theta, theta_dot, x, x_dot, x_ddot, reward = np.loadtxt(FILENAME, delimiter=',', unpack=True)
36+
37+
38+
# ---- Plot the payload angle -------------------------------------------------
39+
# Set the plot size - 3x2 aspect ratio is best
40+
fig = plt.figure(figsize=(6,4))
41+
ax = plt.gca()
42+
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
43+
44+
# Change the axis units font
45+
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
46+
plt.setp(ax.get_xmajorticklabels(),fontsize=18)
47+
48+
ax.spines['right'].set_color('none')
49+
ax.spines['top'].set_color('none')
50+
51+
ax.xaxis.set_ticks_position('bottom')
52+
ax.yaxis.set_ticks_position('left')
53+
54+
# Turn on the plot grid and set appropriate linestyle and color
55+
ax.grid(True,linestyle=':', color='0.75')
56+
ax.set_axisbelow(True)
57+
58+
# Define the X and Y axis labels
59+
plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
60+
plt.ylabel('Angle (deg)', fontsize=22, weight='bold', labelpad=10)
61+
62+
plt.plot(t, theta*180/np.pi, linewidth=2, linestyle='-', label=r'Data 1')
63+
# plt.plot(t, y2, linewidth=2, linestyle='--', label=r'Data 2')
64+
65+
# uncomment below and set limits if needed
66+
# plt.xlim(0,5)
67+
# plt.ylim(0,10)
68+
69+
# Create the legend, then fix the fontsize
70+
# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
71+
# ltext = leg.get_texts()
72+
# plt.setp(ltext,fontsize=18)
73+
74+
# Adjust the page layout filling the page using the new tight_layout command
75+
plt.tight_layout(pad=0.5)
76+
77+
# save the figure as a high-res pdf in the current folder
78+
# plt.savefig('OpenAI_planarCrane_angle.pdf')
79+
80+
81+
# ----- Plot the position of the payload --------------------------------------
82+
# Set the plot size - 3x2 aspect ratio is best
83+
fig = plt.figure(figsize=(6,4))
84+
ax = plt.gca()
85+
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
86+
87+
# Change the axis units font
88+
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
89+
plt.setp(ax.get_xmajorticklabels(),fontsize=18)
90+
91+
ax.spines['right'].set_color('none')
92+
ax.spines['top'].set_color('none')
93+
94+
ax.xaxis.set_ticks_position('bottom')
95+
ax.yaxis.set_ticks_position('left')
96+
97+
# Turn on the plot grid and set appropriate linestyle and color
98+
ax.grid(True,linestyle=':', color='0.75')
99+
ax.set_axisbelow(True)
100+
101+
# Define the X and Y axis labels
102+
plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
103+
plt.ylabel('Position (m)', fontsize=22, weight='bold', labelpad=10)
104+
105+
plt.plot(t, x, linewidth=2, linestyle='--', label=r'Trolley')
106+
plt.plot(t, x - CABLE_LENGTH * np.sin(theta), linewidth=2, linestyle='-', label=r'Payload')
107+
108+
# uncomment below and set limits if needed
109+
# plt.xlim(0,5)
110+
# plt.ylim(0,10)
111+
112+
# Create the legend, then fix the fontsize
113+
leg = plt.legend(ncol = 1, fancybox=True) #, loc='upper right')
114+
ltext = leg.get_texts()
115+
plt.setp(ltext,fontsize=18)
116+
117+
# Adjust the page layout filling the page using the new tight_layout command
118+
plt.tight_layout(pad=0.5)
119+
120+
# save the figure as a high-res pdf in the current folder
121+
# plt.savefig('OpenAI_planarCrane_position.pdf')
122+
123+
124+
125+
126+
127+
# ----- Plot the acceleration input -------------------------------------------
128+
# Set the plot size - 3x2 aspect ratio is best
129+
fig = plt.figure(figsize=(6,4))
130+
ax = plt.gca()
131+
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)
132+
133+
# Change the axis units font
134+
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
135+
plt.setp(ax.get_xmajorticklabels(),fontsize=18)
136+
137+
ax.spines['right'].set_color('none')
138+
ax.spines['top'].set_color('none')
139+
140+
ax.xaxis.set_ticks_position('bottom')
141+
ax.yaxis.set_ticks_position('left')
142+
143+
# Turn on the plot grid and set appropriate linestyle and color
144+
ax.grid(True,linestyle=':', color='0.75')
145+
ax.set_axisbelow(True)
146+
147+
# Define the X and Y axis labels
148+
plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
149+
plt.ylabel('Accel. (m/s$^2$)', fontsize=22, weight='bold', labelpad=10)
150+
151+
plt.plot(t, x_ddot, linewidth=2, linestyle='-', label=r'Accel. Input')
152+
153+
# uncomment below and set limits if needed
154+
# plt.xlim(0,5)
155+
# plt.ylim(0,10)
156+
157+
# Create the legend, then fix the fontsize
158+
# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
159+
# ltext = leg.get_texts()
160+
# plt.setp(ltext,fontsize=18)
161+
162+
# Adjust the page layout filling the page using the new tight_layout command
163+
plt.tight_layout(pad=0.5)
164+
165+
# save the figure as a high-res pdf in the current folder
166+
# plt.savefig('OpenAI_planarCrane_Accelcommand.pdf')
167+
168+
# show the figure
169+
plt.show()

‎OpenAI Gym/openAI_planarCrane_test.py

+11-7
Original file line numberDiff line numberDiff line change
@@ -43,19 +43,23 @@
4343
ENV_NAME = 'planar_crane-v0'
4444

4545
LAYER_SIZE = 2048
46-
NUM_HIDDEN_LAYERS = 8
47-
NUM_STEPS = 50000
48-
DUEL_DQN = False
46+
NUM_HIDDEN_LAYERS = 4
47+
NUM_STEPS = 100000
48+
DUEL_DQN = True
49+
4950
TRIAL_ID = datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')
5051

5152
# TODO: Add file picker GUI - For now, look for files with the format below
5253
# FILENAME = 'weights/dqn_{}_weights_{}_{}_{}.h5f'.format(ENV_NAME, LAYER_SIZE, NUM_STEPS, TRIAL_ID)
5354
# FILENAME = 'weights/dqn_{}_weights_{}_{}.h5f'.format(ENV_NAME, LAYER_SIZE, NUM_STEPS)
54-
FILENAME = 'weights/dqn_planar_crane-v0_weights_2048_8_50000_2017-07-10_154335.h5f'
55+
FILENAME = 'weights/duel_dqn_planar_crane-v0_weights_2048_4_100000_2017-07-12_185754.h5f'
5556

5657
# Get the environment and extract the number of actions.
5758
env = gym.make(ENV_NAME)
5859

60+
# Record episode data?
61+
env.SAVE_DATA = True
62+
5963
# uncomment to record data about the training session, including video if visualize is true
6064
if DUEL_DQN:
6165
MONITOR_FILENAME = 'example_data/duel_dqn_{}_monitor_{}_{}_{}_{}'.format(ENV_NAME,
@@ -69,7 +73,7 @@
6973
NUM_HIDDEN_LAYERS,
7074
NUM_STEPS,
7175
TRIAL_ID)
72-
env = gym.wrappers.Monitor(env, MONITOR_FILENAME, force=True)
76+
# env = gym.wrappers.Monitor(env, MONITOR_FILENAME, force=True)
7377

7478

7579
# np.random.seed(123)
@@ -99,7 +103,7 @@
99103
memory = SequentialMemory(limit=NUM_STEPS, window_length=1)
100104
# train_policy = BoltzmannQPolicy(tau=0.05)
101105
test_policy = EpsGreedyQPolicy()
102-
train_policy = EpsGreedyQPolicy()
106+
train_policy = GreedyQPolicy()
103107

104108
if DUEL_DQN:
105109
dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=100,
@@ -120,4 +124,4 @@
120124
dqn.load_weights(FILENAME)
121125

122126
# Finally, evaluate our algorithm for 1 episode.
123-
dqn.test(env, nb_episodes=5, visualize=True, nb_max_episode_steps=500)
127+
dqn.test(env, nb_episodes=5, visualize=True, nb_max_episode_steps=501)

0 commit comments

Comments
 (0)