Refactoring

Author: Gripnook

src/main.py

@@ -82,12 +82,12 @@ def init_config_frame(self):
         self.learning_rate_entry = Entry(self.config_frame, textvar=self.learning_rate_var)
         self.learning_rate_entry.grid(row=8, column=1, sticky=W)
 
-        self.learning_decay_label = Label(self.config_frame, text='Layer Decay:')
-        self.learning_decay_label.grid(row=9, column=0, sticky=W)
-        self.learning_decay_var = StringVar()
-        self.learning_decay_var.set('1.0')
-        self.learning_decay_entry = Entry(self.config_frame, textvar=self.learning_decay_var)
-        self.learning_decay_entry.grid(row=9, column=1, sticky=W)
+        self.layer_decay_label = Label(self.config_frame, text='Layer Decay:')
+        self.layer_decay_label.grid(row=9, column=0, sticky=W)
+        self.layer_decay_var = StringVar()
+        self.layer_decay_var.set('1.0')
+        self.layer_decay_entry = Entry(self.config_frame, textvar=self.layer_decay_var)
+        self.layer_decay_entry.grid(row=9, column=1, sticky=W)
 
         self.momentum_label = Label(self.config_frame, text='Momentum:')
         self.momentum_label.grid(row=10, column=0, sticky=W)
@@ -139,7 +139,7 @@ def init_validation_frame(self, row):
         self.validation_test_loss_var.grid(row=3, column=1, sticky=W)
 
     def init_test_frame(self):
-        self.canvas_data = np.array(self.test_images[0], dtype='float64')
+        self.canvas_data = np.array(self.test_images[0], dtype='float32')
 
         self.test_frame = Frame(self.master)
         self.test_frame.grid(row=0, column=1)
@@ -185,7 +185,7 @@ def init_results_frame(self, row):
 
     def create_network(self):
         # Set the neural network parameters.
-        self.layer_sizes = tuple([784] + list(literal_eval(self.layer_sizes_var.get())) + [10])
+        self.layer_sizes = (784,) + literal_eval(self.layer_sizes_var.get()) + (10,)
         self.sigmoid = self.sigmoid_var.get()
         if not (self.sigmoid == 'logistic' or self.sigmoid == 'tanh'):
             raise ValueError('Invalid sigmoid function.')
@@ -209,12 +209,12 @@ def train(self):
         # Set the training parameters.
         self.num_iterations = int(self.num_iterations_var.get())
         self.learning_rate = float(self.learning_rate_var.get())
-        self.learning_decay = float(self.learning_decay_var.get())
+        self.layer_decay = float(self.layer_decay_var.get())
         self.momentum = float(self.momentum_var.get())
         self.batch_size = int(self.batch_size_var.get())
 
         stochastic_gradient_descent(self.nn, self.train_input, self.train_output, num_iterations=self.num_iterations,
-                                    learning_rate=self.learning_rate, learning_decay=self.learning_decay,
+                                    learning_rate=self.learning_rate, layer_decay=self.layer_decay,
                                     momentum=self.momentum, batch_size=self.batch_size)
 
         self.test()
@@ -248,13 +248,13 @@ def validate(self):
         else:
             raise ValueError('Invalid sigmoid function.')
 
-        training_prediction_rate = 100 * get_prediction_rate(self.nn, self.train_input, self.train_output)
-        test_prediction_rate = 100 * get_prediction_rate(self.nn, test_input, test_output)
+        training_prediction_accuracy = 100 * get_prediction_accuracy(self.nn, self.train_input, self.train_output)
+        test_prediction_accuracy = 100 * get_prediction_accuracy(self.nn, test_input, test_output)
         training_loss = self.nn.get_loss(self.train_input, self.train_output)
         test_loss = self.nn.get_loss(test_input, test_output)
 
-        self.validation_training_accuracy_var.config(text=('%.2f %%' % (training_prediction_rate)))
-        self.validation_test_accuracy_var.config(text=('%.2f %%' % (test_prediction_rate)))
+        self.validation_training_accuracy_var.config(text=('%.2f %%' % (training_prediction_accuracy)))
+        self.validation_test_accuracy_var.config(text=('%.2f %%' % (test_prediction_accuracy)))
         self.validation_training_loss_var.config(text=('%.4f' % (training_loss)))
         self.validation_test_loss_var.config(text=('%.4f' % (test_loss)))
 

src/state_of_the_art.py src/mnist_convolutional.py

@@ -5,30 +5,13 @@
 import mnist
 
 from neural_network import NeuralNetwork
-from preprocessing import *
 from training import stochastic_gradient_descent
-
-NUM_EXAMPLES = 59999
+from preprocessing import *
 
 
-def test_mnist_one_hot(num_train_examples=-1, num_test_examples=-1, hidden_layers=(24, 32), sigmoid='tanh',
-                       learning_rate=0.01, learning_decay=1.0, momentum=0.0, batch_size=100, num_epochs=100,
+def test_mnist_one_hot(num_train_examples=-1, num_test_examples=-1, hidden_layers=(100,), sigmoid='tanh',
+                       learning_rate=0.01, layer_decay=1.0, momentum=0.0, batch_size=100, num_epochs=100,
                        csv_filename=None, return_test_accuracies=False):
-    layer_sizes = (784,) + hidden_layers + (10,)
-    weight_decay = 0.0
-
-    print('Network Parameters')
-    print('layer_sizes: {}, sigmoid: {}, weight_decay: {}'.format(layer_sizes, sigmoid, weight_decay))
-
-    # Set the training parameters.
-    num_iterations = (NUM_EXAMPLES // batch_size) * num_epochs
-
-    print('Training Parameters')
-    print('num_iterations: {}, learning_rate: {}, learning_decay: {}, momentum: {}, batch_size: {}'.format(
-        num_iterations, learning_rate, learning_decay, momentum, batch_size))
-
-    print('')
-
     # Collect and preprocess the data.
     if sigmoid == 'logistic':
         train_input = convert_mnist_images_logistic(mnist.train_images()[:num_train_examples])
@@ -46,9 +29,12 @@ def test_mnist_one_hot(num_train_examples=-1, num_test_examples=-1, hidden_layer
         raise ValueError('Invalid sigmoid function.')
 
     # Create and train the neural network.
+    layer_sizes = (784,) + hidden_layers + (10,)
+    weight_decay = 0.0
     nn = NeuralNetwork(layer_sizes, sigmoid=sigmoid, weight_decay=weight_decay)
 
     num_examples = train_input.shape[0]
+    num_iterations = (num_examples // batch_size) * num_epochs
 
     rows = None
     if csv_filename is not None:
@@ -61,23 +47,31 @@ def test_mnist_one_hot(num_train_examples=-1, num_test_examples=-1, hidden_layer
     def callback(iteration):
         if iteration % (num_examples // batch_size) == 0:
             epoch = iteration // (num_examples // batch_size)
-            training_prediction_rate = get_prediction_rate(nn, train_input, train_output)
-            test_prediction_rate = get_prediction_rate(nn, test_input, test_output)
+            training_prediction_accuracy = get_prediction_accuracy(nn, train_input, train_output)
+            test_prediction_accuracy = get_prediction_accuracy(nn, test_input, test_output)
             training_loss = nn.get_loss(train_input, train_output)
             test_loss = nn.get_loss(test_input, test_output)
-            print('{},{:.6f},{:.6f},{:.6f},{:.6f}'.format(epoch, training_prediction_rate, test_prediction_rate,
+            print('{},{:.6f},{:.6f},{:.6f},{:.6f}'.format(epoch, training_prediction_accuracy, test_prediction_accuracy,
                                                           training_loss, test_loss))
             if csv_filename is not None:
-                rows.append((epoch, training_prediction_rate, test_prediction_rate, training_loss, test_loss))
+                rows.append((epoch, training_prediction_accuracy, test_prediction_accuracy, training_loss, test_loss))
             if return_test_accuracies:
-                test_accuracies.append(test_prediction_rate)
+                test_accuracies.append(test_prediction_accuracy)
+
+    print('Network Parameters')
+    print('layer_sizes: {}, sigmoid: {}, weight_decay: {}'.format(layer_sizes, sigmoid, weight_decay))
+    print('Training Parameters')
+    print('num_iterations: {}, learning_rate: {}, layer_decay: {}, momentum: {}, batch_size: {}'.format(
+        num_iterations, learning_rate, layer_decay, momentum, batch_size))
+    print('')
 
     header = 'epoch,training_accuracy,test_accuracy,training_loss,test_loss'
     print(header)
     stochastic_gradient_descent(nn, train_input, train_output, num_iterations=num_iterations,
-                                learning_rate=learning_rate, learning_decay=learning_decay,
+                                learning_rate=learning_rate, layer_decay=layer_decay,
                                 momentum=momentum, batch_size=batch_size,
                                 callback=callback)
+
     if csv_filename is not None:
         save_rows_to_csv(csv_filename, rows, header.split(','))
 

src/mnist_hello_world.py src/mnist_fully_connected.py

@@ -37,15 +37,15 @@ def flatten_input_data(images):
 
 
 def convert_mnist_labels_one_hot(labels, positive, negative):
-    lst = []
+    data = []
     for label in labels:
         label_one_hot = negative * np.ones(10)
         label_one_hot[label] = positive
-        lst.append(np.array([label_one_hot]))
-    return np.array(lst)
+        data.append(np.array([label_one_hot]))
+    return np.array(data)
 
 
-def get_prediction_rate(nn, test_input, test_output):
+def get_prediction_accuracy(nn, test_input, test_output):
     prediction = nn.predict(test_input)
     diff = np.argmax(prediction, 2) - np.argmax(test_output, 2)
     error = np.count_nonzero(diff) / diff.size

src/preprocessing.py

@@ -4,15 +4,15 @@
 
 
 def stochastic_gradient_descent(nn, input_vectors, output_vectors, num_iterations=1000,
-                                learning_rate=0.1, learning_decay=1.0, momentum=0.0, batch_size=1,
+                                learning_rate=0.1, layer_decay=1.0, momentum=0.0, batch_size=1,
                                 callback=lambda iteration: None):
     """
     Trains the neural network by using stochastic gradient descent
     with the given training examples.
     """
 
     delta_weights = [np.zeros(weight.shape) for weight in nn.get_weights()]
-    learning_rates = get_learning_rates(nn, learning_rate, learning_decay)
+    learning_rates = get_learning_rates(nn, learning_rate, layer_decay)
     for iteration in range(num_iterations):
         # Get a random batch of examples.
         random_indices = np.random.randint(input_vectors.shape[0], size=batch_size)
@@ -27,14 +27,14 @@ def stochastic_gradient_descent(nn, input_vectors, output_vectors, num_iteration
 
 
 def batch_gradient_descent(nn, input_vectors, output_vectors, num_iterations=1000,
-                           learning_rate=0.1, learning_decay=1.0, momentum=0.0, callback=lambda iteration: None):
+                           learning_rate=0.1, layer_decay=1.0, momentum=0.0, callback=lambda iteration: None):
     """
     Trains the neural network by using standard batch gradient
     descent with the given training examples.
     """
 
     delta_weights = [np.zeros(weight.shape) for weight in nn.get_weights()]
-    learning_rates = get_learning_rates(nn, learning_rate, learning_decay)
+    learning_rates = get_learning_rates(nn, learning_rate, layer_decay)
     for iteration in range(num_iterations):
         # Update the weights using the examples.
         update_weights(nn, input_vectors, output_vectors, delta_weights, learning_rates, momentum)
@@ -52,11 +52,11 @@ def update_weights(nn, input_vectors, output_vectors, delta_weights, learning_ra
     nn.set_weights(weights)
 
 
-def get_learning_rates(nn, learning_rate, learning_decay):
+def get_learning_rates(nn, learning_rate, layer_decay):
     learning_rates = []
     running_learning_rate = learning_rate
     for i in range(nn.num_layers - 1):
         learning_rates.append(running_learning_rate)
-        running_learning_rate *= learning_decay
+        running_learning_rate *= layer_decay
     learning_rates *= 2
     return learning_rates