training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Test if network works as intended"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from neural_network import Layer\n",
    "from neural_network import LoadData\n",
    "from neural_network import ActivationFunction\n",
    "from neural_network import NeuralNetwork\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creation of the network\n",
    "dimensions = np.array([4, 8, 5, 3])\n",
    "functions = np.array([ActivationFunction.sigmoid(), ActivationFunction.sigmoid(), ActivationFunction.sigmoid()])\n",
    "ff_nn = NeuralNetwork.NeuralNetwork(functions, dimensions, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load training and testing data\n",
    "DataLoader = LoadData.LoadData()\n",
    "X_train, X_test, y_train, y_test = DataLoader.partition_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_error_datasets(X_dataset, y_dataset, network):\n",
    "    error_list = np.zeros((X_dataset.shape[2], ))\n",
    "    for i in range(X_dataset.shape[2]):\n",
    "        X_in = X_dataset[:,:,i]\n",
    "        y    = y_dataset[:,:,i]\n",
    "        y_out = ff_nn.propagation(X_in)\n",
    "        diff = y - y_out\n",
    "        error = np.dot(diff.T, diff)\n",
    "        error_list[i] = error[0,0]\n",
    "    return error_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10000\n",
    "batch_size = 5\n",
    "training_error = np.zeros(shape=(epochs, X_train.shape[2])) # Container of the training error\n",
    "testing_error = np.zeros(shape=(epochs, X_test.shape[2])) #Container for the testing error\n",
    "for k in range(epochs):\n",
    "    # Select withing the training set the samples that will be given for the batch\n",
    "    indexes = np.random.randint(0, high=99, size=(batch_size,))\n",
    "    X_train_ = X_train[:,:,indexes].copy()\n",
    "    y_train_ = y_train[:,:,indexes].copy()\n",
    "    \n",
    "    #Bear in mind this is not the most efficient way to implement a nn. \n",
    "    #It was used merely to illustrate the mathematical concepts\n",
    "    #At this point the weights and biases of the network were updated for this specific\n",
    "    #sample\n",
    "    for i in range(X_train_.shape[2]):\n",
    "        X_in = X_train_[:,:,i]\n",
    "        y_in = y_train_[:,:,i]\n",
    "        ff_nn.backpropagation_algorithm(X_in, y_in)\n",
    "        \n",
    "    # It is interesting to see how the error evolves with the time\n",
    "    # At this point we run the network as is for the test set\n",
    "    training_error[k, :] = evaluate_error_datasets(X_train, y_train, ff_nn)\n",
    "    testing_error[k, :]  = evaluate_error_datasets(X_test, y_test, ff_nn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_error_stat = np.mean(training_error, axis=1)\n",
    "testing_error_stat  = np.mean(testing_error, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'mean squared error')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAEkCAYAAAB33tUkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYnFWd9//3t9Ze0+mkOwtZCMEABiQEYthUFhEJIjijPoqguCCP4Ia4jOi4ofP8XEdHURERHVzAERERFURlGwRMgOwkIYGENNk6a6e36lq+vz/uu0ml6XRXJ12pqs7ndV11Vd37uVOET865T51j7o6IiEiliJS6ACIiIkOh4BIRkYqi4BIRkYqi4BIRkYoSK3UBRERkaJ544olxsVjsJuA4RnYFJAcszWQyl5900klbelcquEREKkwsFrtpwoQJL29ubt4RiURGbNfwXC5nra2tMzdt2nQTcGHv+pGc1CIiI9Vxzc3NbSM5tAAikYg3NzfvIqhZ7llfovKIiMj+i4z00OoV3udeWaXgEhGRIdu6dWv0q1/9avNQjzvjjDNetnXr1uiBXFvBJSIiQ7Zt27boT37yk3F912cymQGPe/DBB1c3NTVlD+Ta6pwhIiJD9vGPf3zy+vXrk8ccc8zMWCzmtbW12XHjxqWXL19es2bNmmXnnHPOkRs3bkykUqnIBz7wgc2f+MQntgJMmjTpFQsWLHi6ra0tMm/evBlz585tX7BgQd348eN77r333tV1dXWDNoEquEREKtgnb180ZdWm3TXDec6jJtR3fuMts9YPtM+3vvWtlgsuuKB6xYoVy+++++76t771rS976qmnlh1zzDE9AL/85S/Xjh8/Ptve3m6zZ8+eeemll+6YMGHCXjWt559/vuoXv/jFs6eddtq6888/f/ott9zSeNVVV20frHwKLhEROWDHH398R29oAXzta18b/8c//nE0wKZNm+LLli2rmjBhQkf+MZMmTUqddtppXQCzZ8/uXLt2bbKQaym4REQq2GA1o4OlpqYm1/v57rvvrn/wwQfrFyxYsKK+vj43d+7co7u6ul7SpyKRSLzYLBiNRr2/ffqjzhkiIjJkDQ0N2Y6Ojn4zZOfOndGGhoZsfX197qmnnqpatGhR7XBeWzUuEREZsgkTJmRPOumk9hkzZhybTCZzzc3N6d5tb37zm3fdeOONzUcdddTMI488snvWrFkdA51rqEwTSYqIVJZFixatnTVr1tZSl+NgWbRoUdOsWbOm9S6rqVBERCqKgktERCqKgktERCqKgktERCqKgktERCqKgktERCqKgktERIZsf6c1AbjuuuvG7d69e7/zR8ElIiJDtq9pTQrxox/9aHx7e/t+549GzhARkSHLn9bkjDPOaBs3blz6d7/73Zienh57wxvesPPb3/72hra2tsiFF144fePGjYlcLmef+tSnNmzevDm+ZcuW+BlnnHFUY2Nj5vHHH1811GsruEREKtmdH5zCluXDOq0J42Z28qbvFzytyR133DHqN7/5TePixYufdnfOOeecl/35z3+u27x5c2zChAnpBx54YDUEtbSxY8dmf/jDH45/8MEHV02cOHHgWSf3QU2FIiJyQO65555RDz300KiZM2fOPPbYY2euWbOmasWKFVUnnnhi18MPPzzqyiuvnHTPPffUjR079oBmPu6lGpeISCUbpGZ0MLg7V1999cZPfvKTLxk/8cknn1z+29/+tuGzn/3spL/+9a9t3/zmNzce6PVU4xIRkSHLn9Zk3rx5bT//+c+bdu3aFQF47rnn4i+88EJs7dq18fr6+txVV121/eqrr968cOHCGoDa2tps7777QzUuEREZsvxpTc4+++xdb33rW7e/8pWvPAaCSSV/+ctfPrdixYrktddeOzkSiRCLxfwHP/jBOoDLLrts67x582aMGzcuvT+dMzStiYhIhdG0JiIiIhVEwSUiIhVFwSUiIhVlRHXOaGpq8mnTppW6GCIiRfW1r32NZcuWHW5mB3yuVCqVmT179qJhKFZR5HI5A3L560ZUcE2bNo0FCxaUuhgiIkX13HPPUV9fz9ixYznQ8Fq6dGnPMBVr2OVyOWttbW0AluavH1HBJSJyKJg8eTItLS20trYe8Lk2bdoUy2azTcNQrGLIAUszmczl+SsVXCIiFSYej3PEEUcMy7lmzpy5xN3nDMvJDhJ1zhARkYqi4BIRkYqi4BIRkYqi4BIRkYqi4BIRkYpyyAdXLud872/P8OCqA+9WKiIixXfIB1fE4FUPXULXQ98rdVFERKQAh3xwYca0yCZq29aUuiQiIlIABRfQFm0kmdpW6mKIiEgBFFxAV6KJ2rSCS0SkEii4gHRVEw3ZHWg2aBGR8le04DKzm81si5kt3cf2T5rZwvC11MyyZjYm3LbWzJaE24o+3HuutpkmdtKRyhT7UiIicoCKWeP6GXDevja6+zfc/QR3PwG4FnjQ3bfn7XJWuL3ogz/masdRZWna23YU+1IiInKAihZc7v4QsH3QHQMXA7cWqyyDiVSNAqBrt4JLRKTclfwZl5nVENTMfpu32oG/mNkTZnbFIMdfYWYLzGzB/s5NE6tpAKCro22/jhcRkYOn5MEFvBF4pE8z4enufiIwD/igmb1mXwe7+43uPsfd5zQ3N+9XARI1QY0r1b5zv44XEZGDpxyC6+30aSZ09w3h+xbgd8DcYhYgURvUuNKdqnGJiJS7kgaXmTUAZwC/z1tXa2b1vZ+Bc4F+eyYOl6re4OraVczLiIjIMIgV68RmditwJtBkZi3AF4A4gLvfEO72L8Bf3L0j79DxwO/MrLd8v3L3e4pVToDqukYAsl27i3kZEREZBkULLne/uIB9fkbQbT5/3bPArOKUqn819UGNK9etpkIRkXJXDs+4Si5WHXTOoKe9tAUREZFBKbgAYkl6iEFKTYUiIuVOwRXqpIZoumPwHUVEpKQUXKGuSDXxjJoKRUTKnYIrlLYk0Vyq1MUQEZFBKLhCGUsQyfaUuhgiIjIIBVcoE0moxiUiUgEUXKFMpIp4TjUuEZFyp+AKZaMJYq4al4hIuVNwhXKRJHFXjUtEpNwpuEK5aJK4p0tdDBERGYSCK+SxJAk1FYqIlD0FV8ijVSRQjUtEpNwpuEIeS5IkTTbnpS6KiIgMQMHVK1ZFFT30pLOlLomIiAxAwRWyWJKIOame7lIXRUREBqDgClm8GoBUV1eJSyIiIgNRcIUsXgVAT6qzxCUREZGBKLhCkbDGlVZwiYiUNQVXKBLWuNLdCi4RkXJWtOAys5vNbIuZLd3H9jPNbJeZLQxfn8/bdp6ZrTSz1Wb26WKVMV80GdS4Mik94xIRKWfFrHH9DDhvkH0edvcTwtd1AGYWBb4PzANmAheb2cwilhOAaNhUmEmrV6GISDkrWnC5+0PA9v04dC6w2t2fdfce4DbgomEtXD9iiSQA2R7VuEREylmpn3GdamaLzOzPZnZsuG4SsD5vn5ZwXVFFE0GNK6umQhGRshYr4bWfBA5393YzOx+4E5gBWD/77nMcJjO7ArgCYOrUqftdmFj4jCurpkIRkbJWshqXu7e5e3v4+U9A3MyaCGpYU/J2nQxsGOA8N7r7HHef09zcvN/liYfBlVNwiYiUtZIFl5lNMDMLP88Ny7INmA/MMLMjzCwBvB24q9jl2RNcmtpERKScFa2p0MxuBc4EmsysBfgCEAdw9xuAtwBXmlkG6ALe7u4OZMzsQ8C9QBS42d2XFaucvRJhcLlqXCIiZa1oweXuFw+y/Xrg+n1s+xPwp2KUa18SVbXBh4yCS0SknJW6V2HZiCeDkTNcwSUiUtYUXCGLBcFFRs+4RETKmYKrVyRKmiim4BIRKWsKrjw9xLGsmgpFRMqZgitPDwnI9pS6GCIiMgAFV560xYlk1VQoIlLOFFx50pZQcImIlLkBg8vMomb2i4NVmFLLWIJoTk2FIiLlbMDgcvcs0BwOvTTiZSIJojnVuEREylkhI2esBR4xs7uAjt6V7v6fxSpUqWQiSdW4RETKXCHBtSF8RYD64hantLKRBLGM5uMSESlngwaXu38JwMzqg8VgKpKRKBdJkPBdpS6GiIgMYNBehWZ2nJk9BSwFlpnZE3mzFY8o2UiSmKupUESknBXSHf5G4Bp3P9zdDwc+Dvy4uMUqjVw0SdzTpS6GiIgMoJDgqnX3+3sX3P0BoLZoJSohjyZJqMYlIlLWCumc8ayZfQ74ebh8KfBc8YpUOh5LkkA1LhGRclZIjeu9QDNwR/hqAt5TzEKVikeTJOghmIhZRETK0YA1LjOLAp9x948cpPKUVqyKJGlSmRxV8WipSyMiIv0oZOSMkw5SWUrOYlXELEeqR8+5RETKVSHPuJ4KR834DXuPnHFH0UpVKvEkAKlUJ9RWl7gwIiLSn0KCawywDTg7b50TPO8aUSKxILjSXRo9Q0SkXBXyjGuxu397qCc2s5uBC4At7n5cP9svAf4tXGwHrnT3ReG2tcBuIAtk3H3OUK+/PyxeBUBPSsElIlKuCnnGdeF+nvtnwHkDbH8OOMPdjwe+TPBD53xnufsJByu0ACJhcKV7Og/WJUVEZIgKaSr8h5ldD/yavZ9xPTnQQe7+kJlNG2D7P/IWHwMmF1CWoookgudamVR3iUsiIiL7UkhwnRa+X5e3ztn7mdeBeh/w5z7n/4uZOfAjd+9bG3uRmV0BXAEwderUAypENKxxZdRUKCJStgoZHf6sYhbAzM4iCK5X5a0+3d03mNk44D4zW+HuD+2jfDcSNjPOmTPngH45HA1rXNm0alwiIuWqkNHhx5vZT8zsz+HyTDN733Bc3MyOB24CLnL3bb3r3X1D+L4F+B0wdziuN5hYIqhxZVXjEhEpW4UM+fQz4F7gsHB5FXD1gV7YzKYSdKl/p7uvyltfG879hZnVAucSTKlSdPGkalwiIuWukGdcTe7+P2Z2LYC7Z8wsO9hBZnYrcCbQZGYtwBeAeHiOG4DPA2OBH5gZ7On2Ph74XbguBvzK3e8Z6o3tj3iyBlBwiYiUs0KCq8PMxhJ0mMDMTgEGnSbY3S8eZPvlwOX9rH8WmFVAuYZdLBk0FXpaTYUiIuWqkOC6BrgLONLMHiEYKf4tRS1VifQ2FeZ6UiUuiYiI7EshvQqfNLMzgKMBA1a6j8xpghNhU6Fn1FQoIlKuCqlx4e4ZYFmRy1JyiaqgxqXgEhEpX4X0KjxkROPhiPAZNRWKiJQrBVe+aJycG6bgEhEpW/tsKjSzEwc6cLCxCiuSGT0WBzUVioiUrYGecX0rfK8C5gCLCDpnHA88zt5DNI0YKRJYRt3hRUTK1T6bCt39rHCcwnXAie4+x91PAmYDqw9WAQ+2lFURUXCJiJStQp5xHePuS3oX3H0pcELxilRaqUg1sayCS0SkXBXSHf5pM7sJ+AXB6BmXAk8XtVQllI5UEc9qIkkRkXJVSHC9B7gS+Gi4/BDww6KVqMTSqnGJiJS1QkbO6DazG4A/ufvKg1CmkspEq0mktw2+o4iIlEQh83FdCCwE7gmXTzCzu4pdsFLJxqpJ5tQdXkSkXBXSOeMLBBM57gRw94XAtCKWqaRysRqSKLhERMpVIcGVcfdBpzEZKXLxaqpdwSUiUq4KCa6lZvYOIGpmM8zse8A/ilyukvF4LTWk6MnkSl0UERHpRyHB9WHgWCAF/IpgEsmri1mokorXkLQ0Xd09pS6JiIj0Y8BehWYWBb7k7p8EPntwilRalqwDoKuzjYa6qhKXRkRE+hqwxuXuWeCkg1SWshANJ5Ps7txd4pKIiEh/CvkB8lNh9/ffAB29K939jqKVqoQiyVoAejrbS1wSERHpTyHPuMYA24CzgTeGrwsKObmZ3WxmW8xs6T62m5l918xWm9ni/KlUzOwyM3smfF1WyPWGQyxZD0BPl2pcIiLlqJCRM95zAOf/GXA9cMs+ts8DZoSvkwmGkjrZzMYQ/H5sDsH4iE+Y2V3uvuMAylKQWHVY41JwiYiUpUGDy8yqgPcR9Cx8sbeCu793sGPd/SEzmzbALhcBt7i7A4+Z2WgzmwicCdzn7tvDMtwHnAfcOtg1D1S8Kuicke1WU6GISDkqpKnw58AE4PXAg8BkYLiqI5OA9XnLLeG6fa0vukRNEFyZ7o5B9hQRkVIoJLhe5u6fAzrc/b+BNwCvGKbrWz/rfID1Lz2B2RVmtsDMFrS2th5wgZI1DQDkutVUKCJSjgoJrnT4vtPMjgMaGL6xCluAKXnLk4ENA6x/CXe/MZydeU5zc/MBF6hu9FgAsl07D/hcIiIy/AoJrhvNrBH4HHAXsBz4+jBd/y7gXWHvwlOAXe6+EbgXONfMGsNrnxuuK7pkbSMAruASESlLhfQqvCn8+CAwfSgnN7NbCTpaNJlZC0FPwXh43huAPwHnA6uBToJJK3H37Wb2ZWB+eKrrejtqFF00TgdVWKrtoFxORESGppBehZ/vb727XzfYse5+8SDbHfjgPrbdDNw82DWKodPqiPYouEREylEhI2fkd6+rIvjx8dPFKU556IrWEU8ruEREylEhTYXfyl82s28SPJsasbqjdSQz6lUoIlKOCumc0VcNQ3zWVWnS8VFUZ/UDZBGRclTIM64l7PkNVRRoBgZ9vlXJsslRNOx+ptTFEBGRfhTyjCt/QN0MsNndM0UqT1nIJRuop4N0Nkc8uj+VUhERKZZCgqvvw55RZnsGtjho3dQPIqtqoJ4utnemaKqvLnVxREQkTyHB9STBKBY7CIZiGg08H25zRuDzrkhNIxFzdu/cRlP95FIXR0RE8hTSDnYP8EZ3b3L3sQRNh3e4+xHuPuJCCyBaFwwd1blzc4lLIiIifRUSXK909z/1Lrj7n4Ezilek0os3jAege8fGEpdERET6KqSpcKuZ/TvwC4KmwUsJZkQeseqbghlUuncquEREyk0hNa6LCbrA/w64M/w84FBOla5xXBBc6V1qKhQRKTeFjJyxHfgogJlFgVp3H9HjISXrm8kQwXcruEREys2gNS4z+5WZjTKzWmAZsNLMPln8opVQJEqbNRDt3FrqkoiISB+FNBXODGtYbyKYhmQq8M6ilqoM7I6NoSql4BIRKTeFBFfczOIEwfV7d0+zZwioEasr2UR9WsElIlJuCgmuHwFrgVrgITM7HBjRz7gAUvVTmeSb6O4Z0aNbiYhUnEGDy92/6+6T3P38cOLH54Gzil+0EhsznVHWycZNG0pdEhERyTPkEWQ9MOKrIdUTZgCwo2VliUsiIiL5NPT5PjROPgaAzo2rSlwSERHJp+Dah6bJM8i5kd66ptRFERGRPIUM+YSZnQZMy9/f3W8pUpnKgsWraI02Ed+1ttRFERGRPIXMgPxz4EhgIZANVzswaHCZ2XnAfxHMnHyTu3+1z/Zvs6ejRw0wzt1Hh9uywJJw2/PufuGgdzPMdlRPpanzuYN9WRERGUAhNa45BD9CHtJvt8Lhob4PvA5oAeab2V3uvrx3H3f/WN7+HwZm552iy91PGMo1h1vn2OM5rv0Wtu3YydjG0aUsioiIhAp5xrUUmLAf554LrHb3Z929B7gNuGiA/S8Gbt2P6xRN7fRTiFuW55Y9WuqiiIhIqJDgagKWm9m9ZnZX76uA4yYB6/OWW8J1LxH+qPkI4O95q6vMbIGZPWZmb9rXRczsinC/Ba2trQUUq3CTj381AO3PKLhERMpFIU2FX9zPc1s/6/bV3Ph24HZ3z+atm+ruG8xsOvB3M1vi7i/p4ufuNwI3AsyZM2dYh6KqGTOJDZHDqNn42HCeVkREDkAh05o8uJ/nbgGm5C1PBvY1DMXbgQ/2ue6G8P1ZM3uA4PnXQe+bvmPCqcx84Y+s27KDw8c1HuzLi4hIH4VMa3KKmc03s3Yz6zGzrJkVMlbhfGCGmR1hZgmCcHpJE6OZHQ00Ao/mrWs0s2T4uQk4HVje99iDYcKci6izbpY8cHspLi8iIn0U8ozreoKOE88A1cDl4boBhcNCfQi4F3ga+B93X2Zm15lZftf2i4Hb+vRafDmwwMwWAfcDX83vjXgwjZ11PtsjY5m44mf0ZHKlKIKIiOQp6AfI7r7azKLhM6ifmtk/CjzuTwRzeOWv+3yf5S/2c9w/gFcUco2ii8bZccIHOOnJ/+C2W2/g7e+8qtQlEhE5pBVS4+oMm/oWmtnXzexjBFOcHDKOPP9qNlYfxXmrv8J9Dz9S6uKIiBzSCgmud4b7fQjoIOhw8eZiFqrsxBI0ve/XWDTGUX99D0/+86FSl0hE5JBlhQyIYWbVBN3Ty3qOjzlz5viCBQuKdv4dKx/Bbn0bo9nNusQMIpNOYEJ1jnguBUeeDT3tcNQ8GDMdojFI7YZMCqpGQ6oNasYUrWwiIvvDzJ5w9zmlLsdQDBpcZvZG4JtAwt2PMLMTgOtKMXbgYIodXADdbVuZf/t/MvH5P9DoOxhruwc9Jmsxop6hvf4I4pkOuuuPoH3iXKq6t1KzcxXRaJSIOZHDTyfSvQM6WuHw02Hb6iD4qhvh+LfCqElQN66o9ycih5aRGlxPAGcDD7j77HDdYnc//iCUb0gORnD1SmdzLFi7g/nPtrJ7/WLiO9aQ62qjpmcrJ7OYnEfoIcZ6byZOllmRNRwTWU/KYyRtzzycu7wGI/hldoN17nWNLBFyRIizZ/9UfDRrT/kyk7Y/Su0Lj2Bjj4SJs+CcLx6U+xaRkWWkBtfj7n6ymT2l4CqMu5PK5OhOZ8nknK6eLF3pLD2ZHJmc05PJkU51kelsY6fV05nO0ZHKkOrupqMnR3dPirq2Z1kTOZy2HsfaN3LcjvuZnFnHO2L37/O62yaeQd2o0SRf+xmoGRvU1KIFdRwVkUNUJQZXIf9XW2pm7wCiZjYD+AhQUHf4Q5WZURWPUhWPDrJnv0M3hk7qs/wm0tkcazdvpWvVA6zNNLIkM4UdrRu4+tkrGJ3bydiND8JGYOXvAdhVMxU76d2MOuFNMPbIA7gjEZHyUUiNqwb4LHAuwfiD9wJfdvfu4hdvaMqlxlUKbd1p1ixbQGThL1nSM56Xt97DSbmle+3TevoXaH7dNSUqoYiUo0qscRXUq7BSHMrB1Z+169fD7z/ItK17Dze5ZdzpjPm/dxOLFvJrCBEZyUZkcJnZHOAzwDTymhb1jKuybNuygewt/8K49hUvrrvt3H/ylpNnKMBEDmEjNbhWAp8ElgAvDtbn7uuKW7ShU3ANLrt1DdHrTwQg7VG+nLyG0y+6nHNnjsesv5loRGQkq8TgKuSf2q3ufpe7P+fu63pfRS+ZFEW06Uj44i78VdcQtyzX9XyD+b+6jvfe/BhLX9hV6uKJiAyqkBrXawlGcP8bkOpd7+53FLdoQ6ca1xDtegG/5SJs2zN0epKbsvNIn/5Jrnrty6lODNYjUkRGgpFa43oPcAJwHvDG8HVBMQslB0nDJOyqR+Gsz1JjKT4Su5PXPvouLvnOH3hoVWupSyci0q9CalxL3L08phgZhGpcB6C9Ff7y77D4NgBavIk7T7+LD517bIkLJiLFNFJrXI+Z2cyil0RKq64Z/vVHcP43AZhsW3n1/17KNb9eSCarCTRFpHwUElyvIpiLa6WZLTazJWa2uNgFkxKZ+3743FZ8yqnMijzLF5efx0dvW0hnT2bwY0VEDoJCgus8YAbByBm9z7feWMxCSYlF49ilvwFglHVx2oqv8O6fzqc9pfASkdIbNLjyu8CrO/whJFkP/7YWMC6J/o0pz9/J+/97gcJLREpOQybIvlU3wjVPA/Ct+A10r/snl/z4MbbsLrthKkXkEFLU4DKz88JnY6vN7NP9bH+3mbWa2cLwdXnetsvM7JnwdVkxyykDGDUR/s8tAPwu/jm2b1rHOd96kPlrt5e4YCJyqCpacJlZFPg+MA+YCVy8j96Jv3b3E8LXTeGxY4AvACcDc4EvmFljscoqg5h5EVz2BwAejl3FvPR9XPGzx1m9ZfDZn0VEhlsxa1xzgdXu/qy79wC3ARcVeOzrgfvcfbu77wDuI+gkIqVyxGte7Cr/tfiPucH+g3f95J+s29ZR4oKJyKGmmME1CVift9xC/zMnvjnsZn+7mU0Z4rFyMM19P3z4SQBO9iWc2XM/7/7pfLZ39JS4YCJyKClmcPU31HjfYTr+AEwLp0j5K/DfQzg22NHsCjNbYGYLWls1TFHRjT3yxfD6f/49xu5cwrtufpzW3alBDhQRGR7FDK4WYEre8mRgQ/4O7r7N3Xv/j/dj9sxXP+ixeee40d3nuPuc5ubmYSm4DGLskfCWnwJwe+zfWb1lN2/6/iOs3KRnXiJSfMUMrvnADDM7wswSwNuBu/J3MLOJeYsXAk+Hn+8FzjWzxrBTxrnhOikXx/0rnPsVAFZEL6Yz1cP/+dGjLGnR1CgiUlxFCy53zwAfIgicp4H/cfdlZnadmV0Y7vYRM1tmZouAjwDvDo/dDnyZIPzmA9eF66ScnPqhFz/Or/8EdXG44ucLFF4iUlSDjg5fSTQ6fAm4ww9OgdYVdI47kdds/wzbO3r4xOuP5sozjtSsyiJlbqSODi+yb2Zw1WMA1Gx5kofPfpY3HH8YX79nJd/7+2pG0j+MRKQ8KLjkwJnB57fD9LOo/sun+O7uj/H+Y53/vG8V375vValLJyIjjIJLhkckCv9yAzRMxTY8xWfXXMI7Zzfy3b+v5ppfL6RlR2epSygiI4SCS4ZP/QT42BK48HoArtt0JR959WH8ftEG5v3Xw/xhUb+/aBARGRIFlwy/E98J53wJ27GWazq+w/3XnMHLxtXx4Vuf4lO3L9LUKCJyQBRcUhyvuhpe8ylYfidTb30Nvznf+OBZR/KbJ1o451sPcus/n6cnkyt1KUWkAim4pHjOvBaajoZtq4n9bB6fnLiE3155GuMbqrj2jiWc8Y37+ekjz7GtXcNFiUjh9DsuKa5cDnY8B987MVie93V87hU8/MxWvvPXVTz5/E6iEeNVL2virXMmc9LhjYytTZKI6d9UIgdDJf6OS8ElB0eqHX7zblh9H5x4GbzuS1DdyIpNbfxh0QZuf6KFzW1Bzas+GePcYydwyvQxzJoymiOb64hG9ENmkWJQcJWYgqvMZdPwt+vgH9+FRB3MvQI1gqosAAASsElEQVRe/XFI1pHO5li0fieLW3axfGMb9y7bxO7uoBNHbSLKsYc1cNSEOl7WXMeUMTWcOLWRxtpEiW9IpPIpuEpMwVUhNi2B//02LP0tJBvgqHODEJsy98Vdcjnn2a3tLFq/i8UtO1nyQhBo3ek9HToaa+JMGVPDYQ3VNNUnGF9fxZi6BNXxKM31Seqr4tQmotQmY4yuiVMdj2oIKpE+FFwlpuCqMC0LYP5NsOxOyHTBpDkw9mXQOA1mvQ3GTN9r91zO2d7Zw+ot7Sxu2clzWztp2dHJpl3dtLan2NmZHvByEYO6ZIz6qjj1VTHqkjGqE1ES0QhV8SjJePBeFYtSFY+QyTnuTl0yTm0yCMDaZIzaRJTqeJRYNEIsasQjEcyCAUSq41EgmDyu969WVTxCLBKhJ5NjdypNY02C0TVx4tEI8aie5UlpKbhKTMFVobrb4NHvw/Lfw9aV4DnAYNqrYOqpQU1s0klQ3Rikw75Ok86yqytNZ0+W1t0p2lNp2lNZ2rsz7O5Os7s7Q3sqw+5wuT2VobMnSzqboyudJZXO0Z3OBq9MjljEMGOvWt5wS8b2hFcyFiEaMWIRIxJeOxmLEosYyXiUZCwSvqLEoxYEZ7h/LGrEIsHx8agRjUTCdyMe3XPeYN9In/febRGiUSMRjVCTiFJfFWN0TYKG6rgCdgRTcJWYgmuEaNsAC26GVffA5mVhkBE8F2s6CqacDIefCoedCKMOC4abKqJ0NkdnT5bOngwdqeC9qydLJueksznS2eDvUDYXBKBhe+VrdzpLNgexqDGqKs62jhRtXRnS2Ry7u9Nkc+A4PZkcOXcyWSebc3LupDI5MrlgWyqTJZXJkUrnyOSC9b37prO9y+F7+Dk3TH+9E7EItYkoNYkYtcngvabPcm0iSk1YI22ojlOTjFETj1KTjL5Yy6wN18UUhGVDwVViCq4RKNUOG56E9Y9DeytsWR40MWa6gu3x2iC8qkcHzYyJWrAIjD4cqkZBvAZqm4Jmx1GTih5y5SYXhlg256RzObLZ8D0MvXQ2Fwbfnn16Mjk6e4Ka6c7ONG1daTr6BHdHT5bOVPDe1We5EIlohOb6ZBBmiRiJWIRELMLRE+qZ0ljDYaOrmDKmhkmjq6mKH1rf2cFWicEVK3UBRAaUrIMjXhO8emV6YONC2LQYWldB+ybo2Apr7odsD2S6Id3PoL6ROCTroaoh2C+agFgS6sYFoRZLQjYDsUQQmJ1bIRKDqtFBM2V1+F41OvgcS4JFgzDMpKCnAzwbHBOJBcsQBGkkGvSqbHshOEfdhCBkEzUQTQbnilVBNA61zZBqC/aPVQXhG9m/GkokYiTCnxJUU/wAyOWcrnSWnV1punqCptj2VBCAu7rSdKSC2mpHT5bNbd20dQVNtu2pDLt3pXlg5ZaX1BLH1SeZMqaGKY3V4XsNk8dUM7GhmvqqGGNqEkT0c4lDimpcMvK4Q/fOIHx6OqBjC2x/FnasDZ6npXYHYZJLBwG2cz10bgvCJxoP3pN1UNMU7NO1MzhfdxtBt4sii8Qglz+eo4UhVxsEZTQehG40sffnWPieywTNqrkMpLuC54KRWHDPFoG68cF9J+uDZthIbE9NdFdLsK7p6D1lqWoIAjZeE5QhXhMu1+79Hqsa8BlkIdLZHFt2p9iws4v12ztZv72Llh2drN8RfN64q+slwVafjHHG0c2cMn0skxurmTV5tH4qMQSVWONScIkUKpeF7l1BiHXtDALOs8H6WHJPsHg2CIZEXRAUmRRkU0GojJ4K7VuC86R2B8GSTUG6G3rag1pW++agRhdNBuHT0x6GcHsQKtkwcPd6pcPr9ATX7GkPrhevDoK8t5y9549VBdePRPec03PQMDl437luP/6AwoCNVwfh5rmg/NF4WGMNX9l08A+FibOC2isEZc6mgnVjpgc10pqxL6lpprM5Nu3qZv32Tjbs6mZbe4r/Xb2VpzfuZmufocNiEePSUw7n+MkNLN/QxkfPmUF9VXw/v/yRS8FVYgoukWGS6QmbN3uC0Ex3BCHb0xl83uu9M6jZpjv3rE93BSHeW7Pt2hGEfdeOINBqm2DDwiCs9iVWFUyVUzchqPXVNsOoiVA/MXiuWR9+rhuHAxt2dbNuaweLX9jFfcs388S6HXudLmJw+NhaLjl5KvNeMZFJo6uL+2dYIRRcJabgEqkg7nuaFrOZINA2Lgx6lbZvhl3roW1j8Ll7F3S0Bp+9z88T4rVBL9OJs4Lm3fEzYcyREEvSER/DmtZ27njyBbI554FVW1i/PejYc/T4es46ZhyvmzmOE6Y0HrLDiim4SkzBJTLCZTPBM8u2jbB7Q/DcctPSIPC2PhM0ieZrmALjZgadYo59E8w4lzW5Cfx9TQd/X7GF+Wu3kwkfmp125FjOefl4Tp4+hpdPGHXIdPhQcPU9udl5wH8BUeAmd/9qn+3XAJcDGaAVeK+7rwu3ZYEl4a7Pu/uFg11PwSVyCMtloXN70Nt0V0vQM/OFJ2DLCmh9eu99j3gNzHwTu8fP5f7tjXzktkVEjH3+7u3H75rD62aOL/49lICCK//EZlFgFfA6oAWYD1zs7svz9jkLeNzdO83sSuBMd39buK3d3euGck0Fl4j0yx1aV0DrSlj7cPDTie1rgm0WhZdfAEfNY1PNUTy8o5F/u/PpfkNs8RfPZdQI6+Ch4Mo/sdmpwBfd/fXh8rUA7v7/7WP/2cD17n56uKzgEpHicIetq6BlPjz/GKz4I3RtD7a95OcI8NfsbC5PfwIwLp47hXfMPZxXTG44+OUuAgVX/onN3gKc5+6Xh8vvBE529w/tY//rgU3u/pVwOQMsJGhG/Kq737mP464ArgCYOnXqSevW7U83XhE5pOWysG01bFwMm5fCI9/pd7fLan/Ig9v2BNaUMdWs397F2NoEv3r/KRw9of5glXjYKLjyT2z2VuD1fYJrrrt/uJ99LwU+BJzh7qlw3WHuvsHMpgN/B17r7msGuqZqXCIybHauh79+EZbevvfqc77F8nUb+Mky+HtuNofbZg6zbfwjdxwXzjqM7148uzTl3U+VGFzFHPKpBZiStzwZ2NB3JzM7B/gseaEF4O4bwvdnzewBYDYwYHCJiAyb0VPgLT8JXgC3XQIr7mb0Xz/OacBp/QzOcc7irzNt0QYOH1vDLe+dy8SGahIxDSg83IpZ44oRdM54LfACQeeMd7j7srx9ZgO3EzQpPpO3vhHodPeUmTUBjwIX5Xfs6I9qXCJSVLlsMBHqjWfsc5fp3b8gx56wev+rj+Cso8dx2suaAF4cwb9cBg+uxBpXsbvDnw98h6A7/M3u/h9mdh2wwN3vMrO/Aq8ANoaHPO/uF5rZacCPgBwQAb7j7j8Z7HoKLhEpiUd/APdeG3xO1EPPbgDOSn2LDT6WFAkmN1Zz3GENPPbcNnZ2prn+HbO54PjDSljogIKrxBRcIlIy7vDo9bDq3qDLfZ7difG09eR4Kjudq9MfJBM+pYlFjGXXvZ5krHS1LwVXiSm4RKQsZDOwcRHc8f49vxfr4/zU/2O5T3tx+TPnH8OlpxxOTWJP14NszrnjyRb+Zfakok2+qeAqMQWXiJSlzu1w+3vg2Qf63XxP9pV8Pv1uqusaOHvWdJrqklxy8lTuXbaJf/vtEv79DS/n8ldPL0rRFFwlpuASkbLXsTXoobjtmWB6l37M6f4hNdaN4azzCYyrT/Lota9lW0eK5rokdoDznuVTcJWYgktEKs7qv8E9nw5G8tiHy3r+jdHs5s3Rh/kO7+DLV17CsYcNz8gdCq4SU3CJSEXbsQ6+O/ulo9z38b3Mm2g54eP8aelGPveGmbx1zuT9roUpuEpMwSUiI0ZqN/z9K7D418EEnP24MPVlNvpYZk9p4MYPXrBfl1FwlZiCS0RGtMd+GDQr9iP3+Z37NYdYJQZXMYd8EhGR4XTKlcErlwu62d92CWxdCXDITHwJCi4RkcoTiUDTDPjQP0tdkpLQ6I8iIlJRFFwiIlJRFFwiIlJRFFwiIlJRFFwiIlJRFFwiIlJRFFwiIlJRFFwiIlJRRtSQT2bWCqzbz8ObgK3DWJxKoHse+Q61+wXd81Ad7u7Nw1mYYhtRwXUgzGxBpY3XdaB0zyPfoXa/oHs+FKipUEREKoqCS0REKoqCa48bS12AEtA9j3yH2v2C7nnE0zMuERGpKKpxiYhIRTnkg8vMzjOzlWa22sz6n1q0QpjZFDO738yeNrNlZvbRcP0YM7vPzJ4J3xvD9WZm3w3vfbGZnZh3rsvC/Z8xs8tKdU+FMLOomT1lZneHy0eY2eNh2X9tZolwfTJcXh1un5Z3jmvD9SvN7PWluZPCmdloM7vdzFaE3/epI/l7NrOPhf9NLzWzW82saiR+z2Z2s5ltMbOleeuG7Xs1s5PMbEl4zHfNrDJnn3T3Q/YFRIE1wHQgASwCZpa6XAdwPxOBE8PP9cAqYCbwdeDT4fpPA18LP58P/Bkw4BTg8XD9GODZ8L0x/NxY6vsb4L6vAX4F3B0u/w/w9vDzDcCV4eergBvCz28Hfh1+nhl+90ngiPC/iWip72uQe/5v4PLwcwIYPVK/Z2AS8BxQnff9vnskfs/Aa4ATgaV564btewX+CZwaHvNnYF6p73m//pxKXYAS/0dyKnBv3vK1wLWlLtcw3t/vgdcBK4GJ4bqJwMrw84+Ai/P2Xxluvxj4Ud76vfYrpxcwGfgbcDZwd/gXcisQ6/sdA/cCp4afY+F+1vd7z9+vHF/AqPB/5NZn/Yj8nsPgWh/+jzgWfs+vH6nfMzCtT3ANy/cabluRt36v/Srpdag3Ffb+hejVEq6reGHzyGzgcWC8u28ECN/Hhbvt6/4r6c/lO8CngFy4PBbY6e6ZcDm/7C/eV7h9V7h/Jd0vBC0ErcBPwybSm8yslhH6Pbv7C8A3geeBjQTf2xOM/O+513B9r5PCz33XV5xDPbj6a9+t+G6WZlYH/Ba42t3bBtq1n3U+wPqyYmYXAFvc/Yn81f3s6oNsq4j7zRMjaE76obvPBjoImpD2paLvO3ymcxFB895hQC0wr59dR9r3PJih3ueIuf9DPbhagCl5y5OBDSUqy7AwszhBaP3S3e8IV282s4nh9onAlnD9vu6/Uv5cTgcuNLO1wG0EzYXfAUabWSzcJ7/sL95XuL0B2E7l3G+vFqDF3R8Pl28nCLKR+j2fAzzn7q3ungbuAE5j5H/PvYbre20JP/ddX3EO9eCaD8wIeyclCB7k3lXiMu23sIfQT4Cn3f0/8zbdBfT2LLqM4NlX7/p3hb2TTgF2hU0R9wLnmllj+K/dc8N1ZcXdr3X3ye4+jeC7+7u7XwLcD7wl3K3v/fb+Obwl3N/D9W8Pe6MdAcwgeIhdltx9E7DezI4OV70WWM4I/Z4JmghPMbOa8L/x3vsd0d9znmH5XsNtu83slPDP8V1556ospX7IVuoXQc+cVQQ9jD5b6vIc4L28iqDqvxhYGL7OJ2jf/xvwTPg+JtzfgO+H974EmJN3rvcCq8PXe0p9bwXc+5ns6VU4neB/SKuB3wDJcH1VuLw63D497/jPhn8OK6mAnlbACcCC8Lu+k6D32Ij9noEvASuApcDPCXoGjrjvGbiV4DlemqCG9L7h/F6BOeGf4Rrgevp08KmUl0bOEBGRinKoNxWKiEiFUXCJiEhFUXCJiEhFUXCJiEhFUXCJiEhFUXCJ9MPM/hG+TzOzdwzzuT/T37VEpDDqDi8yADM7E/iEu18whGOi7p4dYHu7u9cNR/lEDkWqcYn0w8zaw49fBV5tZgvDOaGiZvYNM5sfzoH0f8P9z7RgLrRfEfwYFDO708yeCOeRuiJc91WgOjzfL/OvFY6A8A0L5pxaYmZvyzv3A7Zn/q1fVuw8SiLDIDb4LiKHtE+TV+MKA2iXu7/SzJLAI2b2l3DfucBx7v5cuPxed99uZtXAfDP7rbt/2sw+5O4n9HOtfyUYEWMW0BQe81C4bTZwLMHYco8QjNP4v8N/uyLlTzUukaE5l2B8uIUEU8aMJRjzDuCfeaEF8BEzWwQ8RjDo6QwG9irgVnfPuvtm4EHglXnnbnH3HMFQXtOG5W5EKpBqXCJDY8CH3X2vwWjDZ2EdfZbPIZiosNPMHiAYQ2+wc+9LKu9zFv3dlUOYalwiA9sN1Oct3wtcGU4fg5kdFU7i2FcDsCMMrWMIplbvle49vo+HgLeFz9GaCaZxr4TRy0UOKv2rTWRgi4FM2OT3M+C/CJrpngw7SLQCb+rnuHuAD5jZYoKRyB/L23YjsNjMnvRgGpZevyOYgn4RwSj/n3L3TWHwiUhI3eFFRKSiqKlQREQqioJLREQqioJLREQqioJLREQqioJLREQqioJLREQqioJLREQqioJLREQqyv8PYkQEAoFJ8AYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_tr = np.array(range(training_error_stat.shape[0]))\n",
    "x_ts = np.array(range(testing_error_stat.shape[0]))\n",
    "fig1 = plt.gcf()\n",
    "lines = plt.plot(x_tr, training_error_stat, x_ts, testing_error_stat)\n",
    "fig1.legend(('train', 'test'),loc='upper right')\n",
    "plt.xlabel('iteration')\n",
    "plt.ylabel('mean squared error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fig1.savefig('error_evolution.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}