Crossing Time Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.24.2'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.mlab as mlab\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import metrics \n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import cross_val_score, train_test_split\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "\n",
    "dataPath = '../PedestrianData/IntentionData'\n",
    "pd.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading the demographic and environmental data from pickle:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File</th>\n",
       "      <th>Scenario</th>\n",
       "      <th>Wait Time (s)</th>\n",
       "      <th>PET (s)</th>\n",
       "      <th>Distace to Collision Point</th>\n",
       "      <th>Card ID</th>\n",
       "      <th>Speed Limit</th>\n",
       "      <th>Lane Width</th>\n",
       "      <th>Minimum Gap</th>\n",
       "      <th>Mean Arrival Rate</th>\n",
       "      <th>AV</th>\n",
       "      <th>Full Braking Before Impact_-1.0</th>\n",
       "      <th>Full Braking Before Impact_1</th>\n",
       "      <th>Full Braking Before Impact_2</th>\n",
       "      <th>Full Braking Before Impact_3</th>\n",
       "      <th>Snowy</th>\n",
       "      <th>One way</th>\n",
       "      <th>two way</th>\n",
       "      <th>Two way with median</th>\n",
       "      <th>Night</th>\n",
       "      <th>name</th>\n",
       "      <th>numcars</th>\n",
       "      <th>Vrexpnum</th>\n",
       "      <th>Age_9-12</th>\n",
       "      <th>Age_15-18</th>\n",
       "      <th>Age_12-15</th>\n",
       "      <th>Age_18 - 24</th>\n",
       "      <th>Age_25 - 29</th>\n",
       "      <th>Age_30 - 39</th>\n",
       "      <th>Age_40 - 49</th>\n",
       "      <th>Age_50 - 59</th>\n",
       "      <th>Age_60+</th>\n",
       "      <th>Gender_Female</th>\n",
       "      <th>Occupation_Employed</th>\n",
       "      <th>Occupation_Student</th>\n",
       "      <th>Occupation_Unemployed</th>\n",
       "      <th>Occupation_kid</th>\n",
       "      <th>Education_Bachelors degree</th>\n",
       "      <th>Education_College/University student</th>\n",
       "      <th>Education_Doctorate degree</th>\n",
       "      <th>Education_High school diploma</th>\n",
       "      <th>Education_Masters degree</th>\n",
       "      <th>Education_Professional degree</th>\n",
       "      <th>driving license_Yes</th>\n",
       "      <th>mode_Bike</th>\n",
       "      <th>mode_Car</th>\n",
       "      <th>mode_Public Transit</th>\n",
       "      <th>mode_Walking</th>\n",
       "      <th>workwalk_No</th>\n",
       "      <th>workwalk_Sometimes</th>\n",
       "      <th>workwalk_Yes</th>\n",
       "      <th>shopwalk_No</th>\n",
       "      <th>shopwalk_Sometimes</th>\n",
       "      <th>shopwalk_Yes</th>\n",
       "      <th>shopwalk_kid</th>\n",
       "      <th>Vrexp_Yes</th>\n",
       "      <th>Heart_Currently</th>\n",
       "      <th>Heart_Over the years</th>\n",
       "      <th>vision_Currently</th>\n",
       "      <th>vision_Over the years</th>\n",
       "      <th>anxiety_Currently</th>\n",
       "      <th>anxiety_Over the years</th>\n",
       "      <th>Headaches_Currently</th>\n",
       "      <th>Headaches_Over the years</th>\n",
       "      <th>dizziness_Over the years</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3228</th>\n",
       "      <td>shabari/16--2018-09-5--17-21-20.txt</td>\n",
       "      <td>16</td>\n",
       "      <td>12.3835</td>\n",
       "      <td>1000</td>\n",
       "      <td>7.4</td>\n",
       "      <td>16</td>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>1.5</td>\n",
       "      <td>530</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>shabari</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     File Scenario Wait Time (s) PET (s)  \\\n",
       "3228  shabari/16--2018-09-5--17-21-20.txt       16       12.3835    1000   \n",
       "\n",
       "     Distace to Collision Point Card ID Speed Limit Lane Width Minimum Gap  \\\n",
       "3228                        7.4      16          40          3         1.5   \n",
       "\n",
       "     Mean Arrival Rate   AV Full Braking Before Impact_-1.0  \\\n",
       "3228               530  100                               0   \n",
       "\n",
       "     Full Braking Before Impact_1 Full Braking Before Impact_2  \\\n",
       "3228                            0                            1   \n",
       "\n",
       "     Full Braking Before Impact_3 Snowy One way two way Two way with median  \\\n",
       "3228                            0     1       1       0                   0   \n",
       "\n",
       "     Night     name numcars Vrexpnum Age_9-12 Age_15-18 Age_12-15 Age_18 - 24  \\\n",
       "3228     0  shabari       2      NaN        0         0         0           1   \n",
       "\n",
       "     Age_25 - 29 Age_30 - 39 Age_40 - 49 Age_50 - 59 Age_60+ Gender_Female  \\\n",
       "3228           0           0           0           0       0             1   \n",
       "\n",
       "     Occupation_Employed Occupation_Student Occupation_Unemployed  \\\n",
       "3228                   0                  1                     0   \n",
       "\n",
       "     Occupation_kid Education_Bachelors degree  \\\n",
       "3228              0                          0   \n",
       "\n",
       "     Education_College/University student Education_Doctorate degree  \\\n",
       "3228                                    0                          0   \n",
       "\n",
       "     Education_High school diploma Education_Masters degree  \\\n",
       "3228                             0                        1   \n",
       "\n",
       "     Education_Professional degree driving license_Yes mode_Bike mode_Car  \\\n",
       "3228                             0                   1         0        0   \n",
       "\n",
       "     mode_Public Transit mode_Walking workwalk_No workwalk_Sometimes  \\\n",
       "3228                   1            0           0                  1   \n",
       "\n",
       "     workwalk_Yes shopwalk_No shopwalk_Sometimes shopwalk_Yes shopwalk_kid  \\\n",
       "3228            0           0                  0            1            0   \n",
       "\n",
       "     Vrexp_Yes Heart_Currently Heart_Over the years vision_Currently  \\\n",
       "3228         0               0                    0                0   \n",
       "\n",
       "     vision_Over the years anxiety_Currently anxiety_Over the years  \\\n",
       "3228                     0                 0                      0   \n",
       "\n",
       "     Headaches_Currently Headaches_Over the years dizziness_Over the years  \n",
       "3228                   0                        1                        0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_pickle('../PedestrianData/Rafael.pkl')\n",
    "pd.options.display.max_columns = None\n",
    "data.sample(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing folder names from file values (some files are in subfolders)\n",
    "#### Cleasing data that contain errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3410, 65)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(data)):\n",
    "    value = data['File'].values[i].split('/')\n",
    "    data['File'].values[i] = value[len(value) - 1]\n",
    "    data['Wait Time (s)'] == \"Err2\"\n",
    "    \n",
    "data = data[data['Wait Time (s)'] != \"Err1\"]\n",
    "data = data[data['Wait Time (s)'] != \"Err2\"]\n",
    "data = data[data['Wait Time (s)'] != \"Err3\"]\n",
    "data = data[data['Wait Time (s)'] != \"Err4\"]\n",
    "data = data[data['Wait Time (s)'] != \"Err5\"]\n",
    "data = data[data['Wait Time (s)'] != 0]\n",
    "    \n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Curating final set of possible features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = os.listdir(dataPath)\n",
    "\n",
    "features = np.empty((len(files), 22), dtype=list)\n",
    "\n",
    "for i in range(len(files)):\n",
    "    file = dataPath + \"/\" + files[i]\n",
    "    df = pd.read_csv(file, sep=\",\", header=None, \n",
    "                 names=[\"Time\", \n",
    "                        \"Pos_x\", \"Pos_y\", \"Pos_z\", \n",
    "                        \"Gaz_x\", \"Gaz_y\", \"Gaz_z\", \n",
    "                        \"Vel_x\", \"Vel_z\",\n",
    "                        \"Acc_x\", \"Acc_z\",\n",
    "                        \"Vel_Mag\", \"Acc_Mag\",\n",
    "                        \"Vel_Ang\", \"Acc_Ang\", \"Gaz_Ang\",\n",
    "                        \"Vel_Bin\", \"Acc_Bin\", \"Gaz_Bin\", \"Chg_Bin\"          \n",
    "                       ])\n",
    "    \n",
    "    try:\n",
    "        features[i, 0] = (data.loc[data['File'] == files[i]])[\"Wait Time (s)\"].item()\n",
    "        features[i, 1] = (data.loc[data['File'] == files[i]])[\"Lane Width\"].item()\n",
    "        features[i, 2] = (data.loc[data['File'] == files[i]])[\"One way\"].item()\n",
    "        features[i, 3] = (data.loc[data['File'] == files[i]])[\"Snowy\"].item()\n",
    "        features[i, 4] = (data.loc[data['File'] == files[i]])[\"Night\"].item()\n",
    "        features[i, 5] = (data.loc[data['File'] == files[i]])[\"Speed Limit\"].item()\n",
    "        features[i, 6] = (data.loc[data['File'] == files[i]])[\"Mean Arrival Rate\"].item()\n",
    "    except:\n",
    "        features[i, 0] = 0\n",
    "        features[i, 1] = 0\n",
    "        features[i, 2] = 0\n",
    "        features[i, 3] = 0\n",
    "        features[i, 4] = 0\n",
    "        features[i, 5] = 0\n",
    "        features[i, 6] = 0\n",
    "    finally:\n",
    "        # Filling the rest of the features with each direction bin\n",
    "        gaze_bins = np.histogram(np.array(df[\"Gaz_Bin\"]), bins=[1, 2, 3, 4, 5, 6, 7, 8], density=False)[0]\n",
    "        change_bins = np.histogram(np.array(df[\"Chg_Bin\"]), bins=[1, 2, 3, 4, 5, 6, 7, 8], density=False)[0]\n",
    "        for j in range(len(gaze_bins)):\n",
    "            features[i, j+7] = gaze_bins[j]\n",
    "        for k in range(len(change_bins)):\n",
    "            features[i, k+j+7] = change_bins[k]\n",
    "            \n",
    "# Removing data with crossing time of '0'\n",
    "features = features[features[:, 0] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11.39143, 2.75, 0.0, ..., 1, None, None],\n",
       "       [14.03475, 2.75, 0.0, ..., 0, None, None],\n",
       "       [14.8416, 2.75, 0.0, ..., 4, None, None],\n",
       "       ...,\n",
       "       [8.032308, 2.75, 0.0, ..., 9, None, None],\n",
       "       [9.176275, 2.75, 0.0, ..., 2, None, None],\n",
       "       [11.65396, 2.75, 0.0, ..., 5, None, None]], dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing regression models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getData(N):\n",
    "    x,y =[],[]\n",
    "    for i in range(N):\n",
    "        c = [\n",
    "            \n",
    "            features[i,1], # Lane width (2, 2.5, 3)\n",
    "            features[i,2], # One Way (0, 1)\n",
    "            features[i,3], # Snowy (0, 1)\n",
    "            features[i,4], # Night (0, 1)\n",
    "            features[i,5], # Speed Limit (30, 40, 50)\n",
    "            features[i,6], # Mean Arrival\n",
    "            # head pose bins\n",
    "            features[i,7], features[i,8], features[i,9], features[i,10], features[i,11], features[i,12], features[i,13], \n",
    "            # change pose bins\n",
    "            #features[i,14], features[i,15], features[i,16], features[i,17], features[i,18], features[i,19]\n",
    "            \n",
    "        ]\n",
    "        yy = features[i, 0]\n",
    "        x.append(c)\n",
    "        y.append(yy)\n",
    "    return np.array(x), np.array(y)\n",
    "\n",
    "def display_scores(scores):\n",
    "    print(\"Scores:\", scores)\n",
    "    print(\"Mean:\", scores.mean())\n",
    "    print(\"Standard deviation:\", scores.std())\n",
    "    \n",
    "def evaluate(model, X_test, y_test):\n",
    "    predictions = model.predict(X_test)\n",
    "    errors = abs(predictions - y_test)\n",
    "    mape = 100 * np.mean(errors / y_test)\n",
    "    accuracy = 100 - mape\n",
    "    print('Model Performance')\n",
    "    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    return accuracy\n",
    "\n",
    "def evaluate_mse(model, X_test, y_test):\n",
    "    predictions = model.predict(X_test)\n",
    "    errors = np.sqrt(mean_squared_error(y_test, predictions))\n",
    "    print('Model Performance')\n",
    "    print('MSE of: ', errors)\n",
    "    \n",
    "    return errors\n",
    "    \n",
    "X,y=getData(len(features))\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1174, 13)\n",
      "(1174,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "X_train.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1174, 13)\n",
      "(1174,)\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'float' object has no attribute 'dtype'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-36-59ec30a12ced>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdatum\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdatum\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-36-59ec30a12ced>\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdatum\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdatum\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'float' object has no attribute 'dtype'"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "X_train.dtype\n",
    "np.unique([str(datum.dtype) for datum in features[:,1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    6.5s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:   38.7s\n",
      "[Parallel(n_jobs=-1)]: Done 341 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.1min finished\n",
      "C:\\Users\\Rafael\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, error_score='raise-deprecating',\n",
       "          estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators='warn', n_jobs=None,\n",
       "           oob_score=False, random_state=None, verbose=0, warm_start=False),\n",
       "          fit_params=None, iid='warn', n_iter=100, n_jobs=-1,\n",
       "          param_distributions={'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]},\n",
       "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
       "          return_train_score='warn', scoring=None, verbose=2)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the param grid\n",
    "param_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "\n",
    "# Use the random grid to search for best hyperparameters\n",
    "# First create the base model to tune\n",
    "rf = RandomForestRegressor()\n",
    "\n",
    "# Random search of parameters, using 3 fold cross validation, \n",
    "# search across 100 different combinations, and use all available cores\n",
    "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = param_grid, n_iter = 100, cv = 5, verbose=2, n_jobs = -1)\n",
    "\n",
    "# Fit the random search model\n",
    "rf_random.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': 200, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'auto', 'max_depth': 30, 'bootstrap': True}\n"
     ]
    }
   ],
   "source": [
    "print(rf_random.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Average Error: 3.0621 degrees.\n",
      "Accuracy = 81.99%.\n",
      "Model Performance\n",
      "MSE of:  5.755589138398937\n",
      "Model Performance\n",
      "Average Error: 3.0068 degrees.\n",
      "Accuracy = 82.40%.\n",
      "Model Performance\n",
      "MSE of:  5.616809676983853\n",
      "\n",
      "\n",
      "Accuracy Improvement of 0.49%.\n",
      "MSE Improvement of 2.41%.\n"
     ]
    }
   ],
   "source": [
    "base_model = RandomForestRegressor(bootstrap = True, n_estimators = 100)\n",
    "base_model.fit(X_train, y_train)\n",
    "base_accuracy = evaluate(base_model, X_test, y_test)\n",
    "base_mse = evaluate_mse(base_model, X_test, y_test)\n",
    "\n",
    "best_random = rf_random.best_estimator_\n",
    "random_accuracy = evaluate(best_random, X_test, y_test)\n",
    "random_mse = evaluate_mse(best_random, X_test, y_test)\n",
    "\n",
    "print('\\n')\n",
    "print('Accuracy Improvement of {:0.2f}%.'.format( 100 * (random_accuracy - base_accuracy) / base_accuracy))\n",
    "print('MSE Improvement of {:0.2f}%.'.format(-100 * (random_mse - base_mse) / base_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rf_random' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-0f937899a9bc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mforest_reg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrf_random\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_estimator_\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mforest_reg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mforest_reg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'rf_random' is not defined"
     ]
    }
   ],
   "source": [
    "forest_reg = rf_random.best_estimator_\n",
    "forest_reg.fit(X_train, y_train)\n",
    "\n",
    "y_pred = forest_reg.predict(X_test)  \n",
    "\n",
    "print('R^2:', metrics.r2_score(y_test,y_pred))\n",
    "print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  \n",
    "print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  \n",
    "print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))\n",
    "\n",
    "scores = cross_val_score(forest_reg, X, y, cv=10)\n",
    "display_scores(scores)\n",
    "\n",
    "x_ax=range(len(y_pred))\n",
    "plt.figure(figsize=(20, 10)) \n",
    "plt.scatter(x_ax, y_test, s=5, color=\"green\", label=\"original\")\n",
    "plt.scatter(x_ax, y_pred, s=5, color=\"red\", label=\"predicted\")\n",
    "    \n",
    "plt.xlabel(\"Trial #\")\n",
    "plt.ylabel(\"Time of Cross (s)\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[START] Pre-training step:\n",
      ">> Epoch 1 finished \tRBM Reconstruction error 94936174309366.562500\n",
      ">> Epoch 2 finished \tRBM Reconstruction error 112726286074821.750000\n",
      ">> Epoch 3 finished \tRBM Reconstruction error 91363288692093.625000\n",
      ">> Epoch 4 finished \tRBM Reconstruction error 98775054571684.609375\n",
      ">> Epoch 5 finished \tRBM Reconstruction error 107875034495918.703125\n",
      ">> Epoch 6 finished \tRBM Reconstruction error 85931213038025.734375\n",
      ">> Epoch 7 finished \tRBM Reconstruction error 82632691726962.859375\n",
      ">> Epoch 8 finished \tRBM Reconstruction error 97621635649013.718750\n",
      ">> Epoch 9 finished \tRBM Reconstruction error 101799952005274.046875\n",
      ">> Epoch 10 finished \tRBM Reconstruction error 83870807677451.593750\n",
      ">> Epoch 11 finished \tRBM Reconstruction error 89312313269159.828125\n",
      ">> Epoch 12 finished \tRBM Reconstruction error 131140743738070.890625\n",
      ">> Epoch 13 finished \tRBM Reconstruction error 104962825247358.875000\n",
      ">> Epoch 14 finished \tRBM Reconstruction error 95033037151846.937500\n",
      ">> Epoch 15 finished \tRBM Reconstruction error 102804412752036.578125\n",
      ">> Epoch 16 finished \tRBM Reconstruction error 118376512502304.796875\n",
      ">> Epoch 17 finished \tRBM Reconstruction error 103107682303485.000000\n",
      ">> Epoch 18 finished \tRBM Reconstruction error 118183141552946.375000\n",
      ">> Epoch 19 finished \tRBM Reconstruction error 107300811767064.625000\n",
      ">> Epoch 20 finished \tRBM Reconstruction error 113737998256385.578125\n",
      "[END] Pre-training step\n",
      "[START] Fine tuning step:\n",
      ">> Epoch 0 finished \tANN training loss 345849114774208512.000000\n",
      ">> Epoch 1 finished \tANN training loss 17391821525090304.000000\n",
      ">> Epoch 2 finished \tANN training loss 874590700044288.000000\n",
      ">> Epoch 3 finished \tANN training loss 43980897124352.000000\n",
      ">> Epoch 4 finished \tANN training loss 2211682975744.000000\n",
      ">> Epoch 5 finished \tANN training loss 111219712000.000000\n",
      ">> Epoch 6 finished \tANN training loss 5592941056.000000\n",
      ">> Epoch 7 finished \tANN training loss 281254784.000000\n",
      ">> Epoch 8 finished \tANN training loss 14143717.000000\n",
      ">> Epoch 9 finished \tANN training loss 711414.375000\n",
      ">> Epoch 10 finished \tANN training loss 35831.011719\n",
      ">> Epoch 11 finished \tANN training loss 1927.684570\n",
      ">> Epoch 12 finished \tANN training loss 225.365631\n",
      ">> Epoch 13 finished \tANN training loss 137.778107\n",
      ">> Epoch 14 finished \tANN training loss 132.941315\n",
      ">> Epoch 15 finished \tANN training loss 132.672684\n",
      ">> Epoch 16 finished \tANN training loss 132.659637\n",
      ">> Epoch 17 finished \tANN training loss 132.659042\n",
      ">> Epoch 18 finished \tANN training loss 132.666504\n",
      ">> Epoch 19 finished \tANN training loss 132.664719\n",
      ">> Epoch 20 finished \tANN training loss 132.694992\n",
      ">> Epoch 21 finished \tANN training loss 132.659744\n",
      ">> Epoch 22 finished \tANN training loss 132.705688\n",
      ">> Epoch 23 finished \tANN training loss 132.669983\n",
      ">> Epoch 24 finished \tANN training loss 132.687469\n",
      ">> Epoch 25 finished \tANN training loss 132.697372\n",
      ">> Epoch 26 finished \tANN training loss 132.684875\n",
      ">> Epoch 27 finished \tANN training loss 132.688660\n",
      ">> Epoch 28 finished \tANN training loss 132.742905\n",
      ">> Epoch 29 finished \tANN training loss 132.694305\n",
      ">> Epoch 30 finished \tANN training loss 132.670273\n",
      ">> Epoch 31 finished \tANN training loss 132.659393\n",
      ">> Epoch 32 finished \tANN training loss 132.665131\n",
      ">> Epoch 33 finished \tANN training loss 132.658829\n",
      ">> Epoch 34 finished \tANN training loss 132.672287\n",
      ">> Epoch 35 finished \tANN training loss 132.673950\n",
      ">> Epoch 36 finished \tANN training loss 132.668884\n",
      ">> Epoch 37 finished \tANN training loss 132.752991\n",
      ">> Epoch 38 finished \tANN training loss 132.659637\n",
      ">> Epoch 39 finished \tANN training loss 132.674652\n",
      ">> Epoch 40 finished \tANN training loss 132.658905\n",
      ">> Epoch 41 finished \tANN training loss 132.692078\n",
      ">> Epoch 42 finished \tANN training loss 132.672165\n",
      ">> Epoch 43 finished \tANN training loss 132.672745\n",
      ">> Epoch 44 finished \tANN training loss 132.660019\n",
      ">> Epoch 45 finished \tANN training loss 132.663071\n",
      ">> Epoch 46 finished \tANN training loss 132.666458\n",
      ">> Epoch 47 finished \tANN training loss 132.659760\n",
      ">> Epoch 48 finished \tANN training loss 132.659866\n",
      ">> Epoch 49 finished \tANN training loss 132.664062\n",
      ">> Epoch 50 finished \tANN training loss 132.721970\n",
      ">> Epoch 51 finished \tANN training loss 132.659134\n",
      ">> Epoch 52 finished \tANN training loss 132.675781\n",
      ">> Epoch 53 finished \tANN training loss 132.702209\n",
      ">> Epoch 54 finished \tANN training loss 132.660645\n",
      ">> Epoch 55 finished \tANN training loss 132.662460\n",
      ">> Epoch 56 finished \tANN training loss 132.658829\n",
      ">> Epoch 57 finished \tANN training loss 132.658798\n",
      ">> Epoch 58 finished \tANN training loss 132.660858\n",
      ">> Epoch 59 finished \tANN training loss 132.658813\n",
      ">> Epoch 60 finished \tANN training loss 132.659195\n",
      ">> Epoch 61 finished \tANN training loss 132.694473\n",
      ">> Epoch 62 finished \tANN training loss 132.662354\n",
      ">> Epoch 63 finished \tANN training loss 132.663574\n",
      ">> Epoch 64 finished \tANN training loss 132.661591\n",
      ">> Epoch 65 finished \tANN training loss 132.662903\n",
      ">> Epoch 66 finished \tANN training loss 132.675156\n",
      ">> Epoch 67 finished \tANN training loss 132.660995\n",
      ">> Epoch 68 finished \tANN training loss 132.789658\n",
      ">> Epoch 69 finished \tANN training loss 132.685257\n",
      ">> Epoch 70 finished \tANN training loss 132.661118\n",
      ">> Epoch 71 finished \tANN training loss 132.667633\n",
      ">> Epoch 72 finished \tANN training loss 132.659515\n",
      ">> Epoch 73 finished \tANN training loss 132.708496\n",
      ">> Epoch 74 finished \tANN training loss 132.659866\n",
      ">> Epoch 75 finished \tANN training loss 132.659348\n",
      ">> Epoch 76 finished \tANN training loss 132.660645\n",
      ">> Epoch 77 finished \tANN training loss 132.675522\n",
      ">> Epoch 78 finished \tANN training loss 132.665451\n",
      ">> Epoch 79 finished \tANN training loss 132.771698\n",
      ">> Epoch 80 finished \tANN training loss 132.659149\n",
      ">> Epoch 81 finished \tANN training loss 132.660309\n",
      ">> Epoch 82 finished \tANN training loss 132.658829\n",
      ">> Epoch 83 finished \tANN training loss 132.722992\n",
      ">> Epoch 84 finished \tANN training loss 132.658768\n",
      ">> Epoch 85 finished \tANN training loss 132.658981\n",
      ">> Epoch 86 finished \tANN training loss 132.663589\n",
      ">> Epoch 87 finished \tANN training loss 132.671173\n",
      ">> Epoch 88 finished \tANN training loss 132.669388\n",
      ">> Epoch 89 finished \tANN training loss 132.685715\n",
      ">> Epoch 90 finished \tANN training loss 132.667847\n",
      ">> Epoch 91 finished \tANN training loss 132.660797\n",
      ">> Epoch 92 finished \tANN training loss 132.670105\n",
      ">> Epoch 93 finished \tANN training loss 132.692230\n",
      ">> Epoch 94 finished \tANN training loss 132.703400\n",
      ">> Epoch 95 finished \tANN training loss 132.667633\n",
      ">> Epoch 96 finished \tANN training loss 132.690598\n",
      ">> Epoch 97 finished \tANN training loss 132.673050\n",
      ">> Epoch 98 finished \tANN training loss 132.696854\n",
      ">> Epoch 99 finished \tANN training loss 132.665878\n",
      ">> Epoch 100 finished \tANN training loss 132.690186\n",
      ">> Epoch 101 finished \tANN training loss 132.660568\n",
      ">> Epoch 102 finished \tANN training loss 132.659744\n",
      ">> Epoch 103 finished \tANN training loss 132.658813\n",
      ">> Epoch 104 finished \tANN training loss 132.726334\n",
      ">> Epoch 105 finished \tANN training loss 132.658798\n",
      ">> Epoch 106 finished \tANN training loss 132.668747\n",
      ">> Epoch 107 finished \tANN training loss 132.666275\n",
      ">> Epoch 108 finished \tANN training loss 132.663483\n",
      ">> Epoch 109 finished \tANN training loss 132.687714\n",
      ">> Epoch 110 finished \tANN training loss 132.688843\n",
      ">> Epoch 111 finished \tANN training loss 132.667709\n",
      ">> Epoch 112 finished \tANN training loss 132.659073\n",
      ">> Epoch 113 finished \tANN training loss 132.665619\n",
      ">> Epoch 114 finished \tANN training loss 132.659210\n",
      ">> Epoch 115 finished \tANN training loss 132.701355\n",
      ">> Epoch 116 finished \tANN training loss 132.697449\n",
      ">> Epoch 117 finished \tANN training loss 132.674316\n",
      ">> Epoch 118 finished \tANN training loss 132.676422\n",
      ">> Epoch 119 finished \tANN training loss 132.659073\n",
      ">> Epoch 120 finished \tANN training loss 132.660156\n",
      ">> Epoch 121 finished \tANN training loss 132.659714\n",
      ">> Epoch 122 finished \tANN training loss 132.703720\n",
      ">> Epoch 123 finished \tANN training loss 132.658798\n",
      ">> Epoch 124 finished \tANN training loss 132.682037\n",
      ">> Epoch 125 finished \tANN training loss 132.659317\n",
      ">> Epoch 126 finished \tANN training loss 132.668182\n",
      ">> Epoch 127 finished \tANN training loss 132.667313\n",
      ">> Epoch 128 finished \tANN training loss 132.658798\n",
      ">> Epoch 129 finished \tANN training loss 132.659164\n",
      ">> Epoch 130 finished \tANN training loss 132.670029\n",
      ">> Epoch 131 finished \tANN training loss 132.672974\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Epoch 132 finished \tANN training loss 132.659210\n",
      ">> Epoch 133 finished \tANN training loss 132.677505\n",
      ">> Epoch 134 finished \tANN training loss 132.677917\n",
      ">> Epoch 135 finished \tANN training loss 132.669189\n",
      ">> Epoch 136 finished \tANN training loss 132.664734\n",
      ">> Epoch 137 finished \tANN training loss 132.661926\n",
      ">> Epoch 138 finished \tANN training loss 132.700607\n",
      ">> Epoch 139 finished \tANN training loss 132.691620\n",
      ">> Epoch 140 finished \tANN training loss 132.660248\n",
      ">> Epoch 141 finished \tANN training loss 132.658783\n",
      ">> Epoch 142 finished \tANN training loss 132.659332\n",
      ">> Epoch 143 finished \tANN training loss 132.682022\n",
      ">> Epoch 144 finished \tANN training loss 132.679169\n",
      ">> Epoch 145 finished \tANN training loss 132.659698\n",
      ">> Epoch 146 finished \tANN training loss 132.669220\n",
      ">> Epoch 147 finished \tANN training loss 132.686386\n",
      ">> Epoch 148 finished \tANN training loss 132.691895\n",
      ">> Epoch 149 finished \tANN training loss 132.659882\n",
      ">> Epoch 150 finished \tANN training loss 132.693954\n",
      ">> Epoch 151 finished \tANN training loss 132.693634\n",
      ">> Epoch 152 finished \tANN training loss 132.658768\n",
      ">> Epoch 153 finished \tANN training loss 132.660599\n",
      ">> Epoch 154 finished \tANN training loss 132.668167\n",
      ">> Epoch 155 finished \tANN training loss 132.758484\n",
      ">> Epoch 156 finished \tANN training loss 132.715408\n",
      ">> Epoch 157 finished \tANN training loss 132.667252\n",
      ">> Epoch 158 finished \tANN training loss 132.715683\n",
      ">> Epoch 159 finished \tANN training loss 132.658875\n",
      ">> Epoch 160 finished \tANN training loss 132.661865\n",
      ">> Epoch 161 finished \tANN training loss 132.664062\n",
      ">> Epoch 162 finished \tANN training loss 132.681900\n",
      ">> Epoch 163 finished \tANN training loss 132.658783\n",
      ">> Epoch 164 finished \tANN training loss 132.680420\n",
      ">> Epoch 165 finished \tANN training loss 132.676788\n",
      ">> Epoch 166 finished \tANN training loss 132.664322\n",
      ">> Epoch 167 finished \tANN training loss 132.659134\n",
      ">> Epoch 168 finished \tANN training loss 132.665009\n",
      ">> Epoch 169 finished \tANN training loss 132.658890\n",
      ">> Epoch 170 finished \tANN training loss 132.667557\n",
      ">> Epoch 171 finished \tANN training loss 132.658798\n",
      ">> Epoch 172 finished \tANN training loss 132.778885\n",
      ">> Epoch 173 finished \tANN training loss 132.669327\n",
      ">> Epoch 174 finished \tANN training loss 132.667450\n",
      ">> Epoch 175 finished \tANN training loss 132.662521\n",
      ">> Epoch 176 finished \tANN training loss 132.659668\n",
      ">> Epoch 177 finished \tANN training loss 132.664673\n",
      ">> Epoch 178 finished \tANN training loss 132.662964\n",
      ">> Epoch 179 finished \tANN training loss 132.663300\n",
      ">> Epoch 180 finished \tANN training loss 132.658783\n",
      ">> Epoch 181 finished \tANN training loss 132.666199\n",
      ">> Epoch 182 finished \tANN training loss 132.659134\n",
      ">> Epoch 183 finished \tANN training loss 132.675873\n",
      ">> Epoch 184 finished \tANN training loss 132.659836\n",
      ">> Epoch 185 finished \tANN training loss 132.674454\n",
      ">> Epoch 186 finished \tANN training loss 132.668213\n",
      ">> Epoch 187 finished \tANN training loss 132.715744\n",
      ">> Epoch 188 finished \tANN training loss 132.717819\n",
      ">> Epoch 189 finished \tANN training loss 132.672577\n",
      ">> Epoch 190 finished \tANN training loss 132.670059\n",
      ">> Epoch 191 finished \tANN training loss 132.668289\n",
      ">> Epoch 192 finished \tANN training loss 132.691711\n",
      ">> Epoch 193 finished \tANN training loss 132.707443\n",
      ">> Epoch 194 finished \tANN training loss 132.681381\n",
      ">> Epoch 195 finished \tANN training loss 132.669769\n",
      ">> Epoch 196 finished \tANN training loss 132.659134\n",
      ">> Epoch 197 finished \tANN training loss 132.665466\n",
      ">> Epoch 198 finished \tANN training loss 132.663162\n",
      ">> Epoch 199 finished \tANN training loss 132.661453\n",
      "[END] Fine tuning step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SupervisedDBNRegression(batch_size=16, dropout_p=0, l2_regularization=1.0,\n",
       "                        learning_rate=0.01, n_iter_backprop=200, verbose=True)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from dbn.tensorflow import SupervisedDBNRegression\n",
    "# Training\n",
    "regressor = SupervisedDBNRegression(hidden_layers_structure=[100],\n",
    "                                    learning_rate_rbm=0.01,\n",
    "                                    learning_rate=0.01,\n",
    "                                    n_epochs_rbm=20,\n",
    "                                    n_iter_backprop=200,\n",
    "                                    batch_size=16,\n",
    "                                    activation_function='relu')\n",
    "regressor.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABI8AAAJQCAYAAAAHTUTCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3X+YZXddJ/j3t1JopJvfSbVCtUl0YcFBfoSCwUoI3XFUVARl0IERbPwVRWcQ5tmVWZ912EH3eWaeZRNcV2cmDBuaHUBHRBlGV1hmKyNQytAhEBkCooJ0T0h7+RXpzhNI9f3uH12VdHfqVN2quufec+99vZ6nn+qqe+vW555zz/l+z/t8z/eUWmsAAAAAYDNz4y4AAAAAgO4SHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0mh93AYO45JJL6uWXXz7uMgAAAACmxi233PL5Wuul2z1vIsKjyy+/PMeOHRt3GQAAAABTo5Ty14M8z2VrAAAAADQSHgEAAADQSHgEAAAAQKOJmPMIAAAAmC333ntvTpw4kXvuuWfcpUy8iy++OIuLi3nQgx60q98XHgEAAACdc+LEiTzkIQ/J5ZdfnlLKuMuZWLXWfOELX8iJEydyxRVX7Oo1XLYGAAAAdM4999yTRz3qUYKjPSql5FGPetSeRnAJjwAAAIBOEhwNx16Xo/AIAAAAgEbCIwAAAIA9+L7v+758+ctf3vI5/+yf/bO8973v3dXr33zzzXnuc5+7q98dBhNmAwAAAOxCrTW11vzhH/7hts997WtfO4KK2mHkEQAAAECD66+/Pk984hPzxCc+Ma9//evzmc98Jk94whPycz/3c7nyyitz/PjxXH755fn85z+fJPmVX/mVPP7xj893fdd35cUvfnFe97rXJUle9rKX5e1vf3uS5PLLL89rXvOaXHnllfn2b//2fOITn0iS/Jf/8l+yvLycpz71qVleXs4nP/nJ8bzpCwiPAAAAgKnQr/2cPHUytdahvN4tt9ySm266KR/84Afzp3/6p3nDG96QL33pS/nkJz+ZH/uxH8utt96ayy677L7nHzt2LL/7u7+bW2+9Ne94xzty7Nixxte+5JJL8uEPfzgvf/nL7wuYHv/4x+eP//iPc+utt+a1r31tfumXfmko72OvXLYGAAAATLx+7efw0cNZPb6a5YPLWTmykrmytzEz73//+/NDP/RD2bdvX5LkBS94Qd73vvflsssuyzOf+cxNn//85z8/3/AN35Ak+YEf+IHG137BC16QJHna056Wd7zjHUmSu+66K0eOHMmnPvWplFJy77337qn+YTHyCAAAAJh4vdO9rB5fzVp/LavHV9M73dvzazaNYNoIkwZ9/ma+/uu/Pkly0UUXZW1tLUnyy7/8yzl8+HA+9rGP5V3velfuueeeHVbcDuERAAAAMPEW9i1k+eBy5ufms3xwOQv7Fvb8mtdcc01+//d/P3fffXdOnz6d3/u938uznvWsxudfffXV94U+p06dyh/8wR/s6O/dddddecxjHpMkedOb3rSX0ofKZWsAAADAxCulZOXISnqne1nYt5BSyp5f88orr8zLXvayPOMZz0iS/NRP/VQe8YhHND7/6U9/ep73vOflyU9+ci677LIsLS3lYQ972MB/7xd/8Rdz5MiRXH/99bn22mv3XP+wlGFNItWmpaWlutUkUwAAAMB0uf322/OEJzxh3GXs2KlTp7J///7cfffdueaaa3LjjTfmyiuvHHdZmy7PUsottdal7X7XyCMAAACAIbnuuuvy8Y9/PPfcc0+OHDnSieBor4RHAAAAAEPy1re+ddwlDJ0JswEAAABoJDwCAAAAoJHwCGDM+rWfk6dOZhJuYAAAAMwe4RHAGPVrP4ePHs7iDYs5dPRQ+rU/7pIAAADOIzwCGKPe6V5Wj69mrb+W1eOr6Z3ujbskAACgJfv370+S3HHHHXnhC1+45XNf//rX5+67797R699888157nOfu+v6mgiPAMZoYd9Clg8uZ35uPssHl7Owb2HcJQEAADtw5syZHf/Oox/96Lz97W/f8jm7CY/aMj/uAgBmWSklK0dW0jvdy8K+hZRSxl0SAACw7jOf+Uye85zn5O/+3b+bW2+9NY973OPy5je/Od/2bd+Wn/iJn8h73vOe/KN/9I/y9Kc/PT//8z+fXq+XBz/4wXnDG96Qxz/+8fn0pz+df/gP/2HW1tbynOc857zXfe5zn5uPfexjOXPmTF796lfn3e9+d0op+emf/unUWnPHHXfk8OHDueSSS7KyspL3vOc9ec1rXpOvfvWr+dZv/dbcdNNN2b9/f/7oj/4or3zlK3PJJZfkyiuvbGU5GHkEMGZzZS4H9h8QHAEAwF71+8nJk8kQb0bzyU9+Mtddd11uu+22PPShD81v/uZvJkkuvvjivP/978+LXvSiXHfddfn1X//13HLLLXnd616Xn/u5n0uS/MIv/EJe/vKX50Mf+lC+8Ru/cdPXv/HGG/PpT386t956a2677bb86I/+aF7xilfk0Y9+dFZWVrKyspLPf/7z+dVf/dW8973vzYc//OEsLS3l+uuvzz333JOf/umfzrve9a68733vy5133jm0930u4REAAAAw+fr95PDhZHExOXTo7PdDcPDgwVx11VVJkpe85CV5//vfnyT5B//gHyRJTp06ldXV1fzwD/9wnvKUp+RnfuZn8rnPfS5J8oEPfCAvfvGLkyQvfelLN3399773vfnZn/3ZzM+fvTjskY985AOe86d/+qf5+Mc/nquuuipPecpTcvTo0fz1X/91PvGJT+SKK67IYx/72JRS8pKXvGQo7/lCLlsDAAAAJl+vl6yuJmtrZ7/2esmBA3t+2QuvENj4ft++fUmSfr+fhz/84fnIRz4y0O9fqNY60HO+67u+K29729vO+/lHPvKRkVzBYOQRAAAAMPkWFpLl5WR+/uzXheHcjOazn/1s/uRP/iRJ8ra3vS1XX331eY8/9KEPzRVXXJHf+Z3fSXI26PnoRz+aJLnqqqvyW7/1W0mSt7zlLZu+/nd/93fnX//rf521tbUkyRe/+MUkyUMe8pB85StfSZI885nPzAc+8IH8xV/8RZLk7rvvzp//+Z/fN6/SX/7lX95XXxuERwAAAMDkKyVZWUlOnEhuvvns90PwhCc8IUePHs2TnvSkfPGLX8zLX/7yBzznLW95S974xjfmyU9+cv7O3/k7eec735kk+bVf+7X8xm/8Rp7+9Kfnrrvu2vT1f+qnfirf/M3fnCc96Ul58pOfnLe+9a1Jkuuuuy7f+73fm8OHD+fSSy/Nm970prz4xS/Ok570pDzzmc/MJz7xiVx88cW58cYb8/3f//25+uqrc9lllw3lPV+o1CFOItWWpaWleuzYsXGXAQAAAIzI7bffnic84QljreHcu6JNus2WZynlllrr0na/a+QRAAAAAI2ERwAAAACbuPzyy6di1NFeCY8AAACATpqEqXYmwV6Xo/AIAAAA6JyLL744X/jCFwRIe1RrzRe+8IVcfPHFu36N+SHWAwAAADAUi4uLOXHiRHq93rhLmXgXX3xxFhcXd/37wiMAAACgcx70oAfliiuuGHcZxGVrAAAAAGxBeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjVoNj0opryql/NdSysdKKW8rpVxcSrmilPLBUsqnSim/XUr5ujZrAAAAAGD3WguPSimPSfKKJEu11icmuSjJi5L8yyQ31Fofm+RLSX6yrRoAAAAA2Ju2L1ubT/INpZT5JA9O8rkk1yZ5+/rjR5P8YMs1AAAAALBLrYVHtdb/luR1ST6bs6HRXUluSfLlWuva+tNOJHnMZr9fSrmulHKslHKs1+u1VSYAAAAAW2jzsrVHJHl+kiuSPDrJviTfu8lT62a/X2u9sda6VGtduvTSS9sqEwAAAIAttHnZ2t9L8ulaa6/Wem+SdyRZTvLw9cvYkmQxyR0t1gAAAADAHrQZHn02yTNLKQ8upZQk35nk40lWkrxw/TlHkryzxRoAAAAA2IM25zz6YM5OjP3hJH+2/rduTPLqJP+klPIXSR6V5I1t1QAAAADA3sxv/5Tdq7W+JslrLvjxXyV5Rpt/FwAAAIDhaPOyNQAAAAAmnPAIAAAAgEbCIwAAAAAaCY8AAAAAaCQ8AgAAAKCR8AgAAACARsIjAAAAABoJjwAAAABoJDwCAAAAoJHwCAAAAIBGwiMAAAAAGgmPAAAAAGgkPAIAAACgkfAIAAAAgEbCIwAAAAAaCY8AAAAAaCQ8AgAAAKCR8AgAAACARsIjAAAAABoJjwAAAABoJDwCAAAAoJHwCAAAAIBGwiMAAAAAGgmPAAAAAGgkPAIAAACgkfAIAAAAgEbCIwAAAAAaCY8AAAAAaCQ8AgAAAKCR8AgAAACARsIjAAAAABoJjwAAAABoJDwCAGZKv/Zz8tTJ1FrHXQoAwEQQHgEAM6Nf+zl89HAWb1jMoaOH0q/9cZcEANB5wiMAYGb0Tveyenw1a/21rB5fTe90b9wlAQB0nvAIAJgZC/sWsnxwOfNz81k+uJyFfQvjLgkAoPPmx10AAMColFKycmQlvdO9LOxbSCll3CUBAHSekUd0mklNARi2uTKXA/sPCI4AAAYkPKKzTGoKAAAA4yc8orNMagoAAADjJzyis0xqCgAAAONnwmw6y6SmAAAAMH7CIzptY1JTAAAAYDxctgYAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI1aDY9KKQ8vpby9lPKJUsrtpZTvKKU8spTy/5ZSPrX+9RFt1gAAAADA7rU98ujXkvxRrfXxSZ6c5PYk/zTJf6q1PjbJf1r/HgAAAIAOai08KqU8NMk1Sd6YJLXWr9Vav5zk+UmOrj/taJIfbKsGAAAAAPamzZFH35Kkl+SmUsqtpZR/W0rZl+RArfVzSbL+dWGzXy6lXFdKOVZKOdbr9VosEwAAAIAmbYZH80muTPKvaq1PTXI6O7hErdZ6Y611qda6dOmll7ZVIwAAAABbaDM8OpHkRK31g+vfvz1nw6STpZRvSpL1r3/TYg0AAAAA7EFr4VGt9c4kx0sp//36j74zyceT/IckR9Z/diTJO9uqAQAAAIC9mW/59f9xkreUUr4uyV8l+fGcDaz+fSnlJ5N8NskPt1wDAAAAALvUanhUa/1IkqVNHvrONv8uAAAAAMPR5pxHAAAAAEw44REAAAAAjYRHAAAAADQSHgEAAADQSHgEAAAAQCPhEQAAAACNhEcAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAAAA0Eh4BAAAA0Eh4BAAAAEAj4RHAGPRrPydPnUytddylAAAAbGmg8KiUMldKeWop5ftLKdeWUg60XRiwN8KJ7urXfg4fPZzFGxZz6Oih9Gt/3CUBAAA0mt/qwVLKtyZ5dZK/l+RTSXpJLk7yuFLK3Un+TZKjtTrygS7ZCCdWj69m+eByVo6sZK4YaNgVvdO9rB5fzVp/LavHV9M73cuB/TJ5AACgm7Y7mvzVJP8uybfWWr+n1vqSWusLa61PSvK8JA9L8tK2iwR2ZrNwgu5Y2LeQ5YPLmZ+bz/LB5SzsWxh3SQAAAI22HHlUa33xFo/9TZLXD70iYM82womNkUfCiW4ppWTlyEp6p3tZ2LeQUsq4SwIAAGi0ZXi0oZTyw0n+qNb6lVLK/5zkyiS/Wmv9cKvVAbsinOi+uTLnUjUAAGAiDDoJyi+vB0dXJ/meJEeT/Kv2ygL2aiOcEBwBAACwF4OGR2fWv35/kn9Va31nkq9rpyQAAAAAumLQ8Oi/lVL+TZIfSfKHpZSv38HvAgAAADChBg2AfiTJu5M8p9b65SSPTPI/tlYVAAAAAJ2w5YTZpZT9tdZTtda7k7xj4+e11s8l+dy5z2m3TAAAAADGYbuRR+8spfzvpZRrSin7Nn5YSvmWUspPllLeneQ57ZYIAAAAwLhsOfKo1vqdpZTvS/IzSa4qpTwiyVqSTyb5gyRHaq13tl8mAAAAAOOwZXiUJLXWP0zyhyOoBQAAAICOccc0AAAAABoJjwAAAABoJDwCAAAAoNFA4VEp5VtLKV+//v9DpZRXlFIe3m5pAAAAAIzboCOPfjfJmVLKf5fkjUmuSPLW1qoCAAAAoBMGDY/6tda1JD+U5PW11lcl+ab2ygIAAACgCwYNj+4tpbw4yZEk/3H9Zw9qpyQAAAAAumLQ8OjHk3xHkv+11vrpUsoVSf5de2UBAAAA0AXzgzyp1vrxJK9IklLKI5I8pNb6L9osDAAAAIDxG/RuazeXUh5aSnlkko8muamUcn27pQEAAAAwboNetvawWuvfJnlBkptqrU9L8vfaKwsAAADa0a/9nDx1MrXWcZcCE2HQ8Gi+lPJNSX4k90+YDQAAABOlX/s5fPRwFm9YzKGjh9Kv/XGXBJ03aHj02iTvTvKXtdYPlVK+Jcmn2isLAAAAhq93upfV46tZ669l9fhqeqd74y5pIEZLMU4DhUe11t+ptT6p1vry9e//qtb699stDQAAAIZrYd9Clg8uZ35uPssHl7Owb2HcJW3LaCnGbaC7rZVSFpP8epKrktQk70/yC7XWEy3WBgAAAENVSsnKkZX0TveysG8hpZRxl7StzUZLHdh/YNxlMUMGvWztpiT/IcmjkzwmybvWfwYAAAATZa7M5cD+AxMRHCWTOVqK6TLQyKMkl9Zazw2L3lRKeWUbBQGzq1/7E3UGCAAARmESR0sxXQYdefT5UspLSikXrf97SZIvtFkYMFtcxw0AAM0mbbQU02XQ8OgnkvxIkjuTfC7JC9d/BjAUk3rXCwAAgGm3bXhUSrkoyd+vtT6v1npprXWh1vqDtda/HkF9wIzo8nXcbosKAADMsm3nPKq1nimlPD/JDSOoB5hRXb2Oe+NyutXjq1k+uJyVIyuZK4MO2gQAAJh8g06Y/YFSyv+Z5LeTnN74Ya31w61UBcykjeu4u8RtUQEAgFk3aHi0vP71tef8rCa5drjlAHTLxuV0GyOPunQ5HQAAwCgMFB7VWg+3XQhAF3X1cjoAAIBR2XLijlLKPyml/OQmP//HpZRXtlcWQHe4LSoAADDLtpv19SeS/N+b/PzG9ccAAAAAmGLbhUe11vq1TX741SROwQMAAABMuW3vN11KecBthTb7GQAAAADTZ7vw6H9L8gellGeXUh6y/u9QkncleV3r1QEAAAAwVlveba3W+uZSSi/Ja5M8MUlN8l+TvKbW+v+MoD4AAAAAxmjL8ChJ1kMiQREAAADADNp2ziMAAAAAZpfwCAAAAIBGW4ZHpZRfWP961WjKAQAAAKBLtht59OPrX3+97UIAAAAA6J7tJsy+vZTymSSXllJuO+fnJUmttT6ptcoAAAAAGLstw6Na64tLKd+Y5N1JnjeakgAAAADoiu1GHqXWemeSJ5dSvi7J49Z//Mla672tVgYAAADA2A10t7VSyrOTfCrJbyT5zSR/Xkq5ps3CAGAz/drPyVMnU2sddykAADATBgqPklyf5Ltrrc+utV6T5HuS3NBeWQDwQP3az+Gjh7N4w2IOHT2Ufu2PuyQAAJh6g4ZHD6q1fnLjm1rrnyd5UDslAcDmeqd7WT2+mrX+WlaPr6Z3ujfukgAAYOoNGh4dK6W8sZRyaP3fG5Lc0mZhAHChhX0LWT64nPm5+SwfXM7CvoVxlwQAAFNv2wmz1708yc8neUWSkuSPc3buIwAYmVJKVo6spHe6l4V9CymljLskAACYegOFR7XWr+bsvEfXt1sOAGxtrszlwP4D4y4DAABmxqCXrQEAAAAwg4RHAAAAADTaUXhUStnXViEAAAAAdM9A4VEpZbmU8vEkt69//+RSigmzAQAAAKbcoCOPbkjyPUm+kCS11o8muaatogAAAADohoEvW6u1Hr/gR2eGXAsAAAAAHTM/4POOl1KWk9RSytcleUXWL2EDAAAAYHoNOvLoZ5P8fJLHJDmR5Cnr3wMAAAAwxQYaeVRr/XySH225FgAAAAA6ZqDwqJRyRZJ/nOTyc3+n1vq8dsoCAAAAoAsGnfPo95O8Mcm7kvTbKwcAAACALhk0PLqn1vp/tFoJAAAAAJ0zaHj0a6WU1yR5T5Kvbvyw1vrhVqoCAAAAoBMGDY++PclLk1yb+y9bq+vfAwAAADClBg2PfijJt9Rav9ZmMQAAAAB0y9yAz/tokoe3WQgAAAAA3TPoyKMDST5RSvlQzp/z6HmtVAUAAABAJwwaHr2m1SoAAAAA6KSBwqNa639uuxAAGLZ+7ad3upeFfQsppYy7HAAAmEhbznlUSnn/+tevlFL+9px/Xyml/O1oSgSAnevXfg4fPZzFGxZz6Oih9Gt/+18CAAAeYLuRR/uSpNb6kBHUAgBD0zvdy+rx1az117J6fDW9070c2H9g3GUBAMDE2e5ua3UkVQDAkC3sW8jyweXMz81n+eByFvYtjLskAACYSNuNPFoopfyTpgdrrdcPuR4AGIpSSlaOrJjzCAAA9mi78OiiJPuT6HEDMHHmypxL1QAAYI+2C48+V2t97UgqAQAAAKBztpvzyIgjAAAAgBm2XXj0nSOpAgAAAIBO2jI8qrV+cVSFAAAAANA92408AgAAAGCGCY8AAAAAaCQ8AgAAAKCR8AgAAACARsIjAAAAABq1Hh6VUi4qpdxaSvmP699fUUr5YCnlU6WU3y6lfF3bNQAAAMyqfu3n5KmTqbWOuxRgQo1i5NEvJLn9nO//ZZIbaq2PTfKlJD85ghoAAJhiDo5hc/3az+Gjh7N4w2IOHT2Ufu2PuyRgArUaHpVSFpN8f5J/u/59SXJtkrevP+Vokh9sswYAAKabg2No1jvdy+rx1az117J6fDW9071xlwRMoLZHHr0+yS8m2WjBH5Xky7XWtfXvTyR5TMs1AAAwIXYzgsjBMTRb2LeQ5YPLmZ+bz/LB5SzsWxh3ScAEai08KqU8N8nf1FpvOffHmzx1055BKeW6UsqxUsqxXk8HAABg2u12BJGDY2hWSsnKkZWceNWJ3Hzk5py9GARgZ+ZbfO2rkjyvlPJ9SS5O8tCcHYn08FLK/Proo8Ukd2z2y7XWG5PcmCRLS0suXgcAmHKbjSA6sP/Atr+3cXDcO93Lwr4FB8dwgbkyN9C2BNCktZFHtdb/qda6WGu9PMmLkvx/tdYfTbKS5IXrTzuS5J1t1QAAwOTYywiijYNjwREADF+bI4+avDrJb5VSfjXJrUneOIYaAADoGCOIAKCbRhIe1VpvTnLz+v//KskzRvF3AQCYLC6vAYDuaftuawAAAABMMOERAAAAAI2ERwAAAAA0Eh4BAABAB/VrPydPnUytddylMOOERwAAANAx/drP4aOHs3jDYg4dPZR+7Y+7JGaY8AgAAAA6pne6l9Xjq1nrr2X1+Gp6p3vjLokZJjwCAACAjlnYt5Dlg8uZn5vP8sHlLOxbGHdJzLD5cRcAAAAAnK+UkpUjK+md7mVh30JKKeMuiRlm5BEAAACMyE4mwZ4rczmw/4DgiLETHgEAAMAImASbSSU8AgAAgBEwCTaTSngEAAAAI2ASbCaVCbMBAABgBEyCzaQSHgEAAMCIbEyCDZPEZWsAAAAANBIeAQAAANBIeAQAAABAI+ERAAAAAI2ERwAAe9Sv/Zw8dTK11nGXAgAwdMIjAIA96Nd+Dh89nMUbFnPo6KH0a3/cJQEADJXwCABgD3qne1k9vpq1/lpWj6+md7o37pIAWmGUJcwu4REAwB4s7FvI8sHlzM/NZ/ngchb2LYy7JIChM8oSZtv8uAsAADbXr/30TveysG8hpZRxl9OqSX6vpZSsHFmZ2PoBBrHZKMsD+w+MuyxgRIw8AoAOmqUzvNPwXufKXA7sPyA4AqaWUZYw24w8AuA+kzz6Y9rM0hneWXqvAJPKKEuYbUYeAZBkOkZ/TJNZOsM7S+8VYJIZZQmzq0zCTPlLS0v12LFj4y4DYKqdPHUyizcsZq2/lvm5+Zx41QmjP8ZslkaCzdJ7BQDrFdP1AAAgAElEQVToilLKLbXWpe2eZ+QRAEmM/uiiWTrDO0vvFQBg0pjzCIAk5jIAAAA2JzwC4D4boz8AAAA2uGwNAAAAgEbCIwAAAJgR/drPyVMnMwk3z6I7hEcAAAAwA/q1n8NHD2fxhsUcOnoo/dofd0lMCOERAAAAzIDe6V5Wj69mrb+W1eOr6Z3ujbskJoTwCAAAAGbAwr6FLB9czvzcfJYPLmdh38K4S2JCuNsaAAAAzIBSSlaOrKR3upeFfQsppYy7JCaEkUcAAAAdYTJj2jZX5nJg/wHBETsiPAIAAOgAkxkDXSU8AgAA6ACTGQNdJTwCAADoAJMZA11lwmwAAIAOMJkx0FXCIwAAgI7YmMwYoEtctgYAAABAI+ERAAAAAI2ER8DA+rWfk6dOptY67lIAAOAB9FehHcIjYCD92s/ho4ezeMNiDh09lH7tj7skAAC4j/4qtEd4BAykd7qX1eOrWeuvZfX4anqne+MuCQAA7qO/Cu0RHgEDWdi3kOWDy5mfm8/yweUs7FsYd0kAAHAf/VVoT5mEa0GXlpbqsWPHxl0GzIR+7ad3upeFfQsppQz8GAAAjJv+KuxMKeWWWuvSds8z8gi4z3bXic+VuRzYf0BDDABAJ+mvQjuER8B9XCfONHP3FQCmmXYOaJPwCLiP68SZVu6+AsA0084BbZsfdwFAd5RSsnJkxXXiTJ3NRtUd2H9g3GUBwFBo54C2GXkEnMd14rNlVoa4G1UHwDTTzgFtc7c1gBm1McR99fhqlg8uZ+XISubK9J5TcPcVAKaZdg7YDXdbA2BL0zJB+qCjp4yqA2CaaeeANgmPAGbUNAxxN0EoANBVszI9ALPBhNkAM2oaJkg3QSgA0EWjnh7AZYu0zcgjgBk26UPcp2H0FAAwfUY5PYCR2IyCkUcATKxpGD0FAEyfjRNcGyOP2jzBZSQ2oyA8AmCibYyeAgDoilGe4BplUMXsEh4BAADAkI3qBJeR2IyC8AgAAAAmmJHYtM2E2QAAAAA0Eh4BQAv6tZ+Tp06m1jruUgAAYE+ERwDsmaDkfG6ZC+yU/SgAXSY8AmBPBCUPtNktc4HZsJsQyH6UURNWAjslPIIh0xgzawQlD7Rxy9z5uXm3zIUZstsQyH6UURJWArshPIIh0hgziwQlD7Rxy9wTrzqRm4/c7Ja5MCN2GwLZjzJKwkpgN+bHXQBMk80aY7fMZNptBCW9070s7FsQlKxzy1yYPRsh0Orx1R2FQPajjNJuP6fAbBMewRDtpDHu175OIlNDUAKwtxDIfpRREVYCuyE8giEatDHeuLxtI2RaObKSueIqUgCYdEIgJsG0fU6dlIX2OVqFIdtojLdquFxrDgDQXW6AMjnMOTqdbIPdIzyCMTAxJgBANwkjJouTstPHNthNwiNG7sIUeRZTZXdiAgDoJmHEZHFSdvrYBrtJeMRIXZgir/XXZjZVHuTyNuiSWQx6AZg9wojJ4qTs9LENdlOZhIOApaWleuzYsXGXwRCcPHUyizcsZq2/lvm5+dx63a156o1Pve/7E686MVWT9zF6Jkxsh0neAXZHuzSZrDcYL9vg6JRSbqm1Lm33PD1/RurCFPnbLv02qTJD4/ro9hg+DLBz2qXtDWtU67BHxxohDuNlG+ye+XEXwGzZ7Fb2g9zaHgaxWcBhJNtwbAS/GyOPBL0A29MubW1Yo1qNjgVon70qI3dhiixVZlhcH90e8wkA7Jx2aWvDGtVqdCxA+4w8AqaGkWzt2gh6AfZqVuay0C5tbaejWps+N0bHArTPhNkAQCtmJSBgZ1xixLn7hpo60H5iu8+N/Q1MHtttN5gwGwAYGxMF08QlRrPtwn1DkoGmL9juc2MaBJgs+gmTR3gEAFNs2HcgGpSAgCZdmwdoXNvIrNrtvqFrnxtgb/QTJo/wCFqkQ7o9ywjaM86zeg70aNKlCfid+R693e4buvS5AfZOP2HymPMIWmJOh+1ZRtCuk6dOZvGGxaz11zI/N58Trzox0knPzWVA1417GxnUtG1L0/Z+gN2xL+gGcx7BmBmKuT3LCNo1zLN6uxklaA4Sum4SznxP4+go+wYgsS+YNPPjLgCmldvGbs8ygnYN6zbhRgkyrYa1jbRpsxMtXRwdBcwuI4hmg/AIWjIJHdJxs4ygfRtn9fbCwSvTbBjbSJucaGHSCRammxNMs8NahRYZirm9ri4jE3nD/Sbh0h6YBLtpW0wUzSSbxssuOZ9pKGaH8AjgAjo6cD4Hr7B3e2lbunqiBbYjWJh+TjDNDuERMFKTMKJHRwceyMEr7I22hVkkWJh+TjDNDuERMDKTMqJn3B2dSQjYANiZcbctMA6ChdngBNNsKJNwcLK0tFSPHTs27jKAPTp56mQWb1jMWn8t83PzOfGqE52dpHRckzuadBBgepk4GICuKaXcUmtd2u55jkiAkZmks67jOoMyKZc1GB0FsHPOzgMwqebHXQAwOzaGLjvr2mwSbslsdBQAAMwW4REwUhtnXdncJARsm42Osk4BAGB6OVUM0DFdv6xhki4/BAAA9s7II2CqmZx0+CZhdBQAwE7oM8LWjDwChq4rkylvzM2zeMNiDh09lH7tj7WeadL10VEAAIPSZ4TtCY+AoepS4zspdy4DAGB89Blhe8IjYKi61PiamwceqCsjA3kg6wZgPPQZYXvmPAKGqku3mjc3D5xvY2Tgxva5cmQlc8V5pN0a5vwY1g3A+OgzwvaER8BQda3x3ZibB9h8ZKDtY3eGHfZYNwDjpc8IW3NKC2ZQ25dGmEwZusmw/OEZ9iW61g0A0GWtjTwqpRxM8uYk35ikn+TGWuuvlVIemeS3k1ye5DNJfqTW+qW26gDO59IImF1dGxk4yYZ9ia51AwB0WWlr5EEp5ZuSfFOt9cOllIckuSXJDyZ5WZIv1lr/RSnlnyZ5RK311Vu91tLSUj127FgrdcKsOXnqZBZvWMxafy3zc/M58aoTEz9Ed5jzjgAMyr4HAJh0pZRbaq1L2z2vteEGtdbP1Vo/vP7/ryS5Pcljkjw/ydH1px3N2UAJGJFpuzRiYyTV4g2LOXT0UPq1P+6SgBnhEl0AYFaMZMLsUsrlSZ6a5INJDtRaP5ecDZhKKZN95AoTZieXRkzCWXWTzMJkamP/Mgn7LACASdT6RCellP1JfjfJK2utf7uD37uulHKslHKs19vbJJSTYjeTGLc98THTaZCz5ZMyomfaRlLBLGhj/zIp+yygO/SjAQbXanhUSnlQzgZHb6m1vmP9xyfX50PamBfpbzb73VrrjbXWpVrr0qWXXtpmmZ2wm06vjjJtGvadhNqyMZLqxKtO5OYjN28biOkkwvi1sX+ZlH3WtJuV/eysvM9xGNWy1Y8G2JnWwqNy9gjujUlur7Vef85D/yHJkfX/H0nyzrZqmCS76fTqKNOmSRrRM00jqWAWtLF/maR91rSalf3srLzPcRjlstWPBtiZNuc8uirJS5P8WSnlI+s/+6Uk/yLJvy+l/GSSzyb54RZrmBi7ueXvsG8TDOeatttGj3tuJHOxwP3a2L9s95q2wfaNez87KrPyPsdhlMtWPxpgZ1oLj2qt70/S1Dv7zrb+7qTaTUd62g7uuV9XDnI2RvTsVRfezzg7iRtnUjf+9sqRlcyV1qecg04b1v5lkNe0DY7GKPazs96eTLtRLlv9aICdKZNwrfbS0lI9duzYuMuAkZi2g5wuvZ+2DzqaXv/kqZNZvGExa/21zM/N58SrTjhLzdTq4l3UbIOj0+Z+dpbak1lm2QKMVinlllrr0nbPm9wjUhiyrkx+OW3X4Hfp/QwyN9JubTVPg7lYmBVdvYvarG2D42zP2tzPzkp7Muumedl2pa8JsBvCo47SuIxWlya/nLaDnGl7P022OqjZyR3hYJJ19S5qs7QNdqk9G7ZZaU+YTtO8bQKzoc0Js9mlLg3LnhVdmvxy2q7Bn7b302S7eRramN8FuqaN+UqG9Zqzsg12qT0btllpT5g8g1xqN83bJjAbhEcdpHEZva5NfjltBznT9n4246AGxnMXNc7XtfZs2GahPWGyDHrSd9q3TWD6mTC7g2qtOXT00H2Ny7QPse8KEzQCMA20ZzA6O5mQ37YJ3WF7vJ8JsyfYOOZmMMfSdE/QmFjHwAPZL0ynaW/PZoXtczLsZC4u2yZ0gznIdkd4NGr9fnLyZFLr+f+/4LG5mhw4nZQLf2cHrzHoY/3az7U3HcqVv/KYHHrTs9M/szbYa7Zc164f62pdY6zZOlaXutR14WP9/pnz9wu134m6urq81DUbdfXPrKX3Vx9L7ffHWpd2e3LqKqVk5aX/KXf86K25+cdWUmrtRF2TuCzV1YHHJqSu8/bVu3j93qm/yZ/89QfyyL9dy+pnP3D2Zhy7rXmW1Fo7/+9pT3tanQpnztR6zTW1zs/X+qxn3f//a66p9d57N39s0Oft4bE7v3Si/ufLUr82l/qfvzn1q1c9sxN17eqxrtbVUs39Zz2r9ufna986Hv9j6lLXhNX11eVn3r9fuCz1zi+d6ERdXV1e6pr+uvrPela99XEPrV+bS731cQ+rZ7721bHVpd1WV+fq2mHN/WuuOdtH7VhdnXhMXXuq68zXvlpvfdzDzu6rH/vQs5+xPe7v+1/72u5qPnNm3AnDUCQ5VgfIZcYeDA3yb2rCozvvPPtBS2q96KL7/z8/X+uf/dnmjw36vD081r/ttnrvXKk1qfeW1P4Y69r42/3dvuYIltfQH9vla/QvuqjeO5ez622u1DO3fXQi1vG4lpe61KWurfch/dtu60RdXV1e6pr+uvoXXVS/tr5NfG0utffBlbHVpd2e3Lp20pe977mTsLx2sCzP3PbRnX9+J2gdq2u8dfU+uHL/vrqsbz8tb6uNj91557gThqEQHnVRv791ajlIArzV6KUzZ5pT/mH/3g5ff5DHHnDGb22blHdEdW33WP+ac0b/7OY1d/m3HzBq4K47Or+Oh/KYutSlronff03q8lLXdNf1gDPRa2tjrUu7PXl17aQve2bt3vNHT1zzrPEvkyEtyzvvuqN55NyEr2N1jb+u/traBdvO8P/2wP2jfn/cCcNQCI+66syZswllv3/+/7d6bMDnnemfqc9+47Pqo3/xonrN//Wssw3WIK/fcl2DPnbn336uPuh/uagu/A+p8//8onrnV+7sRF1bPXbmzNr5y7x/ZmQ198+cv777bSyHES5LdamrM4+pS13q6s5jI/7bZ9burX/zl39W+2fOdKquri4vdZ3/2E76snd+5c7zn3vXHd1YJkNYlv1+//w+6tpaJ+rq6vKalLoG3j+OoK7zahny397x8d0UGDQ8Kmef221LS0v12LFj4y6j83Zyq9AuqrXm0NFDWT2+muWDyyO709xejHuZu8Vkt1k/w2NZAtB1O+nLTmK/dye029Nl4+5kG5/XlSMrmSvTee+tcR/fjUMp5ZZa69J2z5sfRTGMxsatQjc26ksefElOnjq54532uHb2pZSsHFmZqIbmwmW+1e1Z27Bxy1e6Z5Ya2bZZlgBMgp30ZSex37sT+qjTpXe6l9Xjq1nrr2X1+Gp6p3tTu37HfXzXZUYeTZmN4OeSB1+Sa9987Y4Pthyk7ZwzK2xmFs9atMWyhNmljWXYfKYYtln4TE37SLkLzcI6PdegI4+kAlNmI+X//N2ff0A6PIjNUmW2trHMZ2HHwuA2zlrMz807a7FHliXMpo0TWos3LObQ0UPp1/64S2LC+UwxbLPymdoYKXfiVSemPjhKHN81cdnalNrtcLud/N6sJbKwE9M+HH2ULEuYTbN0mQSj4TPFsM3SZ8qliBh5NKV2mw4P+nuzkrLDXrR91qJf+zl56mQm4fLjvXIGCEarC/sXow4ZNp8phs1navZ0oX0cF3MesSvmIIHx2sv8ZEYNQjPbR7fmP9xqfVhXO7OT5TXNy3aa3xvj4TM1O7rUPg6TOY9oVZdT9q3S4HEnxeP++0yP3c5PtpNRgz6vjMu4PntG1Z7VpfkPm0YdWleD2diWzvTP7GjfP83L1kjWbpvEvsekf6baWOaTuB4H0aX2cRyER+xKVydN26rDM+7O0Lj/PtNltwHuoI2ezyvjMs7P3qx3Cjd0+QTRButqe+duS1ffdPXAy8uyZVz0PUavjWU+zetxEtrHNgmP2LUupuxbdXjG3Rka999nuuw2wB200fN57Z5pPYt3oXF+9ma9U7ihqyeIzrXdupqV7WUr525LH/pvH8rTH/30gT7btgPGRd9j9NpY5tO8HiehfWyT8IipslWHZ5idod10SnXGGLbdBLiDNno+r90yzWfxLjTOz96sdwrP1cUTROfaal3N0vaylXO3pau++aq878ffN9Bnu43tYNrCvGl7P12h7zF6bSzzaV+PXW8f22TC7Bly7mRuNXVqJ3Zre3JNExUzK3xex2Oz5T5rNynw2WMvZm172UoXJhyftglmp+39dM007/+7+t7aqKur75XNmTCb85x7Fu7Zb3p2Dr9p8s/INZ312SoNHkZSvJehmLOcVA+TM36j4fM6ek0jJqb9LN6FfPbYi1nbXrbShQnHp+0Slml7P10zrfv/Lo+IbGOZ7/Y19fG7TXg0Iy5s6FZPTHajN847RumUjleXG18m16g7K01/r+mgxOVUMDjby/aGFYAMsu+ctn7TtL0fRkPouD19/O4THs2ICxu65cXJbvTGeccondLx0vgybKPurGz197Y6KJnWs7HsnTO1D2R72dowApBB953T1m+atvfDaAgdt6eP333mPJoh0zTnUa01h44euu9686bG27wH02fQdU93dP2691HvJ7b7e11fXnTDxufkkgdfkmvffK35V9ixve5r9LGmV9vt0Ky2c7P6vgeljz8+5jziAc49CzfpZ+TcMWp2OeM3WSZhCPKob/m93d+b9P3ztOniqJ5zt6urb7ramVp2Za/7mlHvOxmNttvtSegXtEX7vjV9/O4z8ogdmcTEfBJrHjXLiLZMypnppm2grbvq2OYmQ1fvqnTudnVRuSjPeMwz8qE7PuRMLSM36n3nNOtKu3Bhu/3ZV342c2VuaHVNSr8AZomRRwzdpJ4pkPJvbVLWqzOYk2lSRv817Sfauv7efqlbdjqB+bidu11d9c1X5X0//j5nalsyaNszq23UqPed06pLfbFz9y/fsfgdedHbXzTUuialX3Chc7fxWd3eYX7cBTA5NusIOFMw+SZhvTqDOZiunLU818YQ5K7VNaiNTu7GZ29SOrkMbqv9S1fX/2bbVdf2212zm/3joG2PNuqBurrtdNW4+2IXbh8b+5daaw6+/uBQ65rEfsG52/h3LH5HSkpWT9jemT0+6QxsUs8UsLVJWK/OYG6vS2ctLzToKJsunslz/f3022r/0uX1b/Ta4Ha7fxy07dFG3W9jP56ks9tOF42zL7bZ9rGxfzmw/0ArdU3a/uvCbXz1hO19UnWxrzlJhEcMrMudaHZvEtbrJARc4zbpBy/TEH7t1ix1ZLr4XncygXkX69+LaXs/TXa7fxy07dFGnXXhfjzJRAUE4zTOvtikBuijdOE2vrxoe59EXe5rTgoTZo9RFy8xga6yvWxt0m9vOqsTaM7S5S5dfq+D7F+6XP9uTNv72cpe9o+Dtj3aqOnfj0/rOp6k/sM418G5f7umTuVnYdpN+z5qL0yY3XGSz+6blTOyk2LShjiP2qSfHezSmfvdbvu7+b1JHzG2E11+r4PsX7pc/1YmZTLwNtvcvewfB217tFHd2o8PW5f77Xvddial/zDudXDuNr7V9u74oR3DWK7TvI8aFeHRmHSt08b5xt1ATQKNY/dM8sFLVzqvu932d/t7w+rIDGt7bHO7nvRO2yTWv9XnskvvZxRt7iTvH8dt0P1CV/bjm9nrvq2r/fZhbTvj3D4GXTddXQfn2sv60K9uNqzP+V72UdbPWcKjMelSp22WTcoZ2a4RrtGGLhzc7Xbb3+3vDeNga1jbY9vbdZcPLAcxifVPylwm2ty9afOgZqf7hS7sxy80jH1bV/vt2207XT/g3cm66eo6ONdu92X61VsbZhuxm1Fj1s/9hEdj0qVO26yalDOyXaSjz7Ta7ba/l33GXg+2hrU9jmK77uKB5U60Uf84R3t1ZTJ4be7utX1QMw3t/TDeQ1f77VttO5NwwLuTdTPqdbCbffNu92XTsJ21aS9txKDrcavtxfq53/y4C5hlG502xmOzHcHG+thooEyGt7mNnfj/397dxth2lXUA/z+2IBYIRW5tgKIQaCJg4gVBW4pAK5GXmBSSmkDUAiFBk5KoMUb0g2LiB40vJEYk8QULBikNiKISkVRESLEUtchbiEVRb8HW8qYFgqEsP8y55TidPXPmzDn77fx+yc3cOXPu3HX2WuvZaz97rbXPbq5ooM9crNv3h4wZm+qP+nX/tr1p9bba5aY3GHfOPb6zddBa6xzLbMIc4sKmPsMYx+2H9Z3DxrmbctINrI9bN33Vwbqxed1YNod+tk3rHtfj1ONh/UX9fJ2nrbGzpvR0iTGa61NHYCq28eSXOfXrsX6W5XLd8cU7Jvfkl1UH455qsz3LdXDpRZemUrnxzPbGMmN5wtVUY9tQ//e2x7mbSn4fdnzW/dlJDRG/tvF5xnoe7Mtx6vGo/jL3Y+lpa3CEsU5BnoqpLz/Zb+z7AsCy/dOrk2ykP86lX491ucb+cp0679TklmutOn3fUrTtWa6D9515X6676rqtjmWGigub7Mdz+AzHte1x7v5YcPtdt681juqqm8OO3baP6xDxa9NtdJP7IU51fHycejyqv8xlfHRSZh4BO++4d8/mfveB8TOr43BjPT4HleuC+18wqXhynNkMYuV27MrM6bH24+OYw2fostwO98+A28QS3MOOXR/HdSrxq6ucmzhG215a3Yep1OPQzDyasSlngGGMjrMRXt93EfV3DmJWx+HGenwOKtfU7mYeZzbD1D7bVOzKzOmx9uPjmMNn6LLcDq+/6vrceGazGwofduz6OK5TiF/bfvjPHDaKnkI9TomZRxMzhwww2yO7vp7j3MXt8y7iSfq7tjB/6vhwYz0+Yy0XjNEc+sscPsNRtjUbbqg9j6biqDHpSY/RVGc5ahvHt+rMI8mjiZnz9FdORmLxZFY90fR5Il13icsc2oITP0yDvgokYsEQ+lhGPLV6ncMYeAiWrc3UnKe/cjJzmFq6X59Ltlad1trncoH9/f3UeadWWjI3dFs4ab2dZGmgZX7Qn7FuTA70z/Kg/q06Jj1JrJ5avQ49Bp47yaOJGfM69+WLtm1cwLkoPNzcEotjvijp60S6v7/f+aU7R/+Uo03U27on/jG3GZgjg3RgauZ2PbHKmHSXYvXcrofGRvJogsaYAV6+aHv6tU/P5ddu9gLOReHRxpxYXMcunegOs9zfVz0hDtkWNlFv6574+2gzcxt0wkkYpO8mcZCp2tXriePE6qn377ldD42NPY/YiOW9Wc6pc1JVG92XyV5Pu2eqm/Rt29jXnm9q/f06n3PbbeagdfRJRl0fczT2PrBr1MdusZ8IU7bL1xOrxGr9e3fZ84he7c9oP+Wizd6JdHdz97hzcLAxzjxctqn19+t8zm23mf0zm26/6/advIM5pF27azyFO8Bjj0ls1lRmBa/bd6bQ56ZQxrHa5esJy9vYBMkjjrTKSWr5ou3dL3533vXizV7AzT2RYCBwsG1flOzKce/7cw45QNlmm9k/6KzUzg6yhuo7B7WbufbjXUuUMQ1TuPhet++Muc+djXN3f+3u0ZZxCuZ+PXFSU+jfDEvyiEMd50S6fNG2jQu4ud7dHPNgZc525biP7XOeHQBfcN4Fkxug7B90XviACyf3GTZhyDa17hMIp8gdYNax7WTqFC6+1+07Y+1zyzH3qX/w1FGWcUrmej2xCX3177ne9NkFkkccaqwn0jlxjIexK8d9TJ9zeQB8+esvzw1X3zDqC5CDLA86p3ARtQ1Dtql1n0A4Re4Ac1x9JXbHfvG9bt8Za59bjrk333ZznvywJ4+ujMxHH7P+53rTZxdIHnGosZ5I58QxHsauHPcxfc79SYfPfOkzo74AWcXYL6K24aDZP0MtixxT+960vpOTfdwJdrd5u8Z0s2BI6/adsd4QWI5zl33rZXnPS94zujLCqsSpafO0tZ5N8akkUyzz1DjGw9iV4z6Wzzm2J+j1eVzGUgebcvbznDrvVK54/RWDPpllbsd2CH08YcdTfA62yfY7thjL5ohzzIU4NU6rPm1N8qhHBk7ArhvLALjPeDzV2L9KXe3yY4/7ts2+00c9aiv3to3YMJYY24dd+qwwJ/ru+KyaPBr/6HVGTNMDdt1Ylnn1GY+nGPtX3ZNgzsvGxmTbe0T0UY/ayr1tIzaMJcZum31TYDr2L1nelTg1R5JHPTJwYh1T2SNiKuWEpN94PETsP2l/XPWidqx7hIzBch30VR/r6qMetZV7My5c3xST8rCLJHrnxbK1npmmN01D1dtUlrtMpZywbK57Hm2iP9qT4GSW6+DSiy5NpXLjmXHUh3HIuKiP9YhRMDzL2+fDnkewIUMmRqYScKdSTtgFm+qPLmrXt1wH59Q5qapR1IdE/x5tex7UIwxn1fNJH4leseDk7HnETtnmkqkhp0ZPZUr7VMoJu2BT/dGeBOvbXwdPuWgc9WGpjyUUXaa49FyMguGMZXm7mN6vc4cuALthmxnhbd9JPXsRcPb395kYORtwx55Nn0o5YRfoj8PbXwctbRT1MeT5bCwOuuDZ9ZmyZqQBx3Wc88nZRO82iOn9smxtAqY+FW/bg5I+lkxNvQ7YHm0DmJJdj1n2yrk3S8+BdYzhfCKmb8aqy9bMPBq5OdwN2nZGuI87qdvMmDNdc+ifwG7Z9fOZmXn3ZkYasI4xnE/E9H5JHo3cHKbibXtQImgwlDn0T4CjjOHu8iaN4YJnTIyjgCkT0/vjFvnIzS+tRGYAAAixSURBVGEj4m1vlJbYNJFhzKF/AhzGZqS7wTiKuZriZvAwVvY8moC53fGDOdE/gTmzHw4wVbYXgNWsuueR3jMB7gbBeOmfwJyZYfl1ZjDAtKz6OPmxEWsYK8kjAAAO1MfS8ymwfA+mZ4rJb7GGMbNhNgAAnWxG6gEJMEVT3AxerGHMzDwCAIBDTHEGAzC97QXEGsbMhtkwMTZoBoD+Of8CfRBr6JsNs2GGrIMGgGFMbQYDME1iDWMleQQTMtWnRgAAADBdkkcwIdZBAwAA0DdPW4MJmeJTIwAAAJg2ySOYGI9MBgAAoE+WrQEAAADQSfIIAAAAgE6SRwAAAAB0kjwCAAAAoJPkEQAAAACdJI8AAAAA6CR5BAAAAEAnySMAAAAAOkkeAQAAANBJ8ggAAACATpJHAAAAAHSSPAIAAACgk+QRAAAAAJ0kjwAAAADoJHkEAAAAQCfJIwAAAAA6SR4BAAAA0EnyCAAAAIBOkkcAAAAAdJI8AgAAAKCT5BEAAAAAnaq1NnQZjlRV/5Xk34YuxwadSnLn0IWAFWmvTI02y9Ros0yNNsuUaK9MTd9t9ttaaxcc9aZJJI/mpqo+0Fp70tDlgFVor0yNNsvUaLNMjTbLlGivTM1Y26xlawAAAAB0kjwCAAAAoJPk0TB+Z+gCwDFor0yNNsvUaLNMjTbLlGivTM0o26w9jwAAAADoZOYRAAAAAJ0kj3pUVc+uqo9X1a1V9YqhywMHqapPVtWHquqWqvrA4rVvrqp3VtU/L74+eOhysruq6rVVdUdVfXjptQPbaO35zUXc/aeqeuJwJWdXdbTZV1bVbYtYe0tVPXfpZz+7aLMfr6pnDVNqdlVVPaKq3lVVH6uqj1TVjy9eF2cZpUParDjLKFXV/arq/VX1wUWb/cXF64+qqpsWcfZNVXXfxevfuPj+1sXPHzlEuSWPelJV5yR5dZLnJHlckhdW1eOGLRV0ury1dnrpEZGvSHJDa+3iJDcsvoehXJvk2fte62qjz0ly8eLPy5K8pqcywrJrc+82mySvWsTa0621tyfJYmzwgiSPX/yb316MIaAvX03yU621xya5JMk1i3YpzjJWXW02EWcZp68kuaK19p1JTid5dlVdkuRXstdmL07yuSQvXbz/pUk+11p7TJJXLd7XO8mj/nx3kltba//SWvvfJNcluXLgMsGqrkzyusXfX5fkeQOWhR3XWvvbJJ/d93JXG70yyevbnr9Lcn5VPbSfksKejjbb5cok17XWvtJa+9ckt2ZvDAG9aK19urX2D4u//0+SjyV5eMRZRuqQNttFnGVQi3h51+Lb+yz+tCRXJHnz4vX9cfZs/H1zku+rquqpuPeQPOrPw5P8x9L3Z3J4UIOhtCR/VVV/X1UvW7x2YWvt08neCTrJtwxWOjhYVxsVexmzly+W+bx2aTmwNstoLJZGPCHJTRFnmYB9bTYRZxmpqjqnqm5JckeSdyb5RJLPt9a+unjLcru8p80ufv6FJA/pt8SSR306KDPoUXeM0WWttSdmbxr6NVX1tKELBCcg9jJWr0ny6OxNV/90kl9fvK7NMgpV9YAkb0nyE621/z7srQe8ps3SuwParDjLaLXW7m6tnU5yUfZmvj32oLctvo6izUoe9edMkkcsfX9Rkk8NVBbo1Fr71OLrHUnemr1gdvvZKeiLr3cMV0I4UFcbFXsZpdba7YuB49eS/G6+vmRCm2VwVXWf7F2Ev6G19seLl8VZRuugNivOMgWttc8n+Zvs7dd1flWdu/jRcru8p80ufv6grL4cfmMkj/pzc5KLFzuo3zd7m7S9beAywf9TVfevqgee/XuS70/y4ey11Rct3vaiJH86TAmhU1cbfVuSqxdPA7okyRfOLruAIe3bE+b52Yu1yV6bfcHiySqPyt4mxO/vu3zsrsU+Gr+f5GOttd9Y+pE4yyh1tVlxlrGqqguq6vzF378pyTOzt1fXu5JctXjb/jh7Nv5eleSvW2u9zzw69+i3sAmtta9W1cuTvCPJOUle21r7yMDFgv0uTPLWxf5r5yb5o9baX1bVzUmur6qXJvn3JD84YBnZcVX1xiTPSHKqqs4k+YUkv5yD2+jbkzw3e5thfinJS3ovMDuvo80+o6pOZ2/a+SeT/GiStNY+UlXXJ/lo9p4gdE1r7e4hys3OuizJjyT50GI/jiT5uYizjFdXm32hOMtIPTTJ6xZP+fuGJNe31v68qj6a5Lqq+qUk/5i9pGgWX/+wqm7N3oyjFwxR6BogYQUAAADARFi2BgAAAEAnySMAAAAAOkkeAQAAANBJ8ggAAACATpJHAAAAAHSSPAIAWKiqh1TVLYs//1lVty19f999731HVT3wiN93pqrOP+Tn11TVD1fVk6rqtzb1OQAANuncoQsAADAWrbXPJDmdJFX1yiR3tdZ+bfk9VVVJqrX2rA38l9+b5GeSXJXkPRv4fQAAGyd5BABwhKp6TJI/SfLeJN+T5Aeq6qYk39Fa+3xV/VmShyW5X5JXtdZ+74jf99NJfijJxUm+Pckjk9xWVU9rrV2zvU8CAHB8kkcAAKt5XJKXtNZ+LEn2JiDd40Wttc9W1XlJPlBVb2mtfa7rF7XWfrWq3pDk1a2151fVza21J2+19AAAa7LnEQDAaj7RWru542c/WVUfTPK+JBclefQKv+8JSW6pqgcnuXNDZQQA2DgzjwAAVvPFg16sqmcmeVqSS1prX66q92Zv+dqBquphSd6e5MIkX05ydZL7V9UtSZ7XWvvkpgsOAHASZh4BAJzMg5J8dpE4enySQ5eftdY+1Vo7neSWJN+V5I1Jrm6tnZY4AgDGSPIIAOBk/iLJeYtlaz+f5Kaj/kFVnZvkQYt9kS5JcuN2iwgAsL5qrQ1dBgAAAABGyswjAAAAADpJHgEAAADQSfIIAAAAgE6SRwAAAAB0kjwCAAAAoJPkEQAAAACdJI8AAAAA6CR5BAAAAECn/wNFvy14op5KZgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R^2: -0.002135972367997674\n",
      "Mean Absolute Error: 7.137601684377658\n",
      "Mean Squared Error: 119.9394967413633\n",
      "Root Mean Squared Error: 10.951689218625742\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.regression import r2_score, mean_squared_error\n",
    "y_pred = regressor.predict(X_test)\n",
    "\n",
    "x_ax=range(len(y_pred))\n",
    "plt.figure(figsize=(20, 10)) \n",
    "plt.scatter(x_ax, y_test, s=5, color=\"green\", label=\"original\")\n",
    "plt.scatter(x_ax, y_pred, s=5, color=\"red\", label=\"predicted\")\n",
    "    \n",
    "plt.xlabel(\"Trial #\")\n",
    "plt.ylabel(\"Time of Cross (s)\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "print('R^2:', metrics.r2_score(y_test,y_pred))\n",
    "print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  \n",
    "print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  \n",
    "print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(294, 13)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}