DeepLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import sklearn\n",
    "import sklearn.preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, plot_confusion_matrix\n",
    "from sklearn.decomposition import PCA\n",
    "import keras\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Conv1D, Flatten, concatenate, Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from dataset import HARDatasetCrops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "dataset = HARDatasetCrops('motionsense-dataset/train', 256, 50, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset size: 4095\n",
      "Datapoints shape: (256, 12)\n"
     ]
    }
   ],
   "source": [
    "print('Dataset size:', len(dataset))\n",
    "\n",
    "sample, _ = dataset[0]\n",
    "print('Datapoints shape:', sample.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our dataset is composed of 5205 datapoints, each have shape `(256, 12)` because we have 12 signals of 256 samples each."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For our base line model we will use the **user-acceleration**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "dataset = HARDatasetCrops('motionsense-dataset/train', 256, 50, 50, metadata_file='motionsense-dataset/data_subjects_info.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "X = np.array([sample[:,-3:] for sample, _, _ in dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = np.array([metadata for _, _, metadata in dataset])\n",
    "metadata = metadata.reshape((-1, 4, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to fit a `sklearn` model we should encode each class with a unique integer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "label_encoder = sklearn.preprocessing.LabelEncoder()\n",
    "label_encoder.fit(list(dataset.CLASSES.keys()));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "y = to_categorical(label_encoder.transform([cls for _, cls, _ in dataset]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train-test splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "indices = np.random.choice(np.arange(X.shape[0]), X.shape[0], replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indices = indices[:int(X.shape[0] * 0.8)]\n",
    "test_indices = indices[int(X.shape[0] * 0.8):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = X[train_indices], y[train_indices]\n",
    "X_test, y_test = X[test_indices], y[test_indices]\n",
    "metadata_train = metadata[train_indices]\n",
    "metadata_test = metadata[test_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X train shape: {} (3276, 256, 3)\n",
      "y train shape: {} (3276, 6)\n",
      "X test shape: {} (819, 256, 3)\n",
      "y test shape: {} (819, 6)\n",
      "metdata train shape: {} (3276, 4, 1)\n",
      "metdata test shape: {} (819, 4, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"X train shape: {}\", X_train.shape)\n",
    "print(\"y train shape: {}\", y_train.shape)\n",
    "print(\"X test shape: {}\", X_test.shape)\n",
    "print(\"y test shape: {}\", y_test.shape)\n",
    "print(\"metdata train shape: {}\", metadata_train.shape)\n",
    "print(\"metdata test shape: {}\", metadata_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: 3276\n",
      "Number of testing examples: 819\n"
     ]
    }
   ],
   "source": [
    "print('Number of training examples:', y_train.shape[0])\n",
    "print('Number of testing examples:', y_test.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create model\n",
    "clf = Sequential()\n",
    "#add model layers\n",
    "clf.add(Conv1D(16, kernel_size=5, activation=\"relu\", input_shape=(256, 3)))\n",
    "clf.add(Conv1D(32, kernel_size=5, activation=\"relu\"))\n",
    "clf.add(Flatten())\n",
    "clf.add(Dense(6, activation=\"softmax\", kernel_regularizer=keras.regularizers.l2(0.01)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3276 samples, validate on 819 samples\n",
      "Epoch 1/30\n",
      "3276/3276 [==============================] - 1s 394us/step - loss: 1.3056 - accuracy: 0.5052 - val_loss: 0.8162 - val_accuracy: 0.6886\n",
      "Epoch 2/30\n",
      "3276/3276 [==============================] - 1s 267us/step - loss: 0.7274 - accuracy: 0.7497 - val_loss: 0.6819 - val_accuracy: 0.7875\n",
      "Epoch 3/30\n",
      "3276/3276 [==============================] - 1s 267us/step - loss: 0.6144 - accuracy: 0.7995 - val_loss: 0.6148 - val_accuracy: 0.7998\n",
      "Epoch 4/30\n",
      "3276/3276 [==============================] - 1s 270us/step - loss: 0.5474 - accuracy: 0.8266 - val_loss: 0.6067 - val_accuracy: 0.8071\n",
      "Epoch 5/30\n",
      "3276/3276 [==============================] - 1s 291us/step - loss: 0.5121 - accuracy: 0.8404 - val_loss: 0.5955 - val_accuracy: 0.8168\n",
      "Epoch 6/30\n",
      "3276/3276 [==============================] - 1s 286us/step - loss: 0.4754 - accuracy: 0.8574 - val_loss: 0.5030 - val_accuracy: 0.8400\n",
      "Epoch 7/30\n",
      "3276/3276 [==============================] - 1s 272us/step - loss: 0.4524 - accuracy: 0.8629 - val_loss: 0.5575 - val_accuracy: 0.8339\n",
      "Epoch 8/30\n",
      "3276/3276 [==============================] - 1s 281us/step - loss: 0.4222 - accuracy: 0.8761 - val_loss: 0.5775 - val_accuracy: 0.8034\n",
      "Epoch 9/30\n",
      "3276/3276 [==============================] - 1s 323us/step - loss: 0.4194 - accuracy: 0.8764 - val_loss: 0.4677 - val_accuracy: 0.8803\n",
      "Epoch 10/30\n",
      "3276/3276 [==============================] - 1s 274us/step - loss: 0.3978 - accuracy: 0.8962 - val_loss: 0.4657 - val_accuracy: 0.8681\n",
      "Epoch 11/30\n",
      "3276/3276 [==============================] - 1s 293us/step - loss: 0.3722 - accuracy: 0.9032 - val_loss: 0.5218 - val_accuracy: 0.8291\n",
      "Epoch 12/30\n",
      "3276/3276 [==============================] - 1s 274us/step - loss: 0.3716 - accuracy: 0.9011 - val_loss: 0.4713 - val_accuracy: 0.8596\n",
      "Epoch 13/30\n",
      "3276/3276 [==============================] - 1s 272us/step - loss: 0.3385 - accuracy: 0.9139 - val_loss: 0.4345 - val_accuracy: 0.8828\n",
      "Epoch 14/30\n",
      "3276/3276 [==============================] - 1s 278us/step - loss: 0.3473 - accuracy: 0.9133 - val_loss: 0.4170 - val_accuracy: 0.8889\n",
      "Epoch 15/30\n",
      "3276/3276 [==============================] - 1s 278us/step - loss: 0.3377 - accuracy: 0.9136 - val_loss: 0.4229 - val_accuracy: 0.8938\n",
      "Epoch 16/30\n",
      "3276/3276 [==============================] - 1s 278us/step - loss: 0.3353 - accuracy: 0.9173 - val_loss: 0.3924 - val_accuracy: 0.9035\n",
      "Epoch 17/30\n",
      "3276/3276 [==============================] - 1s 283us/step - loss: 0.2888 - accuracy: 0.9386 - val_loss: 0.3859 - val_accuracy: 0.8950\n",
      "Epoch 18/30\n",
      "3276/3276 [==============================] - 1s 286us/step - loss: 0.2788 - accuracy: 0.9396 - val_loss: 0.4009 - val_accuracy: 0.8999\n",
      "Epoch 19/30\n",
      "3276/3276 [==============================] - 1s 319us/step - loss: 0.2904 - accuracy: 0.9414 - val_loss: 0.3723 - val_accuracy: 0.9072\n",
      "Epoch 20/30\n",
      "3276/3276 [==============================] - 1s 286us/step - loss: 0.2724 - accuracy: 0.9389 - val_loss: 0.4746 - val_accuracy: 0.8840\n",
      "Epoch 21/30\n",
      "3276/3276 [==============================] - 1s 275us/step - loss: 0.2631 - accuracy: 0.9447 - val_loss: 0.3392 - val_accuracy: 0.9133\n",
      "Epoch 22/30\n",
      "3276/3276 [==============================] - 1s 278us/step - loss: 0.2458 - accuracy: 0.9521 - val_loss: 0.3974 - val_accuracy: 0.8889\n",
      "Epoch 23/30\n",
      "3276/3276 [==============================] - 1s 278us/step - loss: 0.3114 - accuracy: 0.9197 - val_loss: 0.4893 - val_accuracy: 0.8608\n",
      "Epoch 24/30\n",
      "3276/3276 [==============================] - 1s 276us/step - loss: 0.2594 - accuracy: 0.9509 - val_loss: 0.3335 - val_accuracy: 0.9304\n",
      "Epoch 25/30\n",
      "3276/3276 [==============================] - 1s 280us/step - loss: 0.2499 - accuracy: 0.9478 - val_loss: 0.3597 - val_accuracy: 0.9158\n",
      "Epoch 26/30\n",
      "3276/3276 [==============================] - 1s 280us/step - loss: 0.2347 - accuracy: 0.9548 - val_loss: 0.3425 - val_accuracy: 0.9219\n",
      "Epoch 27/30\n",
      "3276/3276 [==============================] - 1s 289us/step - loss: 0.2157 - accuracy: 0.9563 - val_loss: 0.3254 - val_accuracy: 0.9084\n",
      "Epoch 28/30\n",
      "3276/3276 [==============================] - 1s 293us/step - loss: 0.2456 - accuracy: 0.9386 - val_loss: 0.3556 - val_accuracy: 0.9145\n",
      "Epoch 29/30\n",
      "3276/3276 [==============================] - 1s 292us/step - loss: 0.2669 - accuracy: 0.9457 - val_loss: 0.3321 - val_accuracy: 0.9328\n",
      "Epoch 30/30\n",
      "3276/3276 [==============================] - 1s 275us/step - loss: 0.2486 - accuracy: 0.9521 - val_loss: 0.3250 - val_accuracy: 0.9158\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x1117b4250>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "clf.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       0.99      1.00      0.99       298\n",
      "         jog       1.00      1.00      1.00       304\n",
      "         sit       0.86      0.98      0.92       788\n",
      "         std       0.97      0.83      0.90       722\n",
      "         ups       1.00      1.00      1.00       351\n",
      "         wlk       1.00      0.99      1.00       813\n",
      "\n",
      "    accuracy                           0.96      3276\n",
      "   macro avg       0.97      0.97      0.97      3276\n",
      "weighted avg       0.96      0.96      0.96      3276\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       0.81      0.89      0.84        61\n",
      "         jog       0.99      0.99      0.99        80\n",
      "         sit       0.89      0.98      0.93       216\n",
      "         std       0.97      0.85      0.91       178\n",
      "         ups       0.84      0.81      0.83        85\n",
      "         wlk       0.94      0.92      0.93       199\n",
      "\n",
      "    accuracy                           0.92       819\n",
      "   macro avg       0.91      0.91      0.91       819\n",
      "weighted avg       0.92      0.92      0.92       819\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(np.argmax(y_train, axis=1), np.argmax(clf.predict(X_train), axis=1), target_names=label_encoder.classes_))\n",
    "\n",
    "print(classification_report(np.argmax(y_test, axis=1), np.argmax(clf.predict(X_test), axis=1), target_names=label_encoder.classes_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create model\n",
    "conv = Sequential()\n",
    "# add model layers\n",
    "conv.add(Conv1D(16, kernel_size=5, activation=\"relu\", input_shape=(256, 3)))\n",
    "conv.add(Conv1D(32, kernel_size=5, activation=\"relu\"))\n",
    "conv.add(Flatten(name=\"coefs\"))\n",
    "\n",
    "metadata_input_tensor = Input(shape=(4, 1))\n",
    "metadata_input = Flatten(name=\"flatten\")(metadata_input_tensor)\n",
    "\n",
    "last_layer = conv.get_layer(\"coefs\").output\n",
    "x = concatenate([last_layer, metadata_input], axis=1)\n",
    "out = Dense(6, activation=\"softmax\")(x)\n",
    "\n",
    "clf = Model([conv.input, metadata_input_tensor], out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3276 samples, validate on 819 samples\n",
      "Epoch 1/30\n",
      "3276/3276 [==============================] - 1s 397us/step - loss: 0.9044 - accuracy: 0.6230 - val_loss: 0.5717 - val_accuracy: 0.7534\n",
      "Epoch 2/30\n",
      "3276/3276 [==============================] - 1s 318us/step - loss: 0.5080 - accuracy: 0.7906 - val_loss: 0.5403 - val_accuracy: 0.7717\n",
      "Epoch 3/30\n",
      "3276/3276 [==============================] - 1s 316us/step - loss: 0.3964 - accuracy: 0.8379 - val_loss: 0.4283 - val_accuracy: 0.8327\n",
      "Epoch 4/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.3124 - accuracy: 0.8794 - val_loss: 0.4200 - val_accuracy: 0.8449\n",
      "Epoch 5/30\n",
      "3276/3276 [==============================] - 1s 316us/step - loss: 0.2570 - accuracy: 0.9051 - val_loss: 0.3268 - val_accuracy: 0.8791\n",
      "Epoch 6/30\n",
      "3276/3276 [==============================] - 1s 322us/step - loss: 0.2064 - accuracy: 0.9270 - val_loss: 0.3334 - val_accuracy: 0.8852\n",
      "Epoch 7/30\n",
      "3276/3276 [==============================] - 1s 319us/step - loss: 0.1704 - accuracy: 0.9393 - val_loss: 0.3102 - val_accuracy: 0.8913\n",
      "Epoch 8/30\n",
      "3276/3276 [==============================] - 1s 333us/step - loss: 0.1488 - accuracy: 0.9509 - val_loss: 0.3192 - val_accuracy: 0.8950\n",
      "Epoch 9/30\n",
      "3276/3276 [==============================] - 1s 328us/step - loss: 0.1202 - accuracy: 0.9618 - val_loss: 0.2964 - val_accuracy: 0.8950\n",
      "Epoch 10/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.1095 - accuracy: 0.9658 - val_loss: 0.3792 - val_accuracy: 0.8645\n",
      "Epoch 11/30\n",
      "3276/3276 [==============================] - 1s 321us/step - loss: 0.1028 - accuracy: 0.9646 - val_loss: 0.3093 - val_accuracy: 0.9072\n",
      "Epoch 12/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.0972 - accuracy: 0.9670 - val_loss: 0.3886 - val_accuracy: 0.8791\n",
      "Epoch 13/30\n",
      "3276/3276 [==============================] - 1s 323us/step - loss: 0.0931 - accuracy: 0.9661 - val_loss: 0.3555 - val_accuracy: 0.8913\n",
      "Epoch 14/30\n",
      "3276/3276 [==============================] - 1s 323us/step - loss: 0.0902 - accuracy: 0.9676 - val_loss: 0.3337 - val_accuracy: 0.8987\n",
      "Epoch 15/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.0741 - accuracy: 0.9710 - val_loss: 0.3185 - val_accuracy: 0.9096\n",
      "Epoch 16/30\n",
      "3276/3276 [==============================] - 1s 326us/step - loss: 0.0783 - accuracy: 0.9725 - val_loss: 0.3296 - val_accuracy: 0.9121\n",
      "Epoch 17/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.0724 - accuracy: 0.9704 - val_loss: 0.3382 - val_accuracy: 0.9060\n",
      "Epoch 18/30\n",
      "3276/3276 [==============================] - 1s 337us/step - loss: 0.0652 - accuracy: 0.9737 - val_loss: 0.3387 - val_accuracy: 0.9084\n",
      "Epoch 19/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.0590 - accuracy: 0.9756 - val_loss: 0.4005 - val_accuracy: 0.8803\n",
      "Epoch 20/30\n",
      "3276/3276 [==============================] - 1s 324us/step - loss: 0.0568 - accuracy: 0.9768 - val_loss: 0.3626 - val_accuracy: 0.9048\n",
      "Epoch 21/30\n",
      "3276/3276 [==============================] - 1s 334us/step - loss: 0.0502 - accuracy: 0.9795 - val_loss: 0.3740 - val_accuracy: 0.9060\n",
      "Epoch 22/30\n",
      "3276/3276 [==============================] - 1s 423us/step - loss: 0.0491 - accuracy: 0.9805 - val_loss: 0.4005 - val_accuracy: 0.8999\n",
      "Epoch 23/30\n",
      "3276/3276 [==============================] - 1s 428us/step - loss: 0.0495 - accuracy: 0.9805 - val_loss: 0.3913 - val_accuracy: 0.8999\n",
      "Epoch 24/30\n",
      "3276/3276 [==============================] - 1s 419us/step - loss: 0.0478 - accuracy: 0.9817 - val_loss: 0.3982 - val_accuracy: 0.9096\n",
      "Epoch 25/30\n",
      "3276/3276 [==============================] - 1s 363us/step - loss: 0.0445 - accuracy: 0.9802 - val_loss: 0.4119 - val_accuracy: 0.9096\n",
      "Epoch 26/30\n",
      "3276/3276 [==============================] - 1s 335us/step - loss: 0.0606 - accuracy: 0.9811 - val_loss: 0.6159 - val_accuracy: 0.8608\n",
      "Epoch 27/30\n",
      "3276/3276 [==============================] - 1s 325us/step - loss: 0.1303 - accuracy: 0.9573 - val_loss: 0.4247 - val_accuracy: 0.9035\n",
      "Epoch 28/30\n",
      "3276/3276 [==============================] - 1s 345us/step - loss: 0.0513 - accuracy: 0.9823 - val_loss: 0.3889 - val_accuracy: 0.9133\n",
      "Epoch 29/30\n",
      "3276/3276 [==============================] - 1s 315us/step - loss: 0.0417 - accuracy: 0.9853 - val_loss: 0.3924 - val_accuracy: 0.9158\n",
      "Epoch 30/30\n",
      "3276/3276 [==============================] - 1s 359us/step - loss: 0.0371 - accuracy: 0.9869 - val_loss: 0.4257 - val_accuracy: 0.8950\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x13f1a5290>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "clf.fit([X_train, metadata_train], y_train, validation_data=([X_test, metadata_test], y_test), epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       1.00      1.00      1.00       298\n",
      "         jog       1.00      1.00      1.00       304\n",
      "         sit       0.99      0.93      0.96       788\n",
      "         std       0.93      0.99      0.96       722\n",
      "         ups       1.00      1.00      1.00       351\n",
      "         wlk       1.00      1.00      1.00       813\n",
      "\n",
      "    accuracy                           0.98      3276\n",
      "   macro avg       0.99      0.99      0.99      3276\n",
      "weighted avg       0.98      0.98      0.98      3276\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       0.80      0.64      0.71        61\n",
      "         jog       0.96      1.00      0.98        80\n",
      "         sit       0.97      0.90      0.94       216\n",
      "         std       0.89      0.97      0.93       178\n",
      "         ups       0.71      0.76      0.74        85\n",
      "         wlk       0.90      0.91      0.91       199\n",
      "\n",
      "    accuracy                           0.89       819\n",
      "   macro avg       0.87      0.86      0.87       819\n",
      "weighted avg       0.90      0.89      0.89       819\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(np.argmax(y_train, axis=1), np.argmax(clf.predict([X_train, metadata_train]), axis=1), target_names=label_encoder.classes_))\n",
    "\n",
    "print(classification_report(np.argmax(y_test, axis=1), np.argmax(clf.predict([X_test, metadata_test]), axis=1), target_names=label_encoder.classes_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All signals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "X = np.array([sample for sample, _, _ in dataset])\n",
    "\n",
    "X_train = X[train_indices]\n",
    "X_test = X[test_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X train shape: {} (3276, 256, 12)\n",
      "y train shape: {} (3276, 6)\n",
      "X test shape: {} (819, 256, 12)\n",
      "y test shape: {} (819, 6)\n",
      "metdata train shape: {} (3276, 4, 1)\n",
      "metdata test shape: {} (819, 4, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"X train shape: {}\", X_train.shape)\n",
    "print(\"y train shape: {}\", y_train.shape)\n",
    "print(\"X test shape: {}\", X_test.shape)\n",
    "print(\"y test shape: {}\", y_test.shape)\n",
    "print(\"metdata train shape: {}\", metadata_train.shape)\n",
    "print(\"metdata test shape: {}\", metadata_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create model\n",
    "clf = Sequential()\n",
    "#add model layers\n",
    "clf.add(Conv1D(16, kernel_size=5, activation=\"relu\", input_shape=(256, 12)))\n",
    "clf.add(Conv1D(32, kernel_size=5, activation=\"relu\"))\n",
    "clf.add(Flatten())\n",
    "clf.add(Dense(6, activation=\"softmax\", kernel_regularizer=keras.regularizers.l2(0.01)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3276 samples, validate on 819 samples\n",
      "Epoch 1/30\n",
      "3276/3276 [==============================] - 1s 388us/step - loss: 0.9541 - accuracy: 0.7237 - val_loss: 0.5229 - val_accuracy: 0.8620\n",
      "Epoch 2/30\n",
      "3276/3276 [==============================] - 1s 347us/step - loss: 0.4119 - accuracy: 0.9054 - val_loss: 0.3536 - val_accuracy: 0.9219\n",
      "Epoch 3/30\n",
      "3276/3276 [==============================] - 1s 347us/step - loss: 0.2742 - accuracy: 0.9490 - val_loss: 0.3072 - val_accuracy: 0.9280\n",
      "Epoch 4/30\n",
      "3276/3276 [==============================] - 1s 367us/step - loss: 0.2224 - accuracy: 0.9634 - val_loss: 0.2728 - val_accuracy: 0.9536\n",
      "Epoch 5/30\n",
      "3276/3276 [==============================] - 1s 325us/step - loss: 0.1746 - accuracy: 0.9731 - val_loss: 0.2278 - val_accuracy: 0.9548\n",
      "Epoch 6/30\n",
      "3276/3276 [==============================] - 1s 336us/step - loss: 0.1691 - accuracy: 0.9759 - val_loss: 0.2616 - val_accuracy: 0.9512\n",
      "Epoch 7/30\n",
      "3276/3276 [==============================] - 1s 339us/step - loss: 0.1422 - accuracy: 0.9832 - val_loss: 0.2022 - val_accuracy: 0.9573\n",
      "Epoch 8/30\n",
      "3276/3276 [==============================] - 1s 327us/step - loss: 0.1177 - accuracy: 0.9884 - val_loss: 0.2118 - val_accuracy: 0.9524\n",
      "Epoch 9/30\n",
      "3276/3276 [==============================] - 1s 329us/step - loss: 0.1077 - accuracy: 0.9890 - val_loss: 0.1833 - val_accuracy: 0.9597\n",
      "Epoch 10/30\n",
      "3276/3276 [==============================] - 1s 322us/step - loss: 0.1120 - accuracy: 0.9844 - val_loss: 0.1715 - val_accuracy: 0.9646\n",
      "Epoch 11/30\n",
      "3276/3276 [==============================] - 1s 345us/step - loss: 0.0998 - accuracy: 0.9902 - val_loss: 0.1719 - val_accuracy: 0.9658\n",
      "Epoch 12/30\n",
      "3276/3276 [==============================] - 1s 336us/step - loss: 0.1071 - accuracy: 0.9875 - val_loss: 0.2035 - val_accuracy: 0.9536\n",
      "Epoch 13/30\n",
      "3276/3276 [==============================] - 1s 329us/step - loss: 0.1096 - accuracy: 0.9872 - val_loss: 0.1536 - val_accuracy: 0.9768\n",
      "Epoch 14/30\n",
      "3276/3276 [==============================] - 1s 326us/step - loss: 0.1074 - accuracy: 0.9857 - val_loss: 0.1730 - val_accuracy: 0.9634\n",
      "Epoch 15/30\n",
      "3276/3276 [==============================] - 1s 325us/step - loss: 0.0942 - accuracy: 0.9893 - val_loss: 0.1529 - val_accuracy: 0.9731\n",
      "Epoch 16/30\n",
      "3276/3276 [==============================] - 1s 327us/step - loss: 0.0724 - accuracy: 0.9957 - val_loss: 0.1473 - val_accuracy: 0.9695\n",
      "Epoch 17/30\n",
      "3276/3276 [==============================] - 1s 333us/step - loss: 0.0632 - accuracy: 0.9966 - val_loss: 0.1417 - val_accuracy: 0.9670\n",
      "Epoch 18/30\n",
      "3276/3276 [==============================] - 1s 327us/step - loss: 0.0616 - accuracy: 0.9957 - val_loss: 0.1343 - val_accuracy: 0.9731\n",
      "Epoch 19/30\n",
      "3276/3276 [==============================] - 1s 337us/step - loss: 0.1577 - accuracy: 0.9683 - val_loss: 0.1631 - val_accuracy: 0.9731\n",
      "Epoch 20/30\n",
      "3276/3276 [==============================] - 1s 350us/step - loss: 0.0955 - accuracy: 0.9924 - val_loss: 0.1603 - val_accuracy: 0.9683\n",
      "Epoch 21/30\n",
      "3276/3276 [==============================] - 1s 342us/step - loss: 0.0698 - accuracy: 0.9973 - val_loss: 0.1037 - val_accuracy: 0.9829\n",
      "Epoch 22/30\n",
      "3276/3276 [==============================] - 1s 349us/step - loss: 0.0558 - accuracy: 0.9976 - val_loss: 0.1055 - val_accuracy: 0.9817\n",
      "Epoch 23/30\n",
      "3276/3276 [==============================] - 1s 328us/step - loss: 0.0528 - accuracy: 0.9969 - val_loss: 0.1271 - val_accuracy: 0.9707\n",
      "Epoch 24/30\n",
      "3276/3276 [==============================] - 1s 334us/step - loss: 0.0780 - accuracy: 0.9921 - val_loss: 0.1033 - val_accuracy: 0.9792\n",
      "Epoch 25/30\n",
      "3276/3276 [==============================] - 1s 334us/step - loss: 0.0547 - accuracy: 0.9976 - val_loss: 0.0900 - val_accuracy: 0.9817\n",
      "Epoch 26/30\n",
      "3276/3276 [==============================] - 1s 334us/step - loss: 0.0533 - accuracy: 0.9966 - val_loss: 0.0965 - val_accuracy: 0.9805\n",
      "Epoch 27/30\n",
      "3276/3276 [==============================] - 1s 320us/step - loss: 0.0666 - accuracy: 0.9921 - val_loss: 0.1065 - val_accuracy: 0.9780\n",
      "Epoch 28/30\n",
      "3276/3276 [==============================] - 1s 329us/step - loss: 0.1365 - accuracy: 0.9780 - val_loss: 0.1518 - val_accuracy: 0.9841\n",
      "Epoch 29/30\n",
      "3276/3276 [==============================] - 1s 334us/step - loss: 0.0997 - accuracy: 0.9930 - val_loss: 0.1174 - val_accuracy: 0.9866\n",
      "Epoch 30/30\n",
      "3276/3276 [==============================] - 1s 325us/step - loss: 0.0710 - accuracy: 0.9966 - val_loss: 0.0978 - val_accuracy: 0.9866\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x13cd281d0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "clf.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       1.00      1.00      1.00       298\n",
      "         jog       1.00      1.00      1.00       304\n",
      "         sit       1.00      1.00      1.00       788\n",
      "         std       1.00      1.00      1.00       722\n",
      "         ups       1.00      1.00      1.00       351\n",
      "         wlk       1.00      1.00      1.00       813\n",
      "\n",
      "    accuracy                           1.00      3276\n",
      "   macro avg       1.00      1.00      1.00      3276\n",
      "weighted avg       1.00      1.00      1.00      3276\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       1.00      0.93      0.97        61\n",
      "         jog       0.99      1.00      0.99        80\n",
      "         sit       1.00      1.00      1.00       216\n",
      "         std       1.00      1.00      1.00       178\n",
      "         ups       0.99      0.93      0.96        85\n",
      "         wlk       0.96      0.99      0.98       199\n",
      "\n",
      "    accuracy                           0.99       819\n",
      "   macro avg       0.99      0.98      0.98       819\n",
      "weighted avg       0.99      0.99      0.99       819\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(np.argmax(y_train, axis=1), np.argmax(clf.predict(X_train), axis=1), target_names=label_encoder.classes_))\n",
    "\n",
    "print(classification_report(np.argmax(y_test, axis=1), np.argmax(clf.predict(X_test), axis=1), target_names=label_encoder.classes_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create model\n",
    "conv = Sequential()\n",
    "# add model layers\n",
    "conv.add(Conv1D(16, kernel_size=5, activation=\"relu\", input_shape=(256, 12)))\n",
    "conv.add(Conv1D(32, kernel_size=5, activation=\"relu\"))\n",
    "conv.add(Flatten(name=\"coefs\"))\n",
    "\n",
    "metadata_input_tensor = Input(shape=(4, 1))\n",
    "metadata_input = Flatten(name=\"flatten\")(metadata_input_tensor)\n",
    "\n",
    "last_layer = conv.get_layer(\"coefs\").output\n",
    "x = concatenate([last_layer, metadata_input], axis=1)\n",
    "out = Dense(6, activation=\"softmax\")(Dropout(0.5)(x))\n",
    "\n",
    "clf = Model([conv.input, metadata_input_tensor], out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3276 samples, validate on 819 samples\n",
      "Epoch 1/30\n",
      "3276/3276 [==============================] - 2s 500us/step - loss: 1.0899 - accuracy: 0.6401 - val_loss: 0.4741 - val_accuracy: 0.8095\n",
      "Epoch 2/30\n",
      "3276/3276 [==============================] - 1s 398us/step - loss: 0.3710 - accuracy: 0.8712 - val_loss: 0.3171 - val_accuracy: 0.8901\n",
      "Epoch 3/30\n",
      "3276/3276 [==============================] - 1s 390us/step - loss: 0.2279 - accuracy: 0.9225 - val_loss: 0.1691 - val_accuracy: 0.9512\n",
      "Epoch 4/30\n",
      "3276/3276 [==============================] - 1s 384us/step - loss: 0.1652 - accuracy: 0.9460 - val_loss: 0.1597 - val_accuracy: 0.9487\n",
      "Epoch 5/30\n",
      "3276/3276 [==============================] - 1s 379us/step - loss: 0.1263 - accuracy: 0.9603 - val_loss: 0.1335 - val_accuracy: 0.9524\n",
      "Epoch 6/30\n",
      "3276/3276 [==============================] - 1s 394us/step - loss: 0.0842 - accuracy: 0.9731 - val_loss: 0.1293 - val_accuracy: 0.9621\n",
      "Epoch 7/30\n",
      "3276/3276 [==============================] - 1s 405us/step - loss: 0.0660 - accuracy: 0.9792 - val_loss: 0.1051 - val_accuracy: 0.9695\n",
      "Epoch 8/30\n",
      "3276/3276 [==============================] - 1s 421us/step - loss: 0.0521 - accuracy: 0.9844 - val_loss: 0.0827 - val_accuracy: 0.9768\n",
      "Epoch 9/30\n",
      "3276/3276 [==============================] - 1s 406us/step - loss: 0.0367 - accuracy: 0.9887 - val_loss: 0.0801 - val_accuracy: 0.9731\n",
      "Epoch 10/30\n",
      "3276/3276 [==============================] - 1s 401us/step - loss: 0.0317 - accuracy: 0.9896 - val_loss: 0.0946 - val_accuracy: 0.9731\n",
      "Epoch 11/30\n",
      "3276/3276 [==============================] - 1s 392us/step - loss: 0.0313 - accuracy: 0.9893 - val_loss: 0.0907 - val_accuracy: 0.9731\n",
      "Epoch 12/30\n",
      "3276/3276 [==============================] - 1s 401us/step - loss: 0.0252 - accuracy: 0.9936 - val_loss: 0.1030 - val_accuracy: 0.9744\n",
      "Epoch 13/30\n",
      "3276/3276 [==============================] - 1s 403us/step - loss: 0.0211 - accuracy: 0.9921 - val_loss: 0.1232 - val_accuracy: 0.9670\n",
      "Epoch 14/30\n",
      "3276/3276 [==============================] - 1s 406us/step - loss: 0.0298 - accuracy: 0.9893 - val_loss: 0.1300 - val_accuracy: 0.9695\n",
      "Epoch 15/30\n",
      "3276/3276 [==============================] - 1s 430us/step - loss: 0.0220 - accuracy: 0.9921 - val_loss: 0.1008 - val_accuracy: 0.9744\n",
      "Epoch 16/30\n",
      "3276/3276 [==============================] - 1s 400us/step - loss: 0.0118 - accuracy: 0.9982 - val_loss: 0.1052 - val_accuracy: 0.9744\n",
      "Epoch 17/30\n",
      "3276/3276 [==============================] - 1s 398us/step - loss: 0.0184 - accuracy: 0.9954 - val_loss: 0.1385 - val_accuracy: 0.9683\n",
      "Epoch 18/30\n",
      "3276/3276 [==============================] - 1s 424us/step - loss: 0.0313 - accuracy: 0.9911 - val_loss: 0.0831 - val_accuracy: 0.9817\n",
      "Epoch 19/30\n",
      "3276/3276 [==============================] - 1s 416us/step - loss: 0.0098 - accuracy: 0.9973 - val_loss: 0.0840 - val_accuracy: 0.9792\n",
      "Epoch 20/30\n",
      "3276/3276 [==============================] - 1s 396us/step - loss: 0.0148 - accuracy: 0.9948 - val_loss: 0.1571 - val_accuracy: 0.9646\n",
      "Epoch 21/30\n",
      "3276/3276 [==============================] - 1s 398us/step - loss: 0.0303 - accuracy: 0.9881 - val_loss: 0.1010 - val_accuracy: 0.9768\n",
      "Epoch 22/30\n",
      "3276/3276 [==============================] - 1s 391us/step - loss: 0.0219 - accuracy: 0.9927 - val_loss: 0.0860 - val_accuracy: 0.9780\n",
      "Epoch 23/30\n",
      "3276/3276 [==============================] - 1s 389us/step - loss: 0.0053 - accuracy: 0.9982 - val_loss: 0.0893 - val_accuracy: 0.9829\n",
      "Epoch 24/30\n",
      "3276/3276 [==============================] - 1s 404us/step - loss: 0.0059 - accuracy: 0.9979 - val_loss: 0.0865 - val_accuracy: 0.9792\n",
      "Epoch 25/30\n",
      "3276/3276 [==============================] - 1s 430us/step - loss: 0.0045 - accuracy: 0.9991 - val_loss: 0.0782 - val_accuracy: 0.9817\n",
      "Epoch 26/30\n",
      "3276/3276 [==============================] - 1s 402us/step - loss: 0.0060 - accuracy: 0.9988 - val_loss: 0.1112 - val_accuracy: 0.9817\n",
      "Epoch 27/30\n",
      "3276/3276 [==============================] - 1s 395us/step - loss: 0.0120 - accuracy: 0.9957 - val_loss: 0.1016 - val_accuracy: 0.9805\n",
      "Epoch 28/30\n",
      "3276/3276 [==============================] - 1s 406us/step - loss: 0.0212 - accuracy: 0.9933 - val_loss: 0.1188 - val_accuracy: 0.9756\n",
      "Epoch 29/30\n",
      "3276/3276 [==============================] - 1s 400us/step - loss: 0.0146 - accuracy: 0.9960 - val_loss: 0.1081 - val_accuracy: 0.9817\n",
      "Epoch 30/30\n",
      "3276/3276 [==============================] - 1s 410us/step - loss: 0.0076 - accuracy: 0.9976 - val_loss: 0.1003 - val_accuracy: 0.9841\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x13efc1e10>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "clf.fit([X_train, metadata_train], y_train, validation_data=([X_test, metadata_test], y_test), epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       1.00      1.00      1.00       298\n",
      "         jog       1.00      1.00      1.00       304\n",
      "         sit       1.00      1.00      1.00       788\n",
      "         std       1.00      1.00      1.00       722\n",
      "         ups       1.00      1.00      1.00       351\n",
      "         wlk       1.00      1.00      1.00       813\n",
      "\n",
      "    accuracy                           1.00      3276\n",
      "   macro avg       1.00      1.00      1.00      3276\n",
      "weighted avg       1.00      1.00      1.00      3276\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         dws       1.00      0.95      0.97        61\n",
      "         jog       1.00      0.97      0.99        80\n",
      "         sit       1.00      1.00      1.00       216\n",
      "         std       1.00      1.00      1.00       178\n",
      "         ups       0.93      0.94      0.94        85\n",
      "         wlk       0.97      0.98      0.98       199\n",
      "\n",
      "    accuracy                           0.98       819\n",
      "   macro avg       0.98      0.98      0.98       819\n",
      "weighted avg       0.98      0.98      0.98       819\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(np.argmax(y_train, axis=1), np.argmax(clf.predict([X_train, metadata_train]), axis=1), target_names=label_encoder.classes_))\n",
    "\n",
    "print(classification_report(np.argmax(y_test, axis=1), np.argmax(clf.predict([X_test, metadata_test]), axis=1), target_names=label_encoder.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.argmax(y_test, axis=1)\n",
    "correct = (y_true == np.argmax(clf.predict([X_test, metadata_test]), axis=1)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from statsmodels.stats.proportion import proportion_confint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9841269841269841 +- 0.00855975699251288\n"
     ]
    }
   ],
   "source": [
    "inf, sup = proportion_confint(correct, len(y_true))\n",
    "print(\"Accuracy: {} +- {}\".format(correct / len(y_true), (sup - inf) / 2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}