ieee-cis-fraud-random-forest-0-90-lb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- To reduce CPU load, we are using kernel output from this kernel : https://www.kaggle.com/mjbahmani/reducing-memory-size-for-ieee\n",
    "- We are using some fastai v0.7 functions for preprocessing etc, hence I've added the structured.py file as a utility script : https://www.kaggle.com/priteshshrivastava/fastai-structured\n",
    "- This kernel is focussed on model interpretation using Permutation Feature Importance, Partial Dependence Plots and SHAP values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/six.py:31: DeprecationWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
      "  \"(https://pypi.org/project/six/).\", DeprecationWarning)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n",
      "Using TensorFlow backend.\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ieee-fraud-detection', 'reducing-memory-size-for-ieee']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from pandas_summary import DataFrameSummary\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "from pandas_summary import DataFrameSummary\n",
    "from matplotlib import pyplot as plt\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn import tree\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "import graphviz\n",
    "import re\n",
    "\n",
    "import shap\n",
    "import eli5\n",
    "from eli5.sklearn import PermutationImportance\n",
    "from pdpbox import pdp, get_dataset, info_plots\n",
    "\n",
    "import IPython\n",
    "from IPython.display import display\n",
    "print(os.listdir(\"../input/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>isFraud</th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>card6</th>\n",
       "      <th>...</th>\n",
       "      <th>id_31</th>\n",
       "      <th>id_32</th>\n",
       "      <th>id_33</th>\n",
       "      <th>id_34</th>\n",
       "      <th>id_35</th>\n",
       "      <th>id_36</th>\n",
       "      <th>id_37</th>\n",
       "      <th>id_38</th>\n",
       "      <th>DeviceType</th>\n",
       "      <th>DeviceInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>86400</td>\n",
       "      <td>68.5</td>\n",
       "      <td>W</td>\n",
       "      <td>13926</td>\n",
       "      <td>99</td>\n",
       "      <td>150</td>\n",
       "      <td>discover</td>\n",
       "      <td>142</td>\n",
       "      <td>credit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>255</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>86401</td>\n",
       "      <td>29.0</td>\n",
       "      <td>W</td>\n",
       "      <td>2755</td>\n",
       "      <td>404</td>\n",
       "      <td>150</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>102</td>\n",
       "      <td>credit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>255</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>86469</td>\n",
       "      <td>59.0</td>\n",
       "      <td>W</td>\n",
       "      <td>4663</td>\n",
       "      <td>490</td>\n",
       "      <td>150</td>\n",
       "      <td>visa</td>\n",
       "      <td>166</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>255</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>86499</td>\n",
       "      <td>50.0</td>\n",
       "      <td>W</td>\n",
       "      <td>18132</td>\n",
       "      <td>567</td>\n",
       "      <td>150</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>117</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>255</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>86506</td>\n",
       "      <td>50.0</td>\n",
       "      <td>H</td>\n",
       "      <td>4497</td>\n",
       "      <td>514</td>\n",
       "      <td>150</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>102</td>\n",
       "      <td>credit</td>\n",
       "      <td>...</td>\n",
       "      <td>samsung browser 6.2</td>\n",
       "      <td>32</td>\n",
       "      <td>2220x1080</td>\n",
       "      <td>match_status:2</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>T</td>\n",
       "      <td>mobile</td>\n",
       "      <td>SAMSUNG SM-G892A Build/NRD90M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 433 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   isFraud  TransactionDT  TransactionAmt ProductCD  card1  card2  card3  \\\n",
       "0        0          86400            68.5         W  13926     99    150   \n",
       "1        0          86401            29.0         W   2755    404    150   \n",
       "2        0          86469            59.0         W   4663    490    150   \n",
       "3        0          86499            50.0         W  18132    567    150   \n",
       "4        0          86506            50.0         H   4497    514    150   \n",
       "\n",
       "        card4  card5   card6  ...                id_31  id_32      id_33  \\\n",
       "0    discover    142  credit  ...                  NaN    255        NaN   \n",
       "1  mastercard    102  credit  ...                  NaN    255        NaN   \n",
       "2        visa    166   debit  ...                  NaN    255        NaN   \n",
       "3  mastercard    117   debit  ...                  NaN    255        NaN   \n",
       "4  mastercard    102  credit  ...  samsung browser 6.2     32  2220x1080   \n",
       "\n",
       "            id_34 id_35 id_36  id_37  id_38  DeviceType  \\\n",
       "0             NaN   NaN   NaN    NaN    NaN         NaN   \n",
       "1             NaN   NaN   NaN    NaN    NaN         NaN   \n",
       "2             NaN   NaN   NaN    NaN    NaN         NaN   \n",
       "3             NaN   NaN   NaN    NaN    NaN         NaN   \n",
       "4  match_status:2     T     F      T      T      mobile   \n",
       "\n",
       "                      DeviceInfo  \n",
       "0                            NaN  \n",
       "1                            NaN  \n",
       "2                            NaN  \n",
       "3                            NaN  \n",
       "4  SAMSUNG SM-G892A Build/NRD90M  \n",
       "\n",
       "[5 rows x 433 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"../input/reducing-memory-size-for-ieee/train.csv\")\n",
    "test_df = pd.read_csv(\"../input/reducing-memory-size-for-ieee/test.csv\")\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>card6</th>\n",
       "      <th>addr1</th>\n",
       "      <th>...</th>\n",
       "      <th>id_31</th>\n",
       "      <th>id_32</th>\n",
       "      <th>id_33</th>\n",
       "      <th>id_34</th>\n",
       "      <th>id_35</th>\n",
       "      <th>id_36</th>\n",
       "      <th>id_37</th>\n",
       "      <th>id_38</th>\n",
       "      <th>DeviceType</th>\n",
       "      <th>DeviceInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18403224</td>\n",
       "      <td>31.95</td>\n",
       "      <td>W</td>\n",
       "      <td>10409</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>visa</td>\n",
       "      <td>226</td>\n",
       "      <td>debit</td>\n",
       "      <td>170</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18403263</td>\n",
       "      <td>49.00</td>\n",
       "      <td>W</td>\n",
       "      <td>4272</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>visa</td>\n",
       "      <td>226</td>\n",
       "      <td>debit</td>\n",
       "      <td>299</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18403310</td>\n",
       "      <td>171.00</td>\n",
       "      <td>W</td>\n",
       "      <td>4476</td>\n",
       "      <td>574</td>\n",
       "      <td>150</td>\n",
       "      <td>visa</td>\n",
       "      <td>226</td>\n",
       "      <td>debit</td>\n",
       "      <td>472</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18403310</td>\n",
       "      <td>284.95</td>\n",
       "      <td>W</td>\n",
       "      <td>10989</td>\n",
       "      <td>360</td>\n",
       "      <td>150</td>\n",
       "      <td>visa</td>\n",
       "      <td>166</td>\n",
       "      <td>debit</td>\n",
       "      <td>205</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18403317</td>\n",
       "      <td>67.95</td>\n",
       "      <td>W</td>\n",
       "      <td>18018</td>\n",
       "      <td>452</td>\n",
       "      <td>150</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>117</td>\n",
       "      <td>debit</td>\n",
       "      <td>264</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionDT  TransactionAmt ProductCD  card1  card2  card3       card4  \\\n",
       "0       18403224           31.95         W  10409    111    150        visa   \n",
       "1       18403263           49.00         W   4272    111    150        visa   \n",
       "2       18403310          171.00         W   4476    574    150        visa   \n",
       "3       18403310          284.95         W  10989    360    150        visa   \n",
       "4       18403317           67.95         W  18018    452    150  mastercard   \n",
       "\n",
       "   card5  card6  addr1  ...  id_31  id_32  id_33 id_34 id_35  id_36  id_37  \\\n",
       "0    226  debit    170  ...    NaN      7    NaN   NaN   NaN    NaN    NaN   \n",
       "1    226  debit    299  ...    NaN      7    NaN   NaN   NaN    NaN    NaN   \n",
       "2    226  debit    472  ...    NaN      7    NaN   NaN   NaN    NaN    NaN   \n",
       "3    166  debit    205  ...    NaN      7    NaN   NaN   NaN    NaN    NaN   \n",
       "4    117  debit    264  ...    NaN      7    NaN   NaN   NaN    NaN    NaN   \n",
       "\n",
       "   id_38  DeviceType  DeviceInfo  \n",
       "0    NaN         NaN         NaN  \n",
       "1    NaN         NaN         NaN  \n",
       "2    NaN         NaN         NaN  \n",
       "3    NaN         NaN         NaN  \n",
       "4    NaN         NaN         NaN  \n",
       "\n",
       "[5 rows x 432 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll just use a Random Forest Classifier. For that, we need to convert all columns to numeric type. But there are some categorical variables too."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "import fastai_structured   ## Adding structured.py from fastai v0.7 as a utility script to the kernel\n",
    "fastai_structured.train_cats(train_df)\n",
    "fastai_structured.apply_cats(test_df, train_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll replace categories with their numeric codes, handle missing continuous values, and split the dependent variable into a separate variable. Fastai to the rescue again !!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>card6</th>\n",
       "      <th>addr1</th>\n",
       "      <th>...</th>\n",
       "      <th>id_31</th>\n",
       "      <th>id_32</th>\n",
       "      <th>id_33</th>\n",
       "      <th>id_34</th>\n",
       "      <th>id_35</th>\n",
       "      <th>id_36</th>\n",
       "      <th>id_37</th>\n",
       "      <th>id_38</th>\n",
       "      <th>DeviceType</th>\n",
       "      <th>DeviceInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>86400</td>\n",
       "      <td>68.5</td>\n",
       "      <td>5</td>\n",
       "      <td>13926</td>\n",
       "      <td>99</td>\n",
       "      <td>150</td>\n",
       "      <td>2</td>\n",
       "      <td>142</td>\n",
       "      <td>2</td>\n",
       "      <td>315</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>255</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>86401</td>\n",
       "      <td>29.0</td>\n",
       "      <td>5</td>\n",
       "      <td>2755</td>\n",
       "      <td>404</td>\n",
       "      <td>150</td>\n",
       "      <td>3</td>\n",
       "      <td>102</td>\n",
       "      <td>2</td>\n",
       "      <td>325</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>255</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>86469</td>\n",
       "      <td>59.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4663</td>\n",
       "      <td>490</td>\n",
       "      <td>150</td>\n",
       "      <td>4</td>\n",
       "      <td>166</td>\n",
       "      <td>3</td>\n",
       "      <td>330</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>255</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>86499</td>\n",
       "      <td>50.0</td>\n",
       "      <td>5</td>\n",
       "      <td>18132</td>\n",
       "      <td>567</td>\n",
       "      <td>150</td>\n",
       "      <td>3</td>\n",
       "      <td>117</td>\n",
       "      <td>3</td>\n",
       "      <td>476</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>255</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>86506</td>\n",
       "      <td>50.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4497</td>\n",
       "      <td>514</td>\n",
       "      <td>150</td>\n",
       "      <td>3</td>\n",
       "      <td>102</td>\n",
       "      <td>2</td>\n",
       "      <td>420</td>\n",
       "      <td>...</td>\n",
       "      <td>124</td>\n",
       "      <td>32</td>\n",
       "      <td>165</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>955</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionDT  TransactionAmt  ProductCD  card1  card2  card3  card4  \\\n",
       "0          86400            68.5          5  13926     99    150      2   \n",
       "1          86401            29.0          5   2755    404    150      3   \n",
       "2          86469            59.0          5   4663    490    150      4   \n",
       "3          86499            50.0          5  18132    567    150      3   \n",
       "4          86506            50.0          2   4497    514    150      3   \n",
       "\n",
       "   card5  card6  addr1  ...  id_31  id_32  id_33  id_34  id_35  id_36  id_37  \\\n",
       "0    142      2    315  ...      0    255      0      0      0      0      0   \n",
       "1    102      2    325  ...      0    255      0      0      0      0      0   \n",
       "2    166      3    330  ...      0    255      0      0      0      0      0   \n",
       "3    117      3    476  ...      0    255      0      0      0      0      0   \n",
       "4    102      2    420  ...    124     32    165      4      2      1      2   \n",
       "\n",
       "   id_38  DeviceType  DeviceInfo  \n",
       "0      0           0           0  \n",
       "1      0           0           0  \n",
       "2      0           0           0  \n",
       "3      0           0           0  \n",
       "4      2           2         955  \n",
       "\n",
       "[5 rows x 432 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nas = {}\n",
    "df_trn, y_trn, nas = fastai_structured.proc_df(train_df, 'isFraud', na_dict=nas)   ## Avoid creating NA columns as total cols may not match later\n",
    "df_test, _, _ = fastai_structured.proc_df(test_df, na_dict=nas)\n",
    "df_trn.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>card6</th>\n",
       "      <th>addr1</th>\n",
       "      <th>...</th>\n",
       "      <th>id_31</th>\n",
       "      <th>id_32</th>\n",
       "      <th>id_33</th>\n",
       "      <th>id_34</th>\n",
       "      <th>id_35</th>\n",
       "      <th>id_36</th>\n",
       "      <th>id_37</th>\n",
       "      <th>id_38</th>\n",
       "      <th>DeviceType</th>\n",
       "      <th>DeviceInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18403224</td>\n",
       "      <td>31.95</td>\n",
       "      <td>5</td>\n",
       "      <td>10409</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>4</td>\n",
       "      <td>226</td>\n",
       "      <td>3</td>\n",
       "      <td>170</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18403263</td>\n",
       "      <td>49.00</td>\n",
       "      <td>5</td>\n",
       "      <td>4272</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>4</td>\n",
       "      <td>226</td>\n",
       "      <td>3</td>\n",
       "      <td>299</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18403310</td>\n",
       "      <td>171.00</td>\n",
       "      <td>5</td>\n",
       "      <td>4476</td>\n",
       "      <td>574</td>\n",
       "      <td>150</td>\n",
       "      <td>4</td>\n",
       "      <td>226</td>\n",
       "      <td>3</td>\n",
       "      <td>472</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18403310</td>\n",
       "      <td>284.95</td>\n",
       "      <td>5</td>\n",
       "      <td>10989</td>\n",
       "      <td>360</td>\n",
       "      <td>150</td>\n",
       "      <td>4</td>\n",
       "      <td>166</td>\n",
       "      <td>3</td>\n",
       "      <td>205</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18403317</td>\n",
       "      <td>67.95</td>\n",
       "      <td>5</td>\n",
       "      <td>18018</td>\n",
       "      <td>452</td>\n",
       "      <td>150</td>\n",
       "      <td>3</td>\n",
       "      <td>117</td>\n",
       "      <td>3</td>\n",
       "      <td>264</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionDT  TransactionAmt  ProductCD  card1  card2  card3  card4  \\\n",
       "0       18403224           31.95          5  10409    111    150      4   \n",
       "1       18403263           49.00          5   4272    111    150      4   \n",
       "2       18403310          171.00          5   4476    574    150      4   \n",
       "3       18403310          284.95          5  10989    360    150      4   \n",
       "4       18403317           67.95          5  18018    452    150      3   \n",
       "\n",
       "   card5  card6  addr1  ...  id_31  id_32  id_33  id_34  id_35  id_36  id_37  \\\n",
       "0    226      3    170  ...      0      7      0      0      0      0      0   \n",
       "1    226      3    299  ...      0      7      0      0      0      0      0   \n",
       "2    226      3    472  ...      0      7      0      0      0      0      0   \n",
       "3    166      3    205  ...      0      7      0      0      0      0      0   \n",
       "4    117      3    264  ...      0      7      0      0      0      0      0   \n",
       "\n",
       "   id_38  DeviceType  DeviceInfo  \n",
       "0      0           0           0  \n",
       "1      0           0           0  \n",
       "2      0           0           0  \n",
       "3      0           0           0  \n",
       "4      0           0           0  \n",
       "\n",
       "[5 rows x 432 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_df, test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## To handle imbalanced datasets, we'll use [resampling](https://www.kaggle.com/shahules/tackling-class-imbalance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "ran=RandomUnderSampler(return_indices=True) ##intialize to return indices of dropped rows\n",
    "df_trn_sm,y_trn_sm,dropped = ran.fit_sample(df_trn,y_trn)\n",
    "\n",
    "#print(\"The number of removed indices are \",len(dropped))\n",
    "#plot_2d_space(X_rs,y_rs,X,y,'Random under sampling')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the data into training and validation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, val_X, train_y, val_y = train_test_split(df_trn_sm, y_trn_sm, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining function to calculate the evaluation metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "def print_score(m):\n",
    "    res = [roc_auc_score(m.predict(train_X), train_y), roc_auc_score(m.predict(val_X), val_y)]\n",
    "    print(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now pass this processed data frame to Random Forest Classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "##To reduce CPU load, and for faster iteration\n",
    "fastai_structured.set_rf_samples(200000)\n",
    "del df_trn, y_trn, df_trn_sm, y_trn_sm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initially, let's just fit a single decision tree to visualize it properly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 12.6 µs\n",
      "[0.7352222765736182, 0.7282575645287098]\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "m = RandomForestClassifier(n_estimators=1, min_samples_leaf=5, max_depth = 3) ## Use all CPUs available\n",
    "m.fit(train_X, train_y)\n",
    "\n",
    "print_score(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "def draw_tree(t, df, size=10, ratio=0.6, precision=0):\n",
    "    \"\"\" Draws a representation of a random forest in IPython.\n",
    "    Parameters:\n",
    "    -----------\n",
    "    t: The tree you wish to draw\n",
    "    df: The data used to train the tree. This is used to get the names of the features.\n",
    "    \"\"\"\n",
    "    s=export_graphviz(t, out_file=None, feature_names=df.columns, filled=True,\n",
    "                      special_characters=True, rotate=True, precision=precision)\n",
    "    IPython.display.display(graphviz.Source(re.sub('Tree {',\n",
    "       f'Tree {{ size={size}; ratio={ratio}', s)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#draw_tree(m.estimators_[0], train_X, precision=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A single decision tree did not perform so badly. You can read more about the gini impurity metric [here](https://en.wikipedia.org/wiki/Decision_tree_learning#Gini_impurity).\n",
    "\n",
    "Now, let's bag a collection of trees to create a random forest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 11.2 µs\n",
      "[0.9000695984284416, 0.8350216935360076]\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "m = RandomForestClassifier(n_estimators=30, min_samples_leaf=20, max_features=0.7, \n",
    "                                n_jobs=-1, oob_score=True) ## Use all CPUs available\n",
    "m.fit(train_X, train_y)\n",
    "\n",
    "print_score(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionID</th>\n",
       "      <th>isFraud</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3663549</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3663550</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3663551</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3663552</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3663553</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionID  isFraud\n",
       "0        3663549      0.5\n",
       "1        3663550      0.5\n",
       "2        3663551      0.5\n",
       "3        3663552      0.5\n",
       "4        3663553      0.5"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## pred = m.predict(df_test)          ## Gets an AUC of ~0.8\n",
    "pred = m.predict_proba(df_test)[:,1]  ## Gets an AUC of ~0.9\n",
    "submission = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission['isFraud'] = pred   \n",
    "submission.to_csv('rf_submission_vf.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}