5-sklearn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "federal-choir",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/saumik/mambaforge/envs/random2/lib/python3.9/site-packages/scipy/__init__.py:155: UserWarning: A NumPy version >=1.18.5 and <1.25.0 is required for this version of SciPy (detected version 1.25.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib\n",
    "\n",
    "import sklearn\n",
    "from sklearn import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "sns.set_theme()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sklearn.ensemble.RandomForestClassifier\n",
    "# sklearn.ensemble.RandomForestRegressor\n",
    "\n",
    "# sklearn.linear_model.LogisticRegression\n",
    "# sklearn.linear_model.LinearRegression\n",
    "\n",
    "# sklearn.tree.DecisionTreeClassifier\n",
    "# sklearn.tree.DecisionTreeRegressor\n",
    "\n",
    "# sklearn.svm.LinearSVC\n",
    "# sklearn.svm.LinearSVR\n",
    "\n",
    "# sklearn.neighbors.KNeighborsClassifier\n",
    "# sklearn.neighbors.KNeighborsRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>adult_male</th>\n",
       "      <th>alone</th>\n",
       "      <th>sex_male</th>\n",
       "      <th>embarked_Q</th>\n",
       "      <th>...</th>\n",
       "      <th>who_man</th>\n",
       "      <th>who_woman</th>\n",
       "      <th>deck_B</th>\n",
       "      <th>deck_C</th>\n",
       "      <th>deck_D</th>\n",
       "      <th>deck_E</th>\n",
       "      <th>deck_F</th>\n",
       "      <th>deck_G</th>\n",
       "      <th>embark_town_Queenstown</th>\n",
       "      <th>embark_town_Southampton</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.826913</td>\n",
       "      <td>-0.592148</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.502163</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.565228</td>\n",
       "      <td>0.638430</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>0.786404</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.826913</td>\n",
       "      <td>-0.284503</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.488580</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.565228</td>\n",
       "      <td>0.407697</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>0.420494</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.826913</td>\n",
       "      <td>0.407697</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.486064</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.369158</td>\n",
       "      <td>-0.207592</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.386454</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.565228</td>\n",
       "      <td>-0.822881</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.044356</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>888</th>\n",
       "      <td>0</td>\n",
       "      <td>0.826913</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>2.007806</td>\n",
       "      <td>-0.176164</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.565228</td>\n",
       "      <td>-0.284503</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.044356</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>0</td>\n",
       "      <td>0.826913</td>\n",
       "      <td>0.176964</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.492101</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>891 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     survived    pclass       age     sibsp     parch      fare  adult_male  \\\n",
       "0           0  0.826913 -0.592148  0.432550 -0.473408 -0.502163        True   \n",
       "1           1 -1.565228  0.638430  0.432550 -0.473408  0.786404       False   \n",
       "2           1  0.826913 -0.284503 -0.474279 -0.473408 -0.488580       False   \n",
       "3           1 -1.565228  0.407697  0.432550 -0.473408  0.420494       False   \n",
       "4           0  0.826913  0.407697 -0.474279 -0.473408 -0.486064        True   \n",
       "..        ...       ...       ...       ...       ...       ...         ...   \n",
       "886         0 -0.369158 -0.207592 -0.474279 -0.473408 -0.386454        True   \n",
       "887         1 -1.565228 -0.822881 -0.474279 -0.473408 -0.044356       False   \n",
       "888         0  0.826913  0.000000  0.432550  2.007806 -0.176164       False   \n",
       "889         1 -1.565228 -0.284503 -0.474279 -0.473408 -0.044356        True   \n",
       "890         0  0.826913  0.176964 -0.474279 -0.473408 -0.492101        True   \n",
       "\n",
       "     alone  sex_male  embarked_Q  ...  who_man  who_woman  deck_B  deck_C  \\\n",
       "0    False      True       False  ...     True      False   False    True   \n",
       "1    False     False       False  ...    False       True   False    True   \n",
       "2     True     False       False  ...    False       True   False    True   \n",
       "3    False     False       False  ...    False       True   False    True   \n",
       "4     True      True       False  ...     True      False   False    True   \n",
       "..     ...       ...         ...  ...      ...        ...     ...     ...   \n",
       "886   True      True       False  ...     True      False   False    True   \n",
       "887   True     False       False  ...    False       True    True   False   \n",
       "888  False     False       False  ...    False       True   False    True   \n",
       "889   True      True       False  ...     True      False   False    True   \n",
       "890   True      True        True  ...     True      False   False    True   \n",
       "\n",
       "     deck_D  deck_E  deck_F  deck_G  embark_town_Queenstown  \\\n",
       "0     False   False   False   False                   False   \n",
       "1     False   False   False   False                   False   \n",
       "2     False   False   False   False                   False   \n",
       "3     False   False   False   False                   False   \n",
       "4     False   False   False   False                   False   \n",
       "..      ...     ...     ...     ...                     ...   \n",
       "886   False   False   False   False                   False   \n",
       "887   False   False   False   False                   False   \n",
       "888   False   False   False   False                   False   \n",
       "889   False   False   False   False                   False   \n",
       "890   False   False   False   False                    True   \n",
       "\n",
       "     embark_town_Southampton  \n",
       "0                       True  \n",
       "1                      False  \n",
       "2                       True  \n",
       "3                       True  \n",
       "4                       True  \n",
       "..                       ...  \n",
       "886                     True  \n",
       "887                     True  \n",
       "888                     True  \n",
       "889                    False  \n",
       "890                    False  \n",
       "\n",
       "[891 rows x 23 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load data and preprocess (taken from 4-data_cleaning.ipynb)\n",
    "df = pd.read_csv('titanic.csv')\n",
    "df = df.drop(columns='alive')\n",
    "df.fillna(df.mean(numeric_only=True), inplace=True) # Fill numeric NaNs with column mean\n",
    "df.fillna(df.mode().iloc[0], inplace=True) # Fill nonnumeric NaNs with column mode\n",
    "numeric_cols = df.select_dtypes(include=['number']).columns.drop('survived')\n",
    "df[numeric_cols] = df[numeric_cols].apply(lambda x: (x-x.mean())/x.std()) # Standardize all numeric columns except Y\n",
    "df = pd.get_dummies(df, columns=df.columns[df.dtypes=='object'], drop_first=True) # OneHotEncode all nonnumeric columns\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((891,), (891, 22))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y, X = df.survived, df.drop(columns='survived')\n",
    "Y, X = Y.values, X.values.astype(float)\n",
    "Y.shape, X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoring = {'acc': 'accuracy',\n",
    "           'f1': 'f1',\n",
    "           'auc': 'roc_auc'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([0.00233912, 0.00128794, 0.00090885, 0.00068688, 0.00070119]),\n",
       " 'score_time': array([0.20885992, 0.00809097, 0.00847101, 0.0085721 , 0.00929904]),\n",
       " 'test_acc': array([0.79888268, 0.78651685, 0.82022472, 0.83146067, 0.84831461]),\n",
       " 'test_f1': array([0.71875   , 0.71641791, 0.75384615, 0.76190476, 0.79699248]),\n",
       " 'test_auc': array([0.84057971, 0.80040107, 0.86898396, 0.87486631, 0.88106635])}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'fit_time': 0.0011847972869873046,\n",
       " 'score_time': 0.04865860939025879,\n",
       " 'test_acc': 0.817079907099366,\n",
       " 'test_f1': 0.7495822614803369,\n",
       " 'test_auc': 0.8531794789206977}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "estimator = sklearn.neighbors.KNeighborsClassifier()\n",
    "res = sklearn.model_selection.cross_validate(estimator, X=X, y=Y, scoring=scoring)\n",
    "display(res)\n",
    "display({k:v.mean() for k,v in res.items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional, reduce features by performing PCA\n",
    "# pca = sklearn.decomposition.PCA(n_components=10)\n",
    "# newX = pca.fit_transform(X)\n",
    "\n",
    "# estimator = sklearn.neighbors.KNeighborsClassifier()\n",
    "# res = sklearn.model_selection.cross_validate(estimator, X=newX, y=Y)\n",
    "# display(res)\n",
    "# display({k:v.mean() for k,v in res.items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([0.235569  , 0.20223498, 0.2073679 , 0.20653319, 0.20857406]),\n",
       " 'score_time': array([0.02844596, 0.02703404, 0.02702999, 0.02668285, 0.02802801]),\n",
       " 'test_acc': array([0.77653631, 0.78089888, 0.84269663, 0.75280899, 0.85955056]),\n",
       " 'test_f1': array([0.71428571, 0.71111111, 0.78787879, 0.64516129, 0.81751825]),\n",
       " 'test_auc': array([0.83274045, 0.80086898, 0.89745989, 0.84625668, 0.89110491])}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'fit_time': 0.21205582618713378,\n",
       " 'score_time': 0.027444171905517577,\n",
       " 'test_acc': 0.8024982738057874,\n",
       " 'test_f1': 0.7351910303546753,\n",
       " 'test_auc': 0.8536861831435261}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "estimator = sklearn.ensemble.RandomForestClassifier()\n",
    "res = sklearn.model_selection.cross_validate(estimator, X=X, y=Y, scoring=scoring)\n",
    "display(res)\n",
    "display({k:v.mean() for k,v in res.items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = sklearn.neighbors.KNeighborsClassifier()\n",
    "estimator.fit(X,Y);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}