synthetic_sdoh_feature_selection_limit_feature_class.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e01fbc7d",
   "metadata": {},
   "source": [
    "# Overview\n",
    "The purpose of this analysis is to identify which features in the readmissions dataset contribute most to the target variable (Readmitted within 30 days). The Chi-square test determines if there is a significant relationship between the input variables and output variable. The SelectKBest function selects the K number of top features (k=27) based on the Chi-square score.<br>\n",
    "This analysis uses the CSV file produced by the synthetic_feature_selection_input Jupyter Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c9d6061",
   "metadata": {},
   "source": [
    "# Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b2d8e58",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:25.811010Z",
     "start_time": "2023-04-18T14:22:23.345974Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:90% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "from matplotlib.pyplot import figure\n",
    "from matplotlib import rcParams\n",
    "matplotlib.rcParams['figure.figsize'] = (12,8)\n",
    "from sklearn.metrics import confusion_matrix as cm\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
    "import time\n",
    "import math\n",
    "\n",
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:90% !important; }</style>\"))\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "from sklearn.feature_selection import chi2, SelectKBest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5c33ad6",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b14dc8ae",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.191032Z",
     "start_time": "2023-04-18T14:22:25.813009Z"
    }
   },
   "outputs": [],
   "source": [
    "# load data\n",
    "synthetic_feature_selection = pd.read_csv('data/synthetic_feature_selection_input.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f8ee729f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.539078Z",
     "start_time": "2023-04-18T14:22:27.197032Z"
    }
   },
   "outputs": [],
   "source": [
    "# categorize age group columns and assign to values 1-5.\n",
    "# np.where(condition, value if true, value if false)\n",
    "synthetic_feature_selection['AGE_GROUP_0-64'] = np.where(synthetic_feature_selection['AGE_GROUP_0-64']==0,0,1)\n",
    "synthetic_feature_selection['AGE_GROUP_65-69'] = np.where(synthetic_feature_selection['AGE_GROUP_65-69']==0,0,2)\n",
    "synthetic_feature_selection['AGE_GROUP_70-74'] = np.where(synthetic_feature_selection['AGE_GROUP_70-74']==0,0,3)\n",
    "synthetic_feature_selection['AGE_GROUP_75-79'] = np.where(synthetic_feature_selection['AGE_GROUP_75-79']==0,0,4)\n",
    "synthetic_feature_selection['AGE_GROUP_80-max'] = np.where(synthetic_feature_selection['AGE_GROUP_80-max']==0,0,5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5715cd17",
   "metadata": {},
   "source": [
    "# Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fb1d7499",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.555036Z",
     "start_time": "2023-04-18T14:22:27.541054Z"
    }
   },
   "outputs": [],
   "source": [
    "# define function that iterates through each row of age group columns\n",
    "def label_age(row):\n",
    "    if row['AGE_GROUP_0-64']==1:\n",
    "        return '1'\n",
    "    if row['AGE_GROUP_65-69']==2:\n",
    "        return '2'\n",
    "    if row['AGE_GROUP_70-74']==3:\n",
    "        return '3'\n",
    "    if row['AGE_GROUP_75-79']==4:\n",
    "        return '4'\n",
    "    if row['AGE_GROUP_80-max']==5:\n",
    "        return '5'  \n",
    "    return '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c7893bf2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.571070Z",
     "start_time": "2023-04-18T14:22:27.557036Z"
    }
   },
   "outputs": [],
   "source": [
    "# define function that iterates through each row of age group columns\n",
    "def label_gender(row):\n",
    "    if row['SEX_IDENT_CD_1']==1:\n",
    "        return '1'\n",
    "    if row['SEX_IDENT_CD_2']==2:\n",
    "        return '2'\n",
    "    return '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4adcdecc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.587069Z",
     "start_time": "2023-04-18T14:22:27.573049Z"
    }
   },
   "outputs": [],
   "source": [
    "def label_food_insecurity(row):\n",
    "    if row['FOOD_INSECURITY_GROUP_low']==1:\n",
    "        return '1'\n",
    "    if row['FOOD_INSECURITY_GROUP_medium']==1:\n",
    "        return '2'\n",
    "    if row['FOOD_INSECURITY_GROUP_high']==1:\n",
    "        return '3'\n",
    "    return '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7550b5e7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.603069Z",
     "start_time": "2023-04-18T14:22:27.589037Z"
    }
   },
   "outputs": [],
   "source": [
    "def label_no_vehicle_access(row):\n",
    "    if row['NO_VEHICLE_ACCESS_GROUP_low']==1:\n",
    "        return '1'\n",
    "    if row['NO_VEHICLE_ACCESS_GROUP_medium']==1:\n",
    "        return '2'\n",
    "    if row['NO_VEHICLE_ACCESS_GROUP_high']==1:\n",
    "        return '3'\n",
    "    return '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c20eb89b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:27.619037Z",
     "start_time": "2023-04-18T14:22:27.604037Z"
    }
   },
   "outputs": [],
   "source": [
    "def label_severe_housing_cost_burden(row):\n",
    "    if row['SEVERE_HOUSING_COST_BURDEN_GROUP_low']==1:\n",
    "        return '1'\n",
    "    if row['SEVERE_HOUSING_COST_BURDEN_GROUP_medium']==1:\n",
    "        return '2'\n",
    "    if row['SEVERE_HOUSING_COST_BURDEN_GROUP_high']==1:\n",
    "        return '3'\n",
    "    return '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c2600d66",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:29.299061Z",
     "start_time": "2023-04-18T14:22:27.622037Z"
    }
   },
   "outputs": [],
   "source": [
    "synthetic_feature_selection.apply(lambda row:label_age(row), axis=1)\n",
    "synthetic_feature_selection['age_group'] = synthetic_feature_selection.apply (lambda row: label_age(row), axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "52032875",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:31.069125Z",
     "start_time": "2023-04-18T14:22:29.301062Z"
    }
   },
   "outputs": [],
   "source": [
    "synthetic_feature_selection.apply(lambda row:label_gender(row), axis=1)\n",
    "synthetic_feature_selection['gender'] = synthetic_feature_selection.apply (lambda row: label_gender(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7ebb9f7b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:32.998117Z",
     "start_time": "2023-04-18T14:22:31.071090Z"
    }
   },
   "outputs": [],
   "source": [
    "synthetic_feature_selection.apply(lambda row:label_food_insecurity(row), axis=1)\n",
    "synthetic_feature_selection['food_insecurity'] = synthetic_feature_selection.apply (lambda row: label_food_insecurity(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5719c617",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:34.759173Z",
     "start_time": "2023-04-18T14:22:33.000118Z"
    }
   },
   "outputs": [],
   "source": [
    "synthetic_feature_selection.apply(lambda row:label_no_vehicle_access(row), axis=1)\n",
    "synthetic_feature_selection['no_vehicle_access'] = synthetic_feature_selection.apply (lambda row: label_no_vehicle_access(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d1f7607d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:36.736638Z",
     "start_time": "2023-04-18T14:22:34.761144Z"
    }
   },
   "outputs": [],
   "source": [
    "synthetic_feature_selection.apply(lambda row:label_severe_housing_cost_burden(row), axis=1)\n",
    "synthetic_feature_selection['severe_housing_cost_burden'] = synthetic_feature_selection.apply (lambda row: label_severe_housing_cost_burden(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "60f1b25e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:36.751638Z",
     "start_time": "2023-04-18T14:22:36.738661Z"
    }
   },
   "outputs": [],
   "source": [
    "cols_cat = [\n",
    " 'age_group',\n",
    " 'food_insecurity',\n",
    " 'gender',\n",
    " 'no_vehicle_access',\n",
    " 'severe_housing_cost_burden',\n",
    " 'ad',\n",
    " 'afib_flutter',\n",
    " 'ami_ca',\n",
    " 'anemia',\n",
    " 'asthma',\n",
    " 'breast_ca',\n",
    " 'ckd',\n",
    " 'colorectal_ca',\n",
    " 'copd',\n",
    " 'dep_bipolar_others',\n",
    " 'dm',\n",
    " 'hf_non_ischemic_hd',\n",
    " 'hlp',\n",
    " 'ht',\n",
    " 'htn',\n",
    " 'ihd',\n",
    " 'lung_ca',\n",
    " 'nad',\n",
    " 'or_with_without_pathological_fx',\n",
    " 'pneumonia_all_cause',\n",
    " 'prostate_ca',\n",
    " 'ra_oa']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cb371240",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:38.206696Z",
     "start_time": "2023-04-18T14:22:36.753639Z"
    }
   },
   "outputs": [],
   "source": [
    "# convert object type to category\n",
    "synthetic_feature_selection[cols_cat] = synthetic_feature_selection[cols_cat].apply(lambda x:x.astype(\"category\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2d45c3bf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:38.222661Z",
     "start_time": "2023-04-18T14:22:38.208696Z"
    }
   },
   "outputs": [],
   "source": [
    "# encode\n",
    "synthetic_feature_selection[cols_cat] = synthetic_feature_selection[cols_cat].apply(lambda x: x.cat.codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bff7107a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:38.253661Z",
     "start_time": "2023-04-18T14:22:38.225663Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 517 entries, SEX_IDENT_CD_1 to severe_housing_cost_burden\n",
      "dtypes: int32(5), int64(485), int8(27)\n",
      "memory usage: 184.8 MB\n"
     ]
    }
   ],
   "source": [
    "# check datatype is int\n",
    "synthetic_feature_selection.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "817b556c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:38.269666Z",
     "start_time": "2023-04-18T14:22:38.255662Z"
    }
   },
   "outputs": [],
   "source": [
    "# assign X (input) and y (output) \n",
    "X = synthetic_feature_selection[cols_cat]\n",
    "y = synthetic_feature_selection[\"OUTPUT_LABEL_30\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e153aafa",
   "metadata": {},
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9a6b25d0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-18T14:22:38.332662Z",
     "start_time": "2023-04-18T14:22:38.271662Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>P-Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ckd</th>\n",
       "      <td>1782.870600</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age_group</th>\n",
       "      <td>1713.584191</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ihd</th>\n",
       "      <td>1037.379904</td>\n",
       "      <td>1.347006e-227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dm</th>\n",
       "      <td>1011.420315</td>\n",
       "      <td>5.914460e-222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hlp</th>\n",
       "      <td>759.030382</td>\n",
       "      <td>4.364022e-167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prostate_ca</th>\n",
       "      <td>711.788417</td>\n",
       "      <td>8.170820e-157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nad</th>\n",
       "      <td>674.569177</td>\n",
       "      <td>1.013794e-148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ad</th>\n",
       "      <td>612.622586</td>\n",
       "      <td>3.008384e-135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>afib_flutter</th>\n",
       "      <td>541.392040</td>\n",
       "      <td>9.388537e-120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lung_ca</th>\n",
       "      <td>404.259219</td>\n",
       "      <td>6.512807e-90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asthma</th>\n",
       "      <td>359.540300</td>\n",
       "      <td>3.545541e-80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>anemia</th>\n",
       "      <td>309.523187</td>\n",
       "      <td>2.773930e-69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>htn</th>\n",
       "      <td>262.580902</td>\n",
       "      <td>4.698417e-59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ami_ca</th>\n",
       "      <td>172.857801</td>\n",
       "      <td>1.758020e-39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>170.481563</td>\n",
       "      <td>5.807498e-39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>copd</th>\n",
       "      <td>159.393299</td>\n",
       "      <td>1.535348e-36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>or_with_without_pathological_fx</th>\n",
       "      <td>158.624259</td>\n",
       "      <td>2.260680e-36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>colorectal_ca</th>\n",
       "      <td>64.070966</td>\n",
       "      <td>1.200173e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_vehicle_access</th>\n",
       "      <td>33.638289</td>\n",
       "      <td>6.637327e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ht</th>\n",
       "      <td>21.624510</td>\n",
       "      <td>3.315874e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hf_non_ischemic_hd</th>\n",
       "      <td>20.860430</td>\n",
       "      <td>4.939950e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dep_bipolar_others</th>\n",
       "      <td>18.534193</td>\n",
       "      <td>1.668834e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>severe_housing_cost_burden</th>\n",
       "      <td>15.251945</td>\n",
       "      <td>9.407981e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ra_oa</th>\n",
       "      <td>3.327103</td>\n",
       "      <td>6.814681e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>food_insecurity</th>\n",
       "      <td>1.832731</td>\n",
       "      <td>1.758054e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>breast_ca</th>\n",
       "      <td>0.547986</td>\n",
       "      <td>4.591417e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pneumonia_all_cause</th>\n",
       "      <td>0.343626</td>\n",
       "      <td>5.577438e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Score        P-Value\n",
       "ckd                              1782.870600   0.000000e+00\n",
       "age_group                        1713.584191   0.000000e+00\n",
       "ihd                              1037.379904  1.347006e-227\n",
       "dm                               1011.420315  5.914460e-222\n",
       "hlp                               759.030382  4.364022e-167\n",
       "prostate_ca                       711.788417  8.170820e-157\n",
       "nad                               674.569177  1.013794e-148\n",
       "ad                                612.622586  3.008384e-135\n",
       "afib_flutter                      541.392040  9.388537e-120\n",
       "lung_ca                           404.259219   6.512807e-90\n",
       "asthma                            359.540300   3.545541e-80\n",
       "anemia                            309.523187   2.773930e-69\n",
       "htn                               262.580902   4.698417e-59\n",
       "ami_ca                            172.857801   1.758020e-39\n",
       "gender                            170.481563   5.807498e-39\n",
       "copd                              159.393299   1.535348e-36\n",
       "or_with_without_pathological_fx   158.624259   2.260680e-36\n",
       "colorectal_ca                      64.070966   1.200173e-15\n",
       "no_vehicle_access                  33.638289   6.637327e-09\n",
       "ht                                 21.624510   3.315874e-06\n",
       "hf_non_ischemic_hd                 20.860430   4.939950e-06\n",
       "dep_bipolar_others                 18.534193   1.668834e-05\n",
       "severe_housing_cost_burden         15.251945   9.407981e-05\n",
       "ra_oa                               3.327103   6.814681e-02\n",
       "food_insecurity                     1.832731   1.758054e-01\n",
       "breast_ca                           0.547986   4.591417e-01\n",
       "pneumonia_all_cause                 0.343626   5.577438e-01"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# feature selection using SelectKBest and chi2\n",
    "selector = SelectKBest(score_func=chi2,k=27)\n",
    "selector.fit(X,y)\n",
    "feature_score = pd.DataFrame({\"Score\":selector.scores_,\"P-Value\":selector.pvalues_}, index=X.columns)\n",
    "feature_score.nlargest(n=27, columns=\"Score\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "3d6c3a76178f4fb2edda720dedee7804f091838f2cdd2fd5569e4b9adfa17707"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}