tackling_missing_data_imputing_statistics_missing_indicators.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# fillna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "wdi = pd.read_pickle('wdi.pkl')\n",
    "numeric_cols = wdi.select_dtypes(include='number').columns\n",
    "cat_cols = wdi.select_dtypes(exclude='number').columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "access_to_electricity_pct         8.421552e+01\n",
       "atms_per_100000                   4.812857e+01\n",
       "compulsory_education_years        9.701456e+00\n",
       "health_expenditure_pct_of_gdp     6.433350e+00\n",
       "gdp_per_capita_usd                1.359989e+04\n",
       "gdp_per_capita_ppp                1.992523e+04\n",
       "life_expectancy_female            7.456947e+01\n",
       "life_expectancy_male              6.986652e+01\n",
       "life_expectancy                   7.217627e+01\n",
       "population_density                2.099738e+02\n",
       "population                        3.690135e+08\n",
       "alcohol_consumption_per_capita    6.016516e+00\n",
       "unemployment_rate_female          8.404762e+00\n",
       "unemployment_rate_male            6.465617e+00\n",
       "unemployment_rate                 7.047682e+00\n",
       "year                              2.017500e+03\n",
       "is_region                         2.119816e-01\n",
       "dtype: float64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[numeric_cols].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "access_to_electricity_pct         9.891294e+01\n",
       "atms_per_100000                   3.941107e+01\n",
       "compulsory_education_years        1.000000e+01\n",
       "health_expenditure_pct_of_gdp     6.006967e+00\n",
       "gdp_per_capita_usd                5.783711e+03\n",
       "gdp_per_capita_ppp                1.310845e+04\n",
       "life_expectancy_female            7.608850e+01\n",
       "life_expectancy_male              7.081350e+01\n",
       "life_expectancy                   7.358550e+01\n",
       "population_density                6.961068e+01\n",
       "population                        1.604842e+07\n",
       "alcohol_consumption_per_capita    5.837431e+00\n",
       "unemployment_rate_female          5.836087e+00\n",
       "unemployment_rate_male            5.351708e+00\n",
       "unemployment_rate                 5.606500e+00\n",
       "year                              2.017500e+03\n",
       "is_region                         0.000000e+00\n",
       "dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[numeric_cols].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>access_to_electricity_pct</th>\n",
       "      <th>atms_per_100000</th>\n",
       "      <th>compulsory_education_years</th>\n",
       "      <th>health_expenditure_pct_of_gdp</th>\n",
       "      <th>gdp_per_capita_usd</th>\n",
       "      <th>gdp_per_capita_ppp</th>\n",
       "      <th>life_expectancy_female</th>\n",
       "      <th>life_expectancy_male</th>\n",
       "      <th>life_expectancy</th>\n",
       "      <th>population_density</th>\n",
       "      <th>population</th>\n",
       "      <th>alcohol_consumption_per_capita</th>\n",
       "      <th>unemployment_rate_female</th>\n",
       "      <th>unemployment_rate_male</th>\n",
       "      <th>unemployment_rate</th>\n",
       "      <th>year</th>\n",
       "      <th>is_region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97.700000</td>\n",
       "      <td>1.213235</td>\n",
       "      <td>9.0</td>\n",
       "      <td>11.777236</td>\n",
       "      <td>519.884773</td>\n",
       "      <td>2058.383832</td>\n",
       "      <td>65.656000</td>\n",
       "      <td>62.701000</td>\n",
       "      <td>64.130000</td>\n",
       "      <td>55.595993</td>\n",
       "      <td>3.629640e+07</td>\n",
       "      <td>6.016516</td>\n",
       "      <td>14.090000</td>\n",
       "      <td>10.416000</td>\n",
       "      <td>11.184000</td>\n",
       "      <td>2017</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>31.714076</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.010597</td>\n",
       "      <td>4531.020806</td>\n",
       "      <td>13037.010016</td>\n",
       "      <td>80.148000</td>\n",
       "      <td>76.601000</td>\n",
       "      <td>78.333000</td>\n",
       "      <td>104.870693</td>\n",
       "      <td>2.873457e+06</td>\n",
       "      <td>6.016516</td>\n",
       "      <td>12.563000</td>\n",
       "      <td>14.590000</td>\n",
       "      <td>13.750000</td>\n",
       "      <td>2017</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>9.130677</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.380329</td>\n",
       "      <td>4111.294110</td>\n",
       "      <td>11737.409353</td>\n",
       "      <td>77.735000</td>\n",
       "      <td>75.307000</td>\n",
       "      <td>76.499000</td>\n",
       "      <td>17.377715</td>\n",
       "      <td>4.138920e+07</td>\n",
       "      <td>6.016516</td>\n",
       "      <td>21.114000</td>\n",
       "      <td>10.021000</td>\n",
       "      <td>11.996000</td>\n",
       "      <td>2017</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41.962894</td>\n",
       "      <td>19.079250</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.791503</td>\n",
       "      <td>4095.812942</td>\n",
       "      <td>7310.901738</td>\n",
       "      <td>63.252000</td>\n",
       "      <td>57.677000</td>\n",
       "      <td>60.379000</td>\n",
       "      <td>23.916538</td>\n",
       "      <td>2.981675e+07</td>\n",
       "      <td>6.016516</td>\n",
       "      <td>7.467000</td>\n",
       "      <td>6.769000</td>\n",
       "      <td>7.119000</td>\n",
       "      <td>2017</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>90.283638</td>\n",
       "      <td>27.360620</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.561266</td>\n",
       "      <td>6108.588220</td>\n",
       "      <td>14562.367966</td>\n",
       "      <td>73.493366</td>\n",
       "      <td>69.921250</td>\n",
       "      <td>71.622526</td>\n",
       "      <td>36.669804</td>\n",
       "      <td>4.118990e+08</td>\n",
       "      <td>6.016516</td>\n",
       "      <td>20.361396</td>\n",
       "      <td>8.097088</td>\n",
       "      <td>10.641313</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>25.322499</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.917897</td>\n",
       "      <td>2566.596950</td>\n",
       "      <td>7771.189420</td>\n",
       "      <td>79.442000</td>\n",
       "      <td>71.207000</td>\n",
       "      <td>75.317000</td>\n",
       "      <td>308.125246</td>\n",
       "      <td>9.554040e+07</td>\n",
       "      <td>8.660000</td>\n",
       "      <td>1.831000</td>\n",
       "      <td>2.142000</td>\n",
       "      <td>1.993000</td>\n",
       "      <td>2018</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>24.735190</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.433350</td>\n",
       "      <td>3562.330943</td>\n",
       "      <td>6472.121785</td>\n",
       "      <td>75.595000</td>\n",
       "      <td>72.263000</td>\n",
       "      <td>73.895000</td>\n",
       "      <td>758.984551</td>\n",
       "      <td>4.569087e+06</td>\n",
       "      <td>6.016516</td>\n",
       "      <td>41.849998</td>\n",
       "      <td>22.393999</td>\n",
       "      <td>26.256001</td>\n",
       "      <td>2018</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>89.588551</td>\n",
       "      <td>40.949620</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.848779</td>\n",
       "      <td>11385.675227</td>\n",
       "      <td>17025.827360</td>\n",
       "      <td>74.871815</td>\n",
       "      <td>70.398743</td>\n",
       "      <td>72.563282</td>\n",
       "      <td>59.617978</td>\n",
       "      <td>7.591945e+09</td>\n",
       "      <td>6.181240</td>\n",
       "      <td>5.580954</td>\n",
       "      <td>5.271615</td>\n",
       "      <td>5.392213</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>39.812622</td>\n",
       "      <td>11.554664</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.934843</td>\n",
       "      <td>1516.390661</td>\n",
       "      <td>3607.304442</td>\n",
       "      <td>66.447000</td>\n",
       "      <td>60.533000</td>\n",
       "      <td>63.510000</td>\n",
       "      <td>23.341479</td>\n",
       "      <td>1.735182e+07</td>\n",
       "      <td>6.540000</td>\n",
       "      <td>12.448000</td>\n",
       "      <td>10.614000</td>\n",
       "      <td>11.500000</td>\n",
       "      <td>2018</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>41.041584</td>\n",
       "      <td>6.625208</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.734331</td>\n",
       "      <td>1683.740577</td>\n",
       "      <td>3206.277079</td>\n",
       "      <td>62.598000</td>\n",
       "      <td>59.501000</td>\n",
       "      <td>61.195000</td>\n",
       "      <td>37.324591</td>\n",
       "      <td>1.443902e+07</td>\n",
       "      <td>4.670000</td>\n",
       "      <td>5.598000</td>\n",
       "      <td>4.555000</td>\n",
       "      <td>5.087000</td>\n",
       "      <td>2018</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>434 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     access_to_electricity_pct  atms_per_100000  compulsory_education_years  \\\n",
       "0                    97.700000         1.213235                         9.0   \n",
       "1                   100.000000        31.714076                         9.0   \n",
       "2                   100.000000         9.130677                        10.0   \n",
       "3                    41.962894        19.079250                         6.0   \n",
       "4                    90.283638        27.360620                         9.0   \n",
       "..                         ...              ...                         ...   \n",
       "429                 100.000000        25.322499                        10.0   \n",
       "430                 100.000000        24.735190                        10.0   \n",
       "431                  89.588551        40.949620                        10.0   \n",
       "432                  39.812622        11.554664                         7.0   \n",
       "433                  41.041584         6.625208                         7.0   \n",
       "\n",
       "     health_expenditure_pct_of_gdp  gdp_per_capita_usd  gdp_per_capita_ppp  \\\n",
       "0                        11.777236          519.884773         2058.383832   \n",
       "1                         5.010597         4531.020806        13037.010016   \n",
       "2                         6.380329         4111.294110        11737.409353   \n",
       "3                         2.791503         4095.812942         7310.901738   \n",
       "4                         5.561266         6108.588220        14562.367966   \n",
       "..                             ...                 ...                 ...   \n",
       "429                       5.917897         2566.596950         7771.189420   \n",
       "430                       6.433350         3562.330943         6472.121785   \n",
       "431                       9.848779        11385.675227        17025.827360   \n",
       "432                       4.934843         1516.390661         3607.304442   \n",
       "433                       4.734331         1683.740577         3206.277079   \n",
       "\n",
       "     life_expectancy_female  life_expectancy_male  life_expectancy  \\\n",
       "0                 65.656000             62.701000        64.130000   \n",
       "1                 80.148000             76.601000        78.333000   \n",
       "2                 77.735000             75.307000        76.499000   \n",
       "3                 63.252000             57.677000        60.379000   \n",
       "4                 73.493366             69.921250        71.622526   \n",
       "..                      ...                   ...              ...   \n",
       "429               79.442000             71.207000        75.317000   \n",
       "430               75.595000             72.263000        73.895000   \n",
       "431               74.871815             70.398743        72.563282   \n",
       "432               66.447000             60.533000        63.510000   \n",
       "433               62.598000             59.501000        61.195000   \n",
       "\n",
       "     population_density    population  alcohol_consumption_per_capita  \\\n",
       "0             55.595993  3.629640e+07                        6.016516   \n",
       "1            104.870693  2.873457e+06                        6.016516   \n",
       "2             17.377715  4.138920e+07                        6.016516   \n",
       "3             23.916538  2.981675e+07                        6.016516   \n",
       "4             36.669804  4.118990e+08                        6.016516   \n",
       "..                  ...           ...                             ...   \n",
       "429          308.125246  9.554040e+07                        8.660000   \n",
       "430          758.984551  4.569087e+06                        6.016516   \n",
       "431           59.617978  7.591945e+09                        6.181240   \n",
       "432           23.341479  1.735182e+07                        6.540000   \n",
       "433           37.324591  1.443902e+07                        4.670000   \n",
       "\n",
       "     unemployment_rate_female  unemployment_rate_male  unemployment_rate  \\\n",
       "0                   14.090000               10.416000          11.184000   \n",
       "1                   12.563000               14.590000          13.750000   \n",
       "2                   21.114000               10.021000          11.996000   \n",
       "3                    7.467000                6.769000           7.119000   \n",
       "4                   20.361396                8.097088          10.641313   \n",
       "..                        ...                     ...                ...   \n",
       "429                  1.831000                2.142000           1.993000   \n",
       "430                 41.849998               22.393999          26.256001   \n",
       "431                  5.580954                5.271615           5.392213   \n",
       "432                 12.448000               10.614000          11.500000   \n",
       "433                  5.598000                4.555000           5.087000   \n",
       "\n",
       "     year  is_region  \n",
       "0    2017          0  \n",
       "1    2017          0  \n",
       "2    2017          0  \n",
       "3    2017          0  \n",
       "4    2017          1  \n",
       "..    ...        ...  \n",
       "429  2018          0  \n",
       "430  2018          0  \n",
       "431  2018          1  \n",
       "432  2018          0  \n",
       "433  2018          0  \n",
       "\n",
       "[434 rows x 17 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[numeric_cols].fillna(wdi[numeric_cols].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "wdi[numeric_cols] = wdi[numeric_cols].fillna(wdi[numeric_cols].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.016516     219\n",
       "0.690000       3\n",
       "2.050000       2\n",
       "0.019000       2\n",
       "9.230000       2\n",
       "            ... \n",
       "2.300000       1\n",
       "8.940000       1\n",
       "4.065725       1\n",
       "11.350000      1\n",
       "2.710000       1\n",
       "Name: alcohol_consumption_per_capita, Length: 205, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi['alcohol_consumption_per_capita'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_name</th>\n",
       "      <th>country_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>434</td>\n",
       "      <td>282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>217</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Belgium</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       country_name country_category\n",
       "count           434              282\n",
       "unique          217                2\n",
       "top         Belgium       DEVELOPING\n",
       "freq              2              224"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[cat_cols].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "country_name           Belgium\n",
       "country_category    DEVELOPING\n",
       "Name: top, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[cat_cols].describe().loc['top']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_freq = wdi[cat_cols].describe().loc['top']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_name</th>\n",
       "      <th>country_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Albania</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Algeria</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Angola</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Arab World</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>Vietnam</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>West Bank and Gaza</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>World</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>Zambia</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>Zimbabwe</td>\n",
       "      <td>DEVELOPING</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>434 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           country_name country_category\n",
       "0           Afghanistan       DEVELOPING\n",
       "1               Albania       DEVELOPING\n",
       "2               Algeria       DEVELOPING\n",
       "3                Angola       DEVELOPING\n",
       "4            Arab World       DEVELOPING\n",
       "..                  ...              ...\n",
       "429             Vietnam       DEVELOPING\n",
       "430  West Bank and Gaza       DEVELOPING\n",
       "431               World       DEVELOPING\n",
       "432              Zambia       DEVELOPING\n",
       "433            Zimbabwe       DEVELOPING\n",
       "\n",
       "[434 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[cat_cols] = wdi[cat_cols].fillna(most_freq)\n",
    "wdi[cat_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DEVELOPING    376\n",
       "DEVELOPED      58\n",
       "Name: country_category, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi['country_category'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 434 entries, 0 to 433\n",
      "Data columns (total 19 columns):\n",
      " #   Column                          Non-Null Count  Dtype  \n",
      "---  ------                          --------------  -----  \n",
      " 0   country_name                    434 non-null    object \n",
      " 1   access_to_electricity_pct       434 non-null    float64\n",
      " 2   atms_per_100000                 434 non-null    float64\n",
      " 3   compulsory_education_years      434 non-null    float64\n",
      " 4   health_expenditure_pct_of_gdp   434 non-null    float64\n",
      " 5   gdp_per_capita_usd              434 non-null    float64\n",
      " 6   gdp_per_capita_ppp              434 non-null    float64\n",
      " 7   life_expectancy_female          434 non-null    float64\n",
      " 8   life_expectancy_male            434 non-null    float64\n",
      " 9   life_expectancy                 434 non-null    float64\n",
      " 10  population_density              434 non-null    float64\n",
      " 11  population                      434 non-null    float64\n",
      " 12  alcohol_consumption_per_capita  434 non-null    float64\n",
      " 13  unemployment_rate_female        434 non-null    float64\n",
      " 14  unemployment_rate_male          434 non-null    float64\n",
      " 15  unemployment_rate               434 non-null    float64\n",
      " 16  year                            434 non-null    int64  \n",
      " 17  country_category                434 non-null    object \n",
      " 18  is_region                       434 non-null    int64  \n",
      "dtypes: float64(15), int64(2), object(2)\n",
      "memory usage: 64.5+ KB\n"
     ]
    }
   ],
   "source": [
    "wdi.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "wdi = pd.read_pickle('wdi.pkl')\n",
    "numeric_cols = wdi.select_dtypes(include='number').columns\n",
    "cat_cols = wdi.select_dtypes(exclude='number').columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "simple_imp_mean = SimpleImputer(strategy='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputed_num = simple_imp_mean.fit_transform(wdi[numeric_cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(434, 17)\n",
      "(434, 17)\n"
     ]
    }
   ],
   "source": [
    "print(imputed_num.shape)\n",
    "print(wdi[numeric_cols].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "wdi[numeric_cols] = imputed_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.000000     102\n",
       "10.000000    100\n",
       "6.000000      36\n",
       "12.000000     34\n",
       "11.000000     31\n",
       "8.000000      24\n",
       "9.701456      22\n",
       "7.000000      20\n",
       "13.000000     17\n",
       "14.000000     16\n",
       "15.000000     10\n",
       "5.000000       6\n",
       "9.500000       6\n",
       "8.500000       4\n",
       "16.000000      2\n",
       "6.500000       2\n",
       "10.500000      2\n",
       "Name: compulsory_education_years, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi['compulsory_education_years'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 434 entries, 0 to 433\n",
      "Data columns (total 19 columns):\n",
      " #   Column                          Non-Null Count  Dtype  \n",
      "---  ------                          --------------  -----  \n",
      " 0   country_name                    434 non-null    object \n",
      " 1   access_to_electricity_pct       434 non-null    float64\n",
      " 2   atms_per_100000                 434 non-null    float64\n",
      " 3   compulsory_education_years      434 non-null    float64\n",
      " 4   health_expenditure_pct_of_gdp   434 non-null    float64\n",
      " 5   gdp_per_capita_usd              434 non-null    float64\n",
      " 6   gdp_per_capita_ppp              434 non-null    float64\n",
      " 7   life_expectancy_female          434 non-null    float64\n",
      " 8   life_expectancy_male            434 non-null    float64\n",
      " 9   life_expectancy                 434 non-null    float64\n",
      " 10  population_density              434 non-null    float64\n",
      " 11  population                      434 non-null    float64\n",
      " 12  alcohol_consumption_per_capita  434 non-null    float64\n",
      " 13  unemployment_rate_female        434 non-null    float64\n",
      " 14  unemployment_rate_male          434 non-null    float64\n",
      " 15  unemployment_rate               434 non-null    float64\n",
      " 16  year                            434 non-null    float64\n",
      " 17  country_category                282 non-null    object \n",
      " 18  is_region                       434 non-null    float64\n",
      "dtypes: float64(17), object(2)\n",
      "memory usage: 64.5+ KB\n"
     ]
    }
   ],
   "source": [
    "wdi.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# simple_imp_med = SimpleImputer(strategy='median')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_imp_freq = SimpleImputer(strategy='most_frequent')\n",
    "imputed_cat = simple_imp_freq.fit_transform(wdi[cat_cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "wdi[cat_cols] = imputed_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 434 entries, 0 to 433\n",
      "Data columns (total 19 columns):\n",
      " #   Column                          Non-Null Count  Dtype  \n",
      "---  ------                          --------------  -----  \n",
      " 0   country_name                    434 non-null    object \n",
      " 1   access_to_electricity_pct       434 non-null    float64\n",
      " 2   atms_per_100000                 434 non-null    float64\n",
      " 3   compulsory_education_years      434 non-null    float64\n",
      " 4   health_expenditure_pct_of_gdp   434 non-null    float64\n",
      " 5   gdp_per_capita_usd              434 non-null    float64\n",
      " 6   gdp_per_capita_ppp              434 non-null    float64\n",
      " 7   life_expectancy_female          434 non-null    float64\n",
      " 8   life_expectancy_male            434 non-null    float64\n",
      " 9   life_expectancy                 434 non-null    float64\n",
      " 10  population_density              434 non-null    float64\n",
      " 11  population                      434 non-null    float64\n",
      " 12  alcohol_consumption_per_capita  434 non-null    float64\n",
      " 13  unemployment_rate_female        434 non-null    float64\n",
      " 14  unemployment_rate_male          434 non-null    float64\n",
      " 15  unemployment_rate               434 non-null    float64\n",
      " 16  year                            434 non-null    float64\n",
      " 17  country_category                434 non-null    object \n",
      " 18  is_region                       434 non-null    float64\n",
      "dtypes: float64(17), object(2)\n",
      "memory usage: 64.5+ KB\n"
     ]
    }
   ],
   "source": [
    "wdi.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DEVELOPING    376\n",
       "DEVELOPED      58\n",
       "Name: country_category, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi['country_category'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing Indicator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "wdi = pd.read_pickle('wdi.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_imp_ind = SimpleImputer(strategy='mean', add_indicator=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       9.0\n",
       "1       9.0\n",
       "2      10.0\n",
       "3       6.0\n",
       "4       9.0\n",
       "       ... \n",
       "429    10.0\n",
       "430    10.0\n",
       "431    10.0\n",
       "432     7.0\n",
       "433     7.0\n",
       "Name: compulsory_education_years, Length: 434, dtype: float64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi['compulsory_education_years']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(wdi['compulsory_education_years'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compulsory_education_years</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>434 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     compulsory_education_years\n",
       "0                           9.0\n",
       "1                           9.0\n",
       "2                          10.0\n",
       "3                           6.0\n",
       "4                           9.0\n",
       "..                          ...\n",
       "429                        10.0\n",
       "430                        10.0\n",
       "431                        10.0\n",
       "432                         7.0\n",
       "433                         7.0\n",
       "\n",
       "[434 rows x 1 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[['compulsory_education_years']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(wdi[['compulsory_education_years']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputed_ind = simple_imp_ind.fit_transform(wdi[['compulsory_education_years']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 9.        ,  0.        ],\n",
       "       [ 9.        ,  0.        ],\n",
       "       [10.        ,  0.        ],\n",
       "       [ 6.        ,  0.        ],\n",
       "       [ 9.        ,  0.        ],\n",
       "       [14.        ,  0.        ],\n",
       "       [12.        ,  0.        ],\n",
       "       [10.        ,  0.        ],\n",
       "       [13.        ,  0.        ],\n",
       "       [10.        ,  0.        ],\n",
       "       [12.        ,  0.        ],\n",
       "       [ 9.        ,  0.        ],\n",
       "       [ 5.        ,  0.        ],\n",
       "       [11.        ,  0.        ],\n",
       "       [ 9.        ,  0.        ],\n",
       "       [12.        ,  0.        ],\n",
       "       [ 8.        ,  0.        ],\n",
       "       [ 6.        ,  0.        ],\n",
       "       [ 9.70145631,  1.        ],\n",
       "       [14.        ,  0.        ]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputed_ind[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(434, 2)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputed_ind.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "wdi[['compulsory_education_years', 'compulsory_education_years_missing']] = imputed_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compulsory_education_years</th>\n",
       "      <th>compulsory_education_years_missing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9.701456</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    compulsory_education_years  compulsory_education_years_missing\n",
       "0                     9.000000                                 0.0\n",
       "1                     9.000000                                 0.0\n",
       "2                    10.000000                                 0.0\n",
       "3                     6.000000                                 0.0\n",
       "4                     9.000000                                 0.0\n",
       "5                    14.000000                                 0.0\n",
       "6                    12.000000                                 0.0\n",
       "7                    10.000000                                 0.0\n",
       "8                    13.000000                                 0.0\n",
       "9                    10.000000                                 0.0\n",
       "10                   12.000000                                 0.0\n",
       "11                    9.000000                                 0.0\n",
       "12                    5.000000                                 0.0\n",
       "13                   11.000000                                 0.0\n",
       "14                    9.000000                                 0.0\n",
       "15                   12.000000                                 0.0\n",
       "16                    8.000000                                 0.0\n",
       "17                    6.000000                                 0.0\n",
       "18                    9.701456                                 1.0\n",
       "19                   14.000000                                 0.0"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdi[['compulsory_education_years', 'compulsory_education_years_missing']].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}