03-machine-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(246)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import df\n",
    "\n",
    "merged_df = pd.read_pickle(\"merged_df.pkl\")\n",
    "target_cols = ['cooler_condition', 'valve_condition', 'pump_leakage',\n",
    "               'hydraulic_accumulator', 'stable_flag']\n",
    "\n",
    "feature_df = merged_df[[col for col in merged_df.columns if col not in target_cols]]\n",
    "target_df = merged_df[target_cols]\n",
    "\n",
    "feature_df = feature_df.drop(columns=[\"ce\", \"se\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distinguish the target conditions\n",
    "\n",
    "y_cooler_condition = pd.DataFrame(target_df.iloc[:, 0])\n",
    "y_valve_condition = pd.DataFrame(target_df.iloc[:, 1])\n",
    "y_pump_leak = pd.DataFrame(target_df.iloc[:, 2])\n",
    "y_hydraulic_accumulator = pd.DataFrame(target_df.iloc[:, 3])\n",
    "y_stable_flag = pd.DataFrame(target_df.iloc[:, 4])\n",
    "\n",
    "target_columns = [y_cooler_condition, y_valve_condition, y_pump_leak, y_hydraulic_accumulator]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalizing features\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaled = preprocessing.StandardScaler().fit_transform(feature_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MACHINE LEARNING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train and test split\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, test = train_test_split(merged_df, test_size=.2)\n",
    "\n",
    "train_X = train[feature_df.columns].reset_index(drop=True)\n",
    "train_y = train[target_df.columns].reset_index(drop=True)\n",
    "test_X = test[feature_df.columns].reset_index(drop=True)\n",
    "test_y = test[target_df.columns].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the models we want to test\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "models = {\"Logistic Regression\": LogisticRegression(solver=\"liblinear\"),\n",
    "          \"Decision Tree\": DecisionTreeClassifier(),\n",
    "          \"Random Forest\": RandomForestClassifier(n_estimators=10),\n",
    "          \"K Nearest Neighbors\": KNeighborsClassifier(n_neighbors=3),\n",
    "          \"Support Vector Machine\": svm.SVC()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Evaluation\n",
    "\n",
    "from sklearn.model_selection import cross_validate\n",
    "\n",
    "def validate_model(model, train_X, train_y):\n",
    "    validation_results = cross_validate(model,\n",
    "                                        train_X,\n",
    "                                        train_y,\n",
    "                                        cv=5,\n",
    "                                        scoring=\"precision_micro\")\n",
    "    acc_score = validation_results[\"test_score\"].mean()\n",
    "    print(f\"Precision Mean Score: {acc_score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression:\n",
      "Precision Mean Score: 0.9965973292244479\n",
      "========================================\n",
      "Decision Tree:\n",
      "Precision Mean Score: 0.9994318181818181\n",
      "========================================\n",
      "Random Forest:\n",
      "Precision Mean Score: 0.9971623009758602\n",
      "========================================\n",
      "K Nearest Neighbors:\n",
      "Precision Mean Score: 0.9807203389830509\n",
      "========================================\n",
      "Support Vector Machine:\n",
      "Precision Mean Score: 0.9835580380071904\n",
      "========================================\n"
     ]
    }
   ],
   "source": [
    "# Cooler Condition\n",
    "for key, model in models.items():\n",
    "    print(f\"{key}:\")\n",
    "    validate_model(model, train_X, train_y[\"cooler_condition\"])\n",
    "    print(\"========================================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression:\n",
      "Precision Mean Score: 0.7443333063849702\n",
      "========================================\n",
      "Decision Tree:\n",
      "Precision Mean Score: 0.9246515288744126\n",
      "========================================\n",
      "Random Forest:\n",
      "Precision Mean Score: 0.9427579698291995\n",
      "========================================\n",
      "K Nearest Neighbors:\n",
      "Precision Mean Score: 0.6610302985098923\n",
      "========================================\n",
      "Support Vector Machine:\n",
      "Precision Mean Score: 0.5759760302796385\n",
      "========================================\n"
     ]
    }
   ],
   "source": [
    "# Valve Condition\n",
    "for key, model in models.items():\n",
    "    print(f\"{key}:\")\n",
    "    validate_model(model, train_X, train_y[\"valve_condition\"])\n",
    "    print(\"========================================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression:\n",
      "Precision Mean Score: 0.530608199156308\n",
      "========================================\n",
      "Decision Tree:\n",
      "Precision Mean Score: 0.9319223964699865\n",
      "========================================\n",
      "Random Forest:\n",
      "Precision Mean Score: 0.9665493773871662\n",
      "========================================\n",
      "K Nearest Neighbors:\n",
      "Precision Mean Score: 0.8475004352667641\n",
      "========================================\n",
      "Support Vector Machine:\n",
      "Precision Mean Score: 0.8123693836985343\n",
      "========================================\n"
     ]
    }
   ],
   "source": [
    "# Hydraulic Accumulator\n",
    "for key, model in models.items():\n",
    "    print(f\"{key}:\")\n",
    "    validate_model(model, train_X, train_y[\"hydraulic_accumulator\"])\n",
    "    print(\"========================================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression:\n",
      "Precision Mean Score: 0.9449956604989735\n",
      "========================================\n",
      "Decision Tree:\n",
      "Precision Mean Score: 0.9818599763418989\n",
      "========================================\n",
      "Random Forest:\n",
      "Precision Mean Score: 0.9892318628499428\n",
      "========================================\n",
      "K Nearest Neighbors:\n",
      "Precision Mean Score: 0.9535151413302371\n",
      "========================================\n",
      "Support Vector Machine:\n",
      "Precision Mean Score: 0.9467114093593544\n",
      "========================================\n"
     ]
    }
   ],
   "source": [
    "# Pump Leakage\n",
    "for key, model in models.items():\n",
    "    print(f\"{key}:\")\n",
    "    validate_model(model, train_X, train_y[\"pump_leakage\"])\n",
    "    print(\"========================================\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recursive Feature Elimination - RFE and test set evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.metrics import precision_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score\n",
      "0.9958592133\n",
      "==============\n",
      "Confusion Matrix\n",
      "[[140   0   2]\n",
      " [  0 140   0]\n",
      " [  0   0 159]]\n"
     ]
    }
   ],
   "source": [
    "# Cooler Condition\n",
    "\n",
    "selector_rf = RFECV(RandomForestClassifier(n_estimators=10), cv=5, scoring=\"precision_micro\" )\n",
    "\n",
    "selector_rf.fit(train_X, train_y[\"cooler_condition\"]);\n",
    "train_X2 = train_X[train_X.columns[selector_rf.get_support()].tolist()]\n",
    "\n",
    "\n",
    "# Precision Score for test set\n",
    "model = RandomForestClassifier()\n",
    "model.fit(train_X2, train_y[\"cooler_condition\"])\n",
    "\n",
    "test_X2 = test_X[train_X2.columns.to_list()]\n",
    "pred = model.predict(test_X2)\n",
    "\n",
    "print(\"Score\")\n",
    "print(round(precision_score(test_y[\"cooler_condition\"], pred, average='macro'), 10))\n",
    "print(\"==============\")\n",
    "print(\"Confusion Matrix\")\n",
    "print(confusion_matrix(test_y[\"cooler_condition\"], pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9480156397949522"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Valve Condition\n",
    "\n",
    "selector_rf = RFECV(RandomForestClassifier(n_estimators=10), cv=5, scoring=\"precision_micro\" )\n",
    "\n",
    "selector_rf.fit(train_X, train_y[\"valve_condition\"]);\n",
    "train_X2 = train_X[train_X.columns[selector_rf.get_support()].tolist()]\n",
    "\n",
    "\n",
    "# Precision Score for test set\n",
    "model = RandomForestClassifier()\n",
    "model.fit(train_X2, train_y[\"valve_condition\"])\n",
    "\n",
    "test_X2 = test_X[train_X2.columns.to_list()]\n",
    "pred = model.predict(test_X2)\n",
    "\n",
    "precision_score(test_y[\"valve_condition\"], pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9554572499554312"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hydraulic Accumulator\n",
    "\n",
    "selector_rf = RFECV(RandomForestClassifier(n_estimators=10), cv=5, scoring=\"precision_micro\" )\n",
    "\n",
    "selector_rf.fit(train_X, train_y[\"hydraulic_accumulator\"]);\n",
    "train_X2 = train_X[train_X.columns[selector_rf.get_support()].tolist()]\n",
    "\n",
    "\n",
    "# Precision Score for test set\n",
    "model = RandomForestClassifier()\n",
    "model.fit(train_X2, train_y[\"hydraulic_accumulator\"])\n",
    "\n",
    "test_X2 = test_X[train_X2.columns.to_list()]\n",
    "pred = model.predict(test_X2)\n",
    "\n",
    "precision_score(test_y[\"hydraulic_accumulator\"], pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9510003114187224"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pump Leakage\n",
    "\n",
    "selector_rf = RFECV(RandomForestClassifier(n_estimators=10), cv=5, scoring=\"precision_micro\" )\n",
    "\n",
    "selector_rf.fit(train_X, train_y[\"pump_leakage\"]);\n",
    "train_X2 = train_X[train_X.columns[selector_rf.get_support()].tolist()]\n",
    "\n",
    "\n",
    "# Precision Score for test set\n",
    "model = RandomForestClassifier()\n",
    "model.fit(train_X2, train_y[\"pump_leakage\"])\n",
    "\n",
    "test_X2 = test_X[train_X2.columns.to_list()]\n",
    "pred = model.predict(test_X2)\n",
    "\n",
    "precision_score(test_y[\"pump_leakage\"], pred, average='macro')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}