gamma_params_retrieval.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyMF3zq3sqWaS5g8cZA89Xe7",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aladinor/notebooks/blob/origin%2Fmaster/gamma_params_retrieval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kxcioFjkNUhD",
        "outputId": "f7790a7b-4965-4f17-f7e0-9d1447f8cb88"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for tensorflow-docs (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "%load_ext tensorboard\n",
        "# Use some functions from tensorflow_docs\n",
        "!pip install -q git+https://github.com/tensorflow/docs"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "from imblearn.over_sampling import RandomOverSampler\n",
        "from collections import Counter\n",
        "import tensorflow_docs as tfdocs\n",
        "import tensorflow_docs.plots\n",
        "import tensorflow_docs.modeling\n",
        "from tensorflow.keras import regularizers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "drive.mount('/content/gdrive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nIkezIIUNXSk",
        "outputId": "6744a357-0620-4da3-96e5-a249929d2876"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  # raise SystemError('GPU device not found')\n",
        "  print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "  print(\"GPU not found\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K8ybeyCCjtbf",
        "outputId": "5830fb76-71e7-421e-e1db-023c649a5306"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU not found\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_parquet('gdrive/My Drive/Colab Notebooks/all_data.parquet')\n",
        "df['dfr'] = df['dbz_t_ku'] - df['dbz_t_ka']"
      ],
      "metadata": {
        "id": "WGfhFPPuOS7R"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['dm_class'] = (df.dm > 1.0).astype(int)\n",
        "df.shape"
      ],
      "metadata": {
        "id": "jS_reLyelDEl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aad051a5-a78f-498d-b73a-8320e1cc3032"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(8027, 45)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X, Y = df[['dbz_t_ku', 'dfr']], df[['log10_nw', 'dm', 'mu']]"
      ],
      "metadata": {
        "id": "p8eub97xP-jQ"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scaler_x = StandardScaler()\n",
        "scaler_y = StandardScaler()\n",
        "X_scaled = scaler_x.fit_transform(X)\n",
        "Y_scaled = scaler_y.fit_transform(Y)"
      ],
      "metadata": {
        "id": "MOCUd2rMsOkI"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Y_scaled = np.append(Y_scaled, df.dm_class.values[:, np.newaxis], axis=1)"
      ],
      "metadata": {
        "id": "jPJhdgAXyjEG"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X_scaled, Y_scaled, test_size=0.33, random_state=42)"
      ],
      "metadata": {
        "id": "NP-XQjieEgi6"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def build_model():\n",
        "    model = tf.keras.models.Sequential()\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\",\n",
        "                                    input_shape=[X_train.shape[1]]))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(3))\n",
        "    # optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)\n",
        "    optimizer = tf.keras.optimizers.Adamax(learning_rate=0.001)\n",
        "    # optimizer = tf.keras.optimizers.RMSprop(0.001)\n",
        "    # optimizer = tf.keras.optimizers.Adadelta(0.1)\n",
        "    # optimizer = tf.keras.optimizers.AdamW(0.001)\n",
        "    # optimizer = tf.keras.optimizers.SGD(0.001)\n",
        "    model.compile(loss=\"mse\",\n",
        "              optimizer=optimizer,\n",
        "              metrics=['mae', 'mse'])\n",
        "    return model"
      ],
      "metadata": {
        "id": "FmkuGfzYEiIW"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model()"
      ],
      "metadata": {
        "id": "Zr4KEibuOXIJ"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class PrintDot(keras.callbacks.Callback):\n",
        "  def on_epoch_end(self, epoch, logs):\n",
        "    if epoch % 100 == 0: print('')\n",
        "    print('.', end='')\n",
        "\n",
        "EPOCHS = 1000\n",
        "\n",
        "# The patience parameter is the amount of epochs to check for improvement\n",
        "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)\n",
        "\n",
        "\n",
        "history = model.fit(X_train, y_train[:, :-1],\n",
        "                    validation_data=(X_test, y_test[:,:-1]),\n",
        "                    epochs=EPOCHS,\n",
        "                    batch_size=32, verbose=0,\n",
        "                    callbacks=[PrintDot(), early_stop])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SWYJYylVtEuH",
        "outputId": "a1b0905a-05af-411d-de40-c521529150ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "....................................................................................................\n",
            "....................................................................................................\n",
            "......................................................................................"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hist = pd.DataFrame(history.history)"
      ],
      "metadata": {
        "id": "5OhLX-ttuFdy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_history(history):\n",
        "  fig, (ax, ax1) = plt.subplots(1, 2, figsize=(12, 5))\n",
        "  hist['epoch'] = history.epoch\n",
        "\n",
        "\n",
        "  ax.set_xlabel('Epoch')\n",
        "  ax.set_ylabel('Mean Abs Error')\n",
        "  ax.plot(hist['epoch'], hist['mae'],\n",
        "           label='Train Error')\n",
        "  try:\n",
        "    ax.plot(hist['epoch'], hist['val_mae'],\n",
        "             label = 'Val Error')\n",
        "  except KeyError:\n",
        "    pass\n",
        "  # plt.ylim([0,5])\n",
        "  ax.legend()\n",
        "\n",
        "  ax1.set_xlabel('Epoch')\n",
        "  ax1.set_ylabel('Mean Square Error')\n",
        "  ax1.plot(hist['epoch'], hist['mse'],\n",
        "           label='Train Error')\n",
        "  try:\n",
        "    ax1.plot(hist['epoch'], hist['val_mse'],\n",
        "              label = 'Val Error')\n",
        "  except KeyError:\n",
        "    pass\n",
        "  # plt.ylim([0,3])\n",
        "  ax1.legend()\n",
        "  plt.show()\n",
        "\n",
        "\n",
        "plot_history(history)"
      ],
      "metadata": {
        "id": "VvzbWQg-Ix84"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "yhat = model.predict(X_test[:, :2])\n",
        "yhat = scaler_y.inverse_transform(yhat)\n",
        "y_test_unscaled = scaler_y.inverse_transform(y_test[:, :-1])"
      ],
      "metadata": {
        "id": "c6y8IMqQwHPw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def metrics_pd(x, xpred):\n",
        "    rmse = np.sqrt(np.sum((xpred - x) ** 2 / len(x)))\n",
        "    mae = np.sum(np.abs(xpred - x)) / len(x)\n",
        "    corr = np.corrcoef(x, xpred)**2.\n",
        "    return rmse, mae, corr[0, 1]"
      ],
      "metadata": {
        "id": "KxpYGLrjhkwo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 'log10_nw', 'dm', 'mu'\n",
        "fig, (ax, ax1, ax2) = plt.subplots(1, 3, figsize=(13, 4))\n",
        "ax.scatter(y_test_unscaled[:, 0], yhat[:, 0], s=1)\n",
        "ax.set_xlabel(r\"$Log_{10}(Nw) \\ - \\ True $\")\n",
        "ax.set_ylabel(r\"$Log_{10}(Nw) \\ - \\ Est. $\")\n",
        "x = np.linspace(*ax.get_xlim())\n",
        "ax.plot(x, x, c='k', ls='--', lw=0.8)\n",
        "rmse, mae, corr1 = metrics_pd(y_test[:, 0], yhat[:, 0])\n",
        "ax.text(3, 10, r\"$r^2$\" + f\"={corr1:.2f}\")\n",
        "ax.text(3, 9.5, r\"$RMSE$\" + f\"={rmse:.2f}\")\n",
        "ax.text(3, 9, r\"$MAE$\" + f\"={mae:.2f}\")\n",
        "\n",
        "ax1.scatter(y_test_unscaled[:, 1], yhat[:, 1], s=1)\n",
        "ax1.set_xlabel(r\"$Dm \\ - \\ True $\")\n",
        "ax1.set_ylabel(r\"$Dm \\ - \\ Est. $\")\n",
        "x = np.linspace(*ax1.get_xlim())\n",
        "ax1.plot(x, x, c='k', ls='--', lw=0.8)\n",
        "rmse, mae, corr1 = metrics_pd(y_test[:, 1], yhat[:, 1])\n",
        "ax1.text(0.3, 2.4, r\"$r^2$\" + f\"={corr1:.2f}\")\n",
        "ax1.text(0.3, 2.2, r\"$RMSE$\" + f\"={rmse:.2f}\")\n",
        "ax1.text(0.3, 2.0, r\"$MAE$\" + f\"={mae:.2f}\")\n",
        "\n",
        "ax2.scatter(y_test_unscaled[:, 2], yhat[:, 2], s=1)\n",
        "ax2.set_xlabel(r\"$\\mu  \\ - \\ True $\")\n",
        "ax2.set_ylabel(r\"$\\mu \\ - \\ Est. $\")\n",
        "rmse, mae, corr1 = metrics_pd(y_test[:, 2], yhat[:, 2])\n",
        "ax2.text(0, 120, r\"$r^2$\" + f\"={corr1:.2f}\")\n",
        "ax2.text(0, 110, r\"$RMSE$\" + f\"={rmse:.2f}\")\n",
        "ax2.text(0, 100, r\"$MAE$\" + f\"={mae:.2f}\")\n",
        "x = np.linspace(*ax2.get_xlim())\n",
        "ax2.plot(x, x, c='k', ls='--', lw=0.8)\n",
        "\n",
        "plt.tight_layout()"
      ],
      "metadata": {
        "id": "nSEfjmxxYdxZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Trying to balance the data"
      ],
      "metadata": {
        "id": "mxfTFioe0qQC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "lower, greater = np.bincount(df['dm_class'])\n",
        "total = greater + lower\n",
        "print('Examples:\\n    Total: {}\\n    greater: {} ({:.2f}% of total)\\n'.format(\n",
        "    total, greater, 100 * greater / total))"
      ],
      "metadata": {
        "id": "bwH5pFyUsMMz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X, Y = df[['dbz_t_ku', 'dfr', 'log10_nw', 'dm', 'mu']], df['dm_class']"
      ],
      "metadata": {
        "id": "nxcR7M5GvV4Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scaler_x = StandardScaler()\n",
        "X_scaled = scaler_x.fit_transform(X)"
      ],
      "metadata": {
        "id": "vf_KvPSDup25"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X_scaled, Y, test_size=0.33, random_state=42)"
      ],
      "metadata": {
        "id": "B7p9vprU7y9O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "over_sampler = RandomOverSampler(random_state=42)\n",
        "X_res, y_res = over_sampler.fit_resample(X_train, y_train)\n",
        "print(f\"Training target statistics: {Counter(y_res)}\")\n",
        "print(f\"Testing target statistics: {Counter(y_test)}\")"
      ],
      "metadata": {
        "id": "l7sbkIyX0dje"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def build_model():\n",
        "    model = tf.keras.models.Sequential()\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\",\n",
        "                                    input_shape=[X_train.shape[1]-3]))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(3))\n",
        "    # optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)\n",
        "    optimizer = tf.keras.optimizers.Adamax(learning_rate=0.001)\n",
        "    # optimizer = tf.keras.optimizers.RMSprop(0.001)\n",
        "    # optimizer = tf.keras.optimizers.Adadelta(0.1)\n",
        "    # optimizer = tf.keras.optimizers.AdamW(0.001)\n",
        "    # optimizer = tf.keras.optimizers.SGD(0.001)\n",
        "    model.compile(loss=\"mse\",\n",
        "              optimizer=optimizer,\n",
        "              metrics=['mae', 'mse'])\n",
        "    return model"
      ],
      "metadata": {
        "id": "WeZrl_wILVtt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model()"
      ],
      "metadata": {
        "id": "sJvy0JoNLbE-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCHS = 500\n",
        "history = model.fit(X_res[:, :2], X_res[:, 2:],\n",
        "                    validation_data=(X_res[:, :2], X_res[:, 2:]),\n",
        "                    epochs=EPOCHS,\n",
        "                    batch_size=32, verbose=0,\n",
        "                    callbacks=[PrintDot(), early_stop])"
      ],
      "metadata": {
        "id": "nAKbNRdz75Hv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hist = pd.DataFrame(history.history)\n",
        "plot_history(history)"
      ],
      "metadata": {
        "id": "FeFh199MBy1M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "yhat = model.predict(X_test[:, :2])\n",
        "yhat = scaler_y.inverse_transform(yhat)\n",
        "y_test_unscaled = scaler_y.inverse_transform(X_test[:, 2:])"
      ],
      "metadata": {
        "id": "1Zb3Q0vwLxlf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 'log10_nw', 'dm', 'mu'\n",
        "fig, (ax, ax1, ax2) = plt.subplots(1, 3, figsize=(13, 4))\n",
        "ax.scatter(y_test_unscaled[:, 0], yhat[:, 0], s=1)\n",
        "ax.set_xlabel(r\"$Log_{10}(Nw) \\ - \\ True $\")\n",
        "ax.set_ylabel(r\"$Log_{10}(Nw) \\ - \\ Est. $\")\n",
        "x = np.linspace(*ax.get_xlim())\n",
        "ax.plot(x, x, c='k', ls='--', lw=0.8)\n",
        "rmse, mae, corr1 = metrics_pd(y_test_unscaled[:, 0], yhat[:, 0])\n",
        "ax.text(3, 10, r\"$r^2$\" + f\"={corr1:.2f}\")\n",
        "ax.text(3, 9.5, r\"$RMSE$\" + f\"={rmse:.2f}\")\n",
        "ax.text(3, 9, r\"$MAE$\" + f\"={mae:.2f}\")\n",
        "\n",
        "ax1.scatter(y_test_unscaled[:, 1], yhat[:, 1], s=1)\n",
        "ax1.set_xlabel(r\"$Dm \\ - \\ True $\")\n",
        "ax1.set_ylabel(r\"$Dm \\ - \\ Est. $\")\n",
        "x = np.linspace(*ax1.get_xlim())\n",
        "ax1.plot(x, x, c='k', ls='--', lw=0.8)\n",
        "rmse, mae, corr1 = metrics_pd(y_test_unscaled[:, 1], yhat[:, 1])\n",
        "ax1.text(0.3, 2.4, r\"$r^2$\" + f\"={corr1:.2f}\")\n",
        "ax1.text(0.3, 2.2, r\"$RMSE$\" + f\"={rmse:.2f}\")\n",
        "ax1.text(0.3, 2.0, r\"$MAE$\" + f\"={mae:.2f}\")\n",
        "\n",
        "ax2.scatter(y_test_unscaled[:, 2], yhat[:, 2], s=1)\n",
        "ax2.set_xlabel(r\"$\\mu  \\ - \\ True $\")\n",
        "ax2.set_ylabel(r\"$\\mu \\ - \\ Est. $\")\n",
        "rmse, mae, corr1 = metrics_pd(y_test_unscaled[:, 2], yhat[:, 2])\n",
        "ax2.text(0, 120, r\"$r^2$\" + f\"={corr1:.2f}\")\n",
        "ax2.text(0, 110, r\"$RMSE$\" + f\"={rmse:.2f}\")\n",
        "ax2.text(0, 100, r\"$MAE$\" + f\"={mae:.2f}\")\n",
        "x = np.linspace(*ax2.get_xlim())\n",
        "ax2.plot(x, x, c='k', ls='--', lw=0.8)\n",
        "\n",
        "plt.tight_layout()"
      ],
      "metadata": {
        "id": "qsrRmgYHMt0P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bar_colors = ['tab:red', 'tab:blue']\n",
        "fig, ax = plt.subplots()\n",
        "ax.bar(['Lower', \"Greater\"], np.bincount(y_train), color=bar_colors)\n",
        "ax.set_ylabel(\"Counts\")"
      ],
      "metadata": {
        "id": "uWOrK6eTLNYH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bar_colors = ['tab:red', 'tab:blue']\n",
        "fig, ax = plt.subplots()\n",
        "ax.bar(['Lower', \"Greater\"], np.bincount(y_res), color=bar_colors)\n",
        "ax.set_ylabel(\"Counts\")"
      ],
      "metadata": {
        "id": "Q_CYWFxHK_8Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Multi-class classification\n",
        "\n",
        "Lets see if instead of predicting the parameters of the NG psd, we can predict which class it belong to considering our previous kmeans classifier."
      ],
      "metadata": {
        "id": "BMEEbY7rz5x6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X, Y = df[['dbz_t_ku', 'dfr']], df['kmeans_6']\n",
        "scaler_x = StandardScaler()\n",
        "X_scaled = scaler_x.fit_transform(X)"
      ],
      "metadata": {
        "id": "qlcPIlBw8XH6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_test, train_labels, test_labels = train_test_split(X_scaled, Y, test_size=0.33, random_state=42)"
      ],
      "metadata": {
        "id": "OqoBhWRo9IE2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "over_sampler = RandomOverSampler(random_state=42)\n",
        "X_train_res, train_lab_res = over_sampler.fit_resample(X_train, train_labels)\n",
        "print(f\"Training target statistics: {Counter(train_lab_res)}\")\n",
        "print(f\"Testing target statistics: {Counter(test_labels)}\")"
      ],
      "metadata": {
        "id": "jNgzLMDO9H7g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def build_model2():\n",
        "    model = tf.keras.models.Sequential()\n",
        "    model.add(tf.keras.layers.Dense(2, activation=\"relu\",\n",
        "                                    input_shape=[X_train.shape[1]]))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    # model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    # model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(8, activation=\"relu\"))\n",
        "    model.add(tf.keras.layers.Dense(6,  activation=\"softmax\"))\n",
        "    # optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)\n",
        "    # optimizer = tf.keras.optimizers.Adamax(learning_rate=0.001)\n",
        "    # optimizer = tf.keras.optimizers.RMSprop(0.001)\n",
        "    # optimizer = tf.keras.optimizers.Adadelta(0.1)\n",
        "    # optimizer = tf.keras.optimizers.AdamW(0.001)\n",
        "    optimizer = tf.keras.optimizers.SGD(0.001)\n",
        "    model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "                  optimizer=optimizer,\n",
        "                  metrics=['accuracy'])\n",
        "    return model"
      ],
      "metadata": {
        "id": "K09bdzn1POQq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model2()"
      ],
      "metadata": {
        "id": "At7ngKdJ87q9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCHS = 500\n",
        "history = model.fit(X_train_res, train_lab_res,\n",
        "                    validation_data=(X_test, test_labels),\n",
        "                    epochs=EPOCHS,\n",
        "                    batch_size=32, verbose=0,\n",
        "                    callbacks=[PrintDot(), early_stop])"
      ],
      "metadata": {
        "id": "rkmw-gKJ9XOb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hist = pd.DataFrame(history.history)\n",
        "hist"
      ],
      "metadata": {
        "id": "0m25SWRj9zmc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hist.plot()"
      ],
      "metadata": {
        "id": "rmvyMrAi-wAD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lrs = 1e-3 * (10**(np.arange(101)/20))\n",
        "plt.semilogx(lrs, hist[\"loss\"])\n",
        "plt.semilogx(lrs, hist[\"val_loss\"]) # want the x-axis to be log-scale\n",
        "plt.xlabel(\"Learning rate\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.title(\"Finding the ideal learning rate\");"
      ],
      "metadata": {
        "id": "mQ-zDza5_UTN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "pGsghDqKBtkw"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}