LightGBM_Final.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R2hIZAcDl2t-"
      },
      "source": [
        "**IMPORT PACKAGES | BUILD CUSTOM FUNCTIONS | SET PARAMETERS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z7jZlcUq-yWt"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import logging\n",
        "import datetime\n",
        "import lightgbm as lgb\n",
        "import random\n",
        "import os\n",
        "import psutil\n",
        "import argparse\n",
        "import time\n",
        "import warnings\n",
        "import gc\n",
        "import pickle\n",
        "import math\n",
        "import shutil\n",
        "import math, decimal\n",
        "\n",
        "from math import ceil\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b8q54FH0f4v1"
      },
      "outputs": [],
      "source": [
        "random.seed(42)\n",
        "np.random.seed(42)\n",
        "os.environ['PYTHONHASHSEED'] = str(42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XmaLAOnTMUnE"
      },
      "outputs": [],
      "source": [
        "class Util(object):\n",
        "    @staticmethod\n",
        "    def set_seed(seed):\n",
        "        random.seed(seed)\n",
        "        np.random.seed(seed)\n",
        "        os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "        return\n",
        "\n",
        "    @staticmethod\n",
        "    def get_memory_usage():\n",
        "        return np.round(psutil.Process(os.getpid()).memory_info()[0] / 2. ** 30, 2)\n",
        "\n",
        "    @staticmethod\n",
        "    def reduce_mem_usage(df, verbose=False):\n",
        "        numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
        "        start_mem = df.memory_usage().sum() / 1024 ** 2\n",
        "        for col in df.columns:\n",
        "            col_type = df[col].dtypes\n",
        "            if col_type in numerics:\n",
        "                c_min = df[col].min()\n",
        "                c_max = df[col].max()\n",
        "                if str(col_type)[:3] == 'int':\n",
        "                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
        "                        df[col] = df[col].astype(np.int8)\n",
        "                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
        "                        df[col] = df[col].astype(np.int16)\n",
        "                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
        "                        df[col] = df[col].astype(np.int32)\n",
        "                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
        "                        df[col] = df[col].astype(np.int64)\n",
        "                else:\n",
        "                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
        "                        df[col] = df[col].astype(np.float16)\n",
        "                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
        "                        df[col] = df[col].astype(np.float32)\n",
        "                    else:\n",
        "                        df[col] = df[col].astype(np.float64)\n",
        "        end_mem = df.memory_usage().sum() / 1024 ** 2\n",
        "        if verbose:\n",
        "            print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (\n",
        "                    start_mem - end_mem) / start_mem))\n",
        "        return df\n",
        "\n",
        "    @staticmethod\n",
        "    def merge_by_concat(df1, df2, merge_on):\n",
        "        merged_gf = df1[merge_on]\n",
        "        merged_gf = merged_gf.merge(df2, on=merge_on, how='left')\n",
        "        new_columns = [col for col in list(merged_gf) if col not in merge_on]\n",
        "        df1 = pd.concat([df1, merged_gf[new_columns]], axis=1)\n",
        "        return df1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-3pEQYCuHBFl"
      },
      "outputs": [],
      "source": [
        "main_index_list = ['id', 'd']\n",
        "target = 'demand'\n",
        "prediction_horizon = 28\n",
        "start_train_day_x = 1\n",
        "end_train_day_x = 1913\n",
        "\n",
        "remove_features = ['id', 'state_id', 'store_id', 'wm_yr_wk', 'd', target]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9pjGiFFZi7Zq"
      },
      "source": [
        "**OPTION TO DOWNLOAD DATA**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FGcLX8D1BrPM",
        "outputId": "8385d4ca-1c03-4b72-a5a6-38a49f9fafee"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting dask_xgboost\n",
            "  Downloading dask_xgboost-0.2.0-py2.py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: distributed>=1.15.2 in /usr/local/lib/python3.7/dist-packages (from dask_xgboost) (2022.2.0)\n",
            "Requirement already satisfied: xgboost<=0.90 in /usr/local/lib/python3.7/dist-packages (from dask_xgboost) (0.90)\n",
            "Requirement already satisfied: dask in /usr/local/lib/python3.7/dist-packages (from dask_xgboost) (2022.2.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (2.11.3)\n",
            "Requirement already satisfied: msgpack>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (1.0.4)\n",
            "Requirement already satisfied: tornado>=5 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (5.1.1)\n",
            "Requirement already satisfied: click>=6.6 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (7.1.2)\n",
            "Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (2.4.0)\n",
            "Requirement already satisfied: psutil>=5.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (5.4.8)\n",
            "Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (1.7.0)\n",
            "Requirement already satisfied: toolz>=0.8.2 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (0.12.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (21.3)\n",
            "Requirement already satisfied: cloudpickle>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (1.5.0)\n",
            "Requirement already satisfied: zict>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (2.2.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from distributed>=1.15.2->dask_xgboost) (57.4.0)\n",
            "Requirement already satisfied: partd>=0.3.10 in /usr/local/lib/python3.7/dist-packages (from dask->dask_xgboost) (1.3.0)\n",
            "Requirement already satisfied: fsspec>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from dask->dask_xgboost) (2022.8.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->distributed>=1.15.2->dask_xgboost) (3.0.9)\n",
            "Requirement already satisfied: locket in /usr/local/lib/python3.7/dist-packages (from partd>=0.3.10->dask->dask_xgboost) (1.0.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from xgboost<=0.90->dask_xgboost) (1.7.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from xgboost<=0.90->dask_xgboost) (1.21.6)\n",
            "Requirement already satisfied: heapdict in /usr/local/lib/python3.7/dist-packages (from zict>=0.1.3->distributed>=1.15.2->dask_xgboost) (1.0.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->distributed>=1.15.2->dask_xgboost) (2.0.1)\n",
            "Installing collected packages: dask-xgboost\n",
            "Successfully installed dask-xgboost-0.2.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.7/dist-packages (1.5.12)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.8.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.23.0)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.7/dist-packages (from kaggle) (6.1.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kaggle) (2022.9.24)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from kaggle) (4.64.1)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.7/dist-packages (from python-slugify->kaggle) (1.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (2.10)\n",
            "Downloading m5-forecasting-accuracy.zip to /content\n",
            "\n",
            "Archive:  m5-forecasting-accuracy.zip\n",
            "  inflating: m5-forecasting-accuracy/calendar.csv  \n",
            "  inflating: m5-forecasting-accuracy/sales_train_evaluation.csv  \n",
            "  inflating: m5-forecasting-accuracy/sales_train_validation.csv  \n",
            "  inflating: m5-forecasting-accuracy/sample_submission.csv  \n",
            "  inflating: m5-forecasting-accuracy/sell_prices.csv  \n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0.00/45.8M [00:00<?, ?B/s]\r 11%|█         | 5.00M/45.8M [00:00<00:02, 16.1MB/s]\r 20%|█▉        | 9.00M/45.8M [00:00<00:01, 19.4MB/s]\r 72%|███████▏  | 33.0M/45.8M [00:00<00:00, 66.2MB/s]\r 90%|████████▉ | 41.0M/45.8M [00:01<00:00, 41.6MB/s]\r100%|██████████| 45.8M/45.8M [00:01<00:00, 43.9MB/s]\n"
          ]
        }
      ],
      "source": [
        "# %%bash\n",
        "# pip install dask_xgboost\n",
        "# pip install kaggle\n",
        "# export KAGGLE_USERNAME=jmiller558\n",
        "# export KAGGLE_KEY=812fcd89e3a0fc00cb629bf2306b215e\n",
        "\n",
        "# kaggle competitions download -c m5-forecasting-accuracy\n",
        "\n",
        "# unzip -n m5-forecasting-accuracy -d m5-forecasting-accuracy\n",
        "# rm -rf sample_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IyKatBJ6i_9e"
      },
      "source": [
        "**OPTION TO MOUNT DRIVE WITH DATA**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VFq6kjNqhudj",
        "outputId": "f04abc34-22c3-4bc8-ede5-44319353272c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sGNv8eNojn89"
      },
      "source": [
        "**PATHS FOR DATA**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KKebCVdEjnc-"
      },
      "outputs": [],
      "source": [
        "#input paths for base data\n",
        "trainpath = '/content/drive/MyDrive/Capstone/Models/m5-forecasting-accuracy/sales_train_evaluation.csv'\n",
        "pricepath = '/content/drive/MyDrive/Capstone/Models/m5-forecasting-accuracy/sell_prices.csv'\n",
        "calpath = '/content/drive/MyDrive/Capstone/Models/m5-forecasting-accuracy/calendar.csv'\n",
        "submissionpath = '/content/drive/MyDrive/Capstone/Models/m5-forecasting-accuracy/sample_submission.csv'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DveFC0PCjrKV"
      },
      "outputs": [],
      "source": [
        "#input paths for Feature Engineered DFs\n",
        "grid_base_path = '/content/drive/MyDrive/Capstone/Models/Top4_Original/grid_base'\n",
        "calfeats_path = '/content/drive/MyDrive/Capstone/Models/Top4_Original/calfeats'\n",
        "pricefeats_path = '/content/drive/MyDrive/Capstone/Models/Top4_Original/pricefeats'\n",
        "encoding_path = '/content/drive/MyDrive/Capstone/Models/Top4_Original/encodingfeats'\n",
        "lagfeats_path = '/content/drive/MyDrive/Capstone/Models/Top4_Original/lagfeats_'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_97Wz_4WNawJ"
      },
      "source": [
        "**LOAD BASE COMPETITION DATA**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iXPfpO4-9gjY"
      },
      "outputs": [],
      "source": [
        "def load_data():\n",
        "    train_df = pd.read_csv(trainpath)\n",
        "    prices_df = pd.read_csv(pricepath)\n",
        "    calendar_df = pd.read_csv(calpath)\n",
        "    submission_df = pd.read_csv(submissionpath)\n",
        "\n",
        "    return train_df, prices_df, calendar_df, submission_df\n",
        "\n",
        "train_df, prices_df, calendar_df, submission_df = load_data()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kzQlqfnalod_"
      },
      "source": [
        "**BASE FEATURE ENGINEERING (ONLY NEEDED FIRST TIME)**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZsA5z4k9NgOS"
      },
      "source": [
        "\n",
        "\n",
        "*   BUILD GRID_BASE\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2_0TwxFAEbyD"
      },
      "outputs": [],
      "source": [
        "index_columns = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']\n",
        "grid_df = pd.melt(train_df, id_vars=index_columns, var_name='d', value_name=target)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XlM-VXNoEsBC"
      },
      "outputs": [],
      "source": [
        "grid_df['d_org'] = grid_df['d']\n",
        "grid_df['d'] = grid_df['d'].apply(lambda x: x[2:]).astype(np.int16)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x4hJL5X8Iv1Z"
      },
      "outputs": [],
      "source": [
        "grid_df = grid_df[grid_df['d'] <= end_train_day_x]\n",
        "grid_df['d'] = grid_df['d_org']\n",
        "grid_df = grid_df.drop('d_org', axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NmDYe5HhJsKE"
      },
      "outputs": [],
      "source": [
        "add_grid = pd.DataFrame()\n",
        "for i in range(prediction_horizon):\n",
        "    temp_df = train_df[index_columns]\n",
        "    temp_df = temp_df.drop_duplicates()\n",
        "    temp_df['d'] = 'd_' + str(end_train_day_x + i + 1)\n",
        "    temp_df[target] = np.nan\n",
        "    add_grid = pd.concat([add_grid, temp_df])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iFmEvgoBKVud"
      },
      "outputs": [],
      "source": [
        "grid_df = pd.concat([grid_df, add_grid])\n",
        "grid_df = grid_df.reset_index(drop=True)\n",
        "\n",
        "del temp_df, add_grid"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xmXJ7PUDLHN-"
      },
      "outputs": [],
      "source": [
        "for col in index_columns:\n",
        "        grid_df[col] = grid_df[col].astype('category')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O4bPNTXtLkVF"
      },
      "outputs": [],
      "source": [
        "release_df = prices_df.groupby(['store_id', 'item_id'])['wm_yr_wk'].agg(['min']).reset_index()\n",
        "release_df.columns = ['store_id', 'item_id', 'release']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CG7CJ6ojLZuC"
      },
      "outputs": [],
      "source": [
        "grid_df = Util.merge_by_concat(grid_df, release_df, ['store_id', 'item_id'])\n",
        "del release_df\n",
        "grid_df = Util.merge_by_concat(grid_df, calendar_df[['wm_yr_wk', 'd']], ['d'])\n",
        "grid_df = grid_df.reset_index(drop=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3WZ1nHWMMepX"
      },
      "outputs": [],
      "source": [
        "grid_df['release'] = grid_df['release'] - grid_df['release'].min()\n",
        "grid_df['release'] = grid_df['release'].astype(np.int16)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sL7Mrm0Gj85M"
      },
      "outputs": [],
      "source": [
        "grid_df.to_pickle(grid_base_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sxU_dhFWhH0v"
      },
      "outputs": [],
      "source": [
        "del grid_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fM_8RJ_aYSZO"
      },
      "source": [
        "*   BUILD CAL FEATURES\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TyAIhDXjYX56"
      },
      "outputs": [],
      "source": [
        "grid_df = pd.read_pickle(grid_base_path)\n",
        "calfeats_df = grid_df[main_index_list]\n",
        "dec = decimal.Decimal\n",
        "\n",
        "def get_moon_phase(d):  # 0=new, 4=full; 4 days/phase\n",
        "        diff = datetime.datetime.strptime(d, '%Y-%m-%d') - datetime.datetime(2001, 1, 1)\n",
        "        days = dec(diff.days) + (dec(diff.seconds) / dec(86400))\n",
        "        lunations = dec(\"0.20439731\") + (days * dec(\"0.03386319269\"))\n",
        "        phase_index = math.floor((lunations % dec(1) * dec(8)) + dec('0.5'))\n",
        "        return int(phase_index) & 7\n",
        "        \n",
        "calendar_df['moon'] = calendar_df.date.apply(get_moon_phase)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0mM0uATpZFJ8"
      },
      "outputs": [],
      "source": [
        "icols = ['date',\n",
        "          'd',\n",
        "          'event_name_1',\n",
        "          'event_type_1',\n",
        "          'event_name_2',\n",
        "          'event_type_2',\n",
        "          'snap_CA',\n",
        "          'snap_TX',\n",
        "          'snap_WI',\n",
        "          'moon',\n",
        "          ]\n",
        "\n",
        "calfeats_df = calfeats_df.merge(calendar_df[icols], on=['d'], how='left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ef5AajEDZGU1"
      },
      "outputs": [],
      "source": [
        "icols = ['event_name_1',\n",
        "          'event_type_1',\n",
        "          'event_name_2',\n",
        "          'event_type_2',\n",
        "          'snap_CA',\n",
        "          'snap_TX',\n",
        "          'snap_WI']\n",
        "for col in icols:\n",
        "  calfeats_df[col] = calfeats_df[col].astype('category')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ccsgmLdCZNNj"
      },
      "outputs": [],
      "source": [
        "calfeats_df['date'] = pd.to_datetime(calfeats_df['date'])\n",
        "\n",
        "calfeats_df['tm_d'] = calfeats_df['date'].dt.day.astype(np.int8)\n",
        "calfeats_df['tm_w'] = calfeats_df['date'].dt.week.astype(np.int8)\n",
        "calfeats_df['tm_m'] = calfeats_df['date'].dt.month.astype(np.int8)\n",
        "calfeats_df['tm_y'] = calfeats_df['date'].dt.year\n",
        "calfeats_df['tm_y'] = (calfeats_df['tm_y'] - calfeats_df['tm_y'].min()).astype(np.int8)\n",
        "calfeats_df['tm_wm'] = calfeats_df['tm_d'].apply(lambda x: ceil(x / 7)).astype(np.int8)\n",
        "\n",
        "calfeats_df['tm_dw'] = calfeats_df['date'].dt.dayofweek.astype(np.int8)\n",
        "calfeats_df['tm_w_end'] = (calfeats_df['tm_dw'] >= 5).astype(np.int8)\n",
        "\n",
        "del calfeats_df['date']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OWfN6_u_ZNxg"
      },
      "outputs": [],
      "source": [
        "calfeats_df.to_pickle(calfeats_path)\n",
        "del calfeats_df\n",
        "del grid_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iIff0KflNNKr"
      },
      "source": [
        "*   BUILD PRICING FEATURES\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZZ_1SPJFNk4C"
      },
      "outputs": [],
      "source": [
        "calendar_prices = calendar_df[['wm_yr_wk', 'month', 'year']]\n",
        "calendar_prices = calendar_prices.drop_duplicates(subset=['wm_yr_wk'])\n",
        "prices_df = prices_df.merge(calendar_prices[['wm_yr_wk', 'month', 'year']], on=['wm_yr_wk'], how='left')\n",
        "del calendar_prices\n",
        "\n",
        "grid_df = pd.read_pickle(grid_base_path)\n",
        "\n",
        "prices_df = prices_df[prices_df['wm_yr_wk']<=grid_df['wm_yr_wk'].max()]\n",
        "\n",
        "prices_df['price_max'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('max')\n",
        "prices_df['price_min'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('min')\n",
        "prices_df['price_std'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('std')\n",
        "prices_df['price_mean'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('mean')\n",
        "prices_df['price_norm'] = prices_df['sell_price'] / prices_df['price_max']\n",
        "prices_df['price_nunique'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('nunique')\n",
        "prices_df['item_nunique'] = prices_df.groupby(['store_id', 'sell_price'])['item_id'].transform('nunique')\n",
        "\n",
        "prices_df['price_momentum'] = prices_df['sell_price'] / prices_df.groupby(['store_id', 'item_id'])[\n",
        "    'sell_price'].transform(lambda x: x.shift(1))\n",
        "prices_df['price_momentum_m'] = prices_df['sell_price'] / prices_df.groupby(['store_id', 'item_id', 'month'])[\n",
        "    'sell_price'].transform('mean')\n",
        "prices_df['price_momentum_y'] = prices_df['sell_price'] / prices_df.groupby(['store_id', 'item_id', 'year'])[\n",
        "    'sell_price'].transform('mean')\n",
        "\n",
        "prices_df['sell_price_cent'] = [math.modf(p)[0] for p in prices_df['sell_price']]\n",
        "prices_df['price_max_cent'] = [math.modf(p)[0] for p in prices_df['price_max']]\n",
        "prices_df['price_min_cent'] = [math.modf(p)[0] for p in prices_df['price_min']]\n",
        "\n",
        "del prices_df['month'], prices_df['year']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w54wbcXzN1CY"
      },
      "outputs": [],
      "source": [
        "grid_df = pd.read_pickle(grid_base_path)\n",
        "original_columns = list(grid_df)\n",
        "pricefeats_df = grid_df.merge(prices_df, on=['store_id', 'item_id', 'wm_yr_wk'], how='left')\n",
        "keep_columns = [col for col in list(pricefeats_df) if col not in original_columns]\n",
        "pricefeats_df = pricefeats_df[main_index_list + keep_columns]\n",
        "pricefeats_df = Util.reduce_mem_usage(pricefeats_df)\n",
        "del prices_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mt5EAKaLnDtM"
      },
      "outputs": [],
      "source": [
        "pricefeats_df.to_pickle(pricefeats_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sE8DHKaGhMZx"
      },
      "outputs": [],
      "source": [
        "del pricefeats_df\n",
        "del grid_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NNjbwkypP760"
      },
      "source": [
        "*   CREATE ENCODING FEATURES\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5PSvyjOKQJKS"
      },
      "outputs": [],
      "source": [
        "encoding_df = pd.read_pickle(grid_base_path)\n",
        "encoding_df['d'] = encoding_df['d'].apply(lambda x: x[2:]).astype(np.int16)\n",
        "encoding_df[encoding_df['d']>(end_train_day_x)][target] = np.nan\n",
        "                       \n",
        "base_cols = list(encoding_df)\n",
        "\n",
        "icols = [['cat_id'],\n",
        "    ['dept_id'],\n",
        "    ['item_id']]\n",
        "\n",
        "for col in icols:\n",
        "    col_name = '_' + '_'.join(col) + '_'\n",
        "    encoding_df['enc' + col_name + 'mean'] = encoding_df.groupby(col)[target].transform('mean').astype(\n",
        "        np.float16)\n",
        "    encoding_df['enc' + col_name + 'std'] = encoding_df.groupby(col)[target].transform('std').astype(\n",
        "        np.float16)\n",
        "\n",
        "keep_cols = [col for col in list(encoding_df) if col not in base_cols]\n",
        "encoding_df = encoding_df[['id', 'd'] + keep_cols]\n",
        "\n",
        "encoding_df.to_pickle(encoding_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iVbeN1f1hYc0"
      },
      "outputs": [],
      "source": [
        "del encoding_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "btuBOAiQPn5e"
      },
      "source": [
        "**CREATE LAG FEATURES**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4cMCE--mP624"
      },
      "outputs": [],
      "source": [
        "for prediction_horizon in [7,14,21,28]:\n",
        "    num_lag_day_list = []\n",
        "    num_lag_day = 15\n",
        "    for col in range(prediction_horizon, prediction_horizon + num_lag_day):\n",
        "        num_lag_day_list.append(col)\n",
        "    num_rolling_day_list = [7, 14, 30, 60, 180]\n",
        "\n",
        "    lagfeats_df = pd.read_pickle(grid_base_path)\n",
        "    lagfeats_df['d'] = lagfeats_df['d'].apply(lambda x: x[2:]).astype(np.int16)\n",
        "    lagfeats_df = lagfeats_df[['id', 'd', target]]\n",
        "    lagfeats_df.loc[lagfeats_df['d']>end_train_day_x,target] = np.nan\n",
        "\n",
        "    lagfeats_df = lagfeats_df.assign(**{\n",
        "        '{}_lag_{}'.format(col, l): lagfeats_df.groupby(['id'])[col].transform(lambda x: x.shift(l))\n",
        "        for l in num_lag_day_list\n",
        "        for col in [target]\n",
        "    })\n",
        "\n",
        "    for col in list(lagfeats_df):\n",
        "        if 'lag' in col:\n",
        "            lagfeats_df[col] = lagfeats_df[col].astype(np.float16)\n",
        "\n",
        "    for num_rolling_day in num_rolling_day_list:\n",
        "        lagfeats_df['rolling_mean_' + str(num_rolling_day)] = lagfeats_df.groupby(['id'])[target].transform(\n",
        "            lambda x: x.shift(prediction_horizon).rolling(num_rolling_day).mean()).astype(np.float16)\n",
        "        lagfeats_df['rolling_std_' + str(num_rolling_day)] = lagfeats_df.groupby(['id'])[target].transform(\n",
        "            lambda x: x.shift(prediction_horizon).rolling(num_rolling_day).std()).astype(np.float16)\n",
        "\n",
        "    lagfeats_df.to_pickle(lagfeats_path+str(prediction_horizon))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EYgjEtkIYvCk"
      },
      "source": [
        "**CREATE FULL DATASET BY STORE**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qiX04tdpYxtk"
      },
      "outputs": [],
      "source": [
        "def load_df(store_id,end_train_day_x,prediction_horizon):\n",
        "    \n",
        "    grid_base = pd.read_pickle(grid_base_path)\n",
        "    grid_base['d'] = grid_base['d'].apply(lambda x: x[2:]).astype(np.int16)\n",
        "\n",
        "    pricefeats = pd.read_pickle(pricefeats_path)\n",
        "    calfeats = pd.read_pickle(calfeats_path)\n",
        "    encodingfeats = pd.read_pickle(encoding_path)\n",
        "    \n",
        "    full_df = pd.concat([grid_base,\n",
        "                         pricefeats.iloc[:, 2:],\n",
        "                         calfeats.iloc[:, 2:],\n",
        "                         encodingfeats.iloc[:, 2:]],\n",
        "                        axis=1)\n",
        "\n",
        "    del grid_base,pricefeats,calfeats,encodingfeats\n",
        "\n",
        "    if store_id != 'all':\n",
        "            full_df = full_df[full_df['store_id'] == store_id]\n",
        "\n",
        "    full_df = full_df[full_df['d']<=(end_train_day_x+prediction_horizon)]\n",
        "\n",
        "    lagfeats = pd.read_pickle(lagfeats_path+str(prediction_horizon))\n",
        "    lagfeats = lagfeats.iloc[:, 3:]\n",
        "    lagfeats= lagfeats[lagfeats.index.isin(full_df.index)]\n",
        "\n",
        "    full_df = pd.concat([full_df, lagfeats], axis=1)\n",
        "    del lagfeats\n",
        "\n",
        "    enable_features = [col for col in list(full_df) if col not in remove_features]\n",
        "    full_df = full_df[['id', 'd', target] + enable_features]\n",
        "\n",
        "    full_df = full_df[full_df['d'] >= start_train_day_x].reset_index(drop=True)\n",
        "\n",
        "    return full_df, enable_features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yx0gmT8yqyWz"
      },
      "source": [
        "**RUN MODEL**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XEXQXABnq5q2",
        "outputId": "67ab9123-d41e-4ef3-b499-28f58b640fe1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[100]\tvalid_0's rmse: 1.69174\n",
            "[200]\tvalid_0's rmse: 1.67442\n",
            "[300]\tvalid_0's rmse: 1.67118\n",
            "[400]\tvalid_0's rmse: 1.66637\n",
            "[500]\tvalid_0's rmse: 1.6598\n",
            "[600]\tvalid_0's rmse: 1.65214\n",
            "[700]\tvalid_0's rmse: 1.64499\n",
            "[800]\tvalid_0's rmse: 1.63734\n",
            "[900]\tvalid_0's rmse: 1.63355\n",
            "[1000]\tvalid_0's rmse: 1.62805\n",
            "[1100]\tvalid_0's rmse: 1.62391\n",
            "[1200]\tvalid_0's rmse: 1.61904\n",
            "[1300]\tvalid_0's rmse: 1.61344\n",
            "[1400]\tvalid_0's rmse: 1.60904\n",
            "Our val rmse score is 1.6090388370146935\n",
            "[100]\tvalid_0's rmse: 1.64325\n",
            "[200]\tvalid_0's rmse: 1.62582\n",
            "[300]\tvalid_0's rmse: 1.6219\n",
            "[400]\tvalid_0's rmse: 1.61683\n",
            "[500]\tvalid_0's rmse: 1.60954\n",
            "[600]\tvalid_0's rmse: 1.60308\n",
            "[700]\tvalid_0's rmse: 1.59587\n",
            "[800]\tvalid_0's rmse: 1.59038\n",
            "[900]\tvalid_0's rmse: 1.58495\n",
            "[1000]\tvalid_0's rmse: 1.58036\n",
            "[1100]\tvalid_0's rmse: 1.57615\n",
            "[1200]\tvalid_0's rmse: 1.5706\n",
            "[1300]\tvalid_0's rmse: 1.56762\n",
            "[1400]\tvalid_0's rmse: 1.5659\n",
            "Our val rmse score is 1.565897473590644\n",
            "[100]\tvalid_0's rmse: 1.72879\n",
            "[200]\tvalid_0's rmse: 1.7\n",
            "[300]\tvalid_0's rmse: 1.69013\n",
            "[400]\tvalid_0's rmse: 1.67995\n",
            "[500]\tvalid_0's rmse: 1.66842\n",
            "[600]\tvalid_0's rmse: 1.65698\n",
            "[700]\tvalid_0's rmse: 1.64552\n",
            "[800]\tvalid_0's rmse: 1.63749\n",
            "[900]\tvalid_0's rmse: 1.62764\n",
            "[1000]\tvalid_0's rmse: 1.61923\n",
            "[1100]\tvalid_0's rmse: 1.61254\n",
            "[1200]\tvalid_0's rmse: 1.60593\n",
            "[1300]\tvalid_0's rmse: 1.59926\n",
            "[1400]\tvalid_0's rmse: 1.59177\n",
            "Our val rmse score is 1.591765760079572\n",
            "[100]\tvalid_0's rmse: 1.7933\n",
            "[200]\tvalid_0's rmse: 1.75895\n",
            "[300]\tvalid_0's rmse: 1.74065\n",
            "[400]\tvalid_0's rmse: 1.72482\n",
            "[500]\tvalid_0's rmse: 1.70768\n",
            "[600]\tvalid_0's rmse: 1.6934\n",
            "[700]\tvalid_0's rmse: 1.67823\n",
            "[800]\tvalid_0's rmse: 1.66703\n",
            "[900]\tvalid_0's rmse: 1.65907\n",
            "[1000]\tvalid_0's rmse: 1.65054\n",
            "[1100]\tvalid_0's rmse: 1.6404\n",
            "[1200]\tvalid_0's rmse: 1.6309\n",
            "[1300]\tvalid_0's rmse: 1.6237\n",
            "[1400]\tvalid_0's rmse: 1.61526\n",
            "Our val rmse score is 1.6152619593515078\n"
          ]
        }
      ],
      "source": [
        "lgb_params = {\n",
        "    'boosting_type': 'gbdt',\n",
        "    'objective': 'tweedie',\n",
        "    'tweedie_variance_power': 1.1,\n",
        "    'metric': 'rmse',\n",
        "    'subsample': 0.5,\n",
        "    'subsample_freq': 1,\n",
        "    'learning_rate': 0.03,\n",
        "    'num_leaves': 2 ** 11 - 1,\n",
        "    'min_data_in_leaf': 2 ** 12 - 1,\n",
        "    'feature_fraction': 0.5,\n",
        "    'max_bin': 100,\n",
        "    'n_estimators': 1400,\n",
        "    'boost_from_average': False,\n",
        "}\n",
        "\n",
        "store_id_set_list = list(train_df['store_id'].unique())\n",
        "\n",
        "for store_index, store_id in enumerate(store_id_set_list):\n",
        "    for prediction_horizon in [7,14,21,28]:\n",
        "      save_name = '/content/drive/MyDrive/Capstone/Models/Top4_Original/1913-1941/' + str(store_id)+'-'+str(prediction_horizon)+'-'+'.csv'\n",
        "    \n",
        "      grid_df,enable_features = load_df(store_id,end_train_day_x,prediction_horizon)\n",
        "\n",
        "      x_train = grid_df[(grid_df['d'] >= start_train_day_x) & (grid_df['d'] <= end_train_day_x)]\n",
        "      y_train = x_train[target]\n",
        "      x_val = grid_df[(grid_df['d'] > (end_train_day_x - prediction_horizon)) & (grid_df['d'] <= end_train_day_x)]\n",
        "      y_val = x_val[target]\n",
        "      \n",
        "      test = grid_df[grid_df['d'] > end_train_day_x]\n",
        "      \n",
        "      train_data = lgb.Dataset(x_train[enable_features],\n",
        "                                      label=y_train)\n",
        "      \n",
        "      val_data = lgb.Dataset(x_val[enable_features],\n",
        "                                      label=y_val)\n",
        "      \n",
        "      del grid_df, x_train, y_train\n",
        "      gc.collect()\n",
        "      \n",
        "      estimator = lgb.train(lgb_params,train_data,valid_sets = [val_data], verbose_eval = 100)\n",
        "      \n",
        "      val_pred = estimator.predict(x_val[enable_features])\n",
        "      val_score = np.sqrt(mean_squared_error(val_pred, y_val))\n",
        "      print(f'Our val rmse score is {val_score}')\n",
        "\n",
        "      y_pred = estimator.predict(test[enable_features])\n",
        "      test[target] = y_pred\n",
        "\n",
        "      predictions = test[['id', 'd', target]]\n",
        "      predictions = pd.pivot(predictions, index = 'id', columns = 'd', values = target).reset_index()\n",
        "\n",
        "      predictions.to_csv(save_name,index=False)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}