03-holidays_feature.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Holiday Feature\n",
    "Feriendaten werden aus dem Internet gecrawlt und in ein passendes Format überführt. Es werden nur Feiertage für NRW betrachtet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn \n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from dateutil.relativedelta import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_dataframe(df, holyday_type):\n",
    "    \"\"\"df is the dataFrame, holyday_type is a string: name of holyday_type column\"\"\"\n",
    "    # Rename column\n",
    "    tmp = df.rename(columns={'holiday_name' : 'name'})\n",
    "    # Drop column\n",
    "    tmp = tmp.drop('holiday_type', axis=1)\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize DataFrame\n",
    "df_holidays = pd.DataFrame()\n",
    "\n",
    "# Loop over years\n",
    "for year in range(2016, 2018):\n",
    "    \n",
    "    url = 'http://www.schulferien.org/deutschland/feiertage/' + str(year) + '/'\n",
    "\n",
    "    # Fetch soup\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "    # Loop over holiday types (gesetzl. Feiertage, Festtage, Ereignisse)\n",
    "    for holiday_type, html_class in [('holiday_legal', 'row_panel gesetzlich_row'),\n",
    "                                     ('holiday_not_legal', 'row_panel nicht_gesetzlich_row'),\n",
    "                                     ('event', 'row_panel ereignis_row')]:\n",
    "\n",
    "        # Loop over rows\n",
    "        for row in soup.findAll('tr', {'class':html_class}):\n",
    "\n",
    "            name = row.find('a', {'class':'feiertag_name'}).text.replace('*', '').strip()\n",
    "            date = row.find('td', {'class':'feiertag_datum'}).div.contents[0].strip()[3:]\n",
    "\n",
    "            # Sometimes region cell is empty\n",
    "            try:\n",
    "                region = row.find('td', {'class':'feiertag_regionen sf_tooltip sf_hide_w_xs'}).text.strip()\n",
    "            except:\n",
    "                region = 'alle BL'\n",
    "\n",
    "            df_holidays = df_holidays.append(pd.DataFrame([(date, name, holiday_type, region)], \n",
    "                                                          columns=['date', 'holiday_name', 'holiday_type', \n",
    "                                                                   'holiday_region']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to datetime \n",
    "df_holidays['date'] = pd.to_datetime(df_holidays['date'], dayfirst=True)\n",
    "\n",
    "# Sort by date\n",
    "df_holidays.sort_values('date', inplace=True)\n",
    "\n",
    "# Drop 17. Juni 1953 (only legal holiday until 1990)\n",
    "df_holidays = df_holidays[df_holidays['holiday_name'] != '17. Juni 1953']\n",
    "\n",
    "# Drop Augsburger Friedensfest (only important for Augsburg)\n",
    "df_holidays = df_holidays[df_holidays['holiday_name'] != 'Augsburger Friedensfest']\n",
    "\n",
    "df_holidays_legal = df_holidays.loc[df_holidays['holiday_type'] == 'holiday_legal']\n",
    "df_holidays_not_legal = df_holidays.loc[df_holidays['holiday_type'] == 'holiday_not_legal']\n",
    "\n",
    "\n",
    "df_holidays_legal = format_dataframe(df_holidays_legal, \"holyday_legal\")\n",
    "df_holidays_not_legal = format_dataframe(df_holidays_not_legal, 'holiday_not_legal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BRÜCKENTAGE\n",
    "\n",
    "# Add weekday\n",
    "df_holidays_legal['weekday'] = df_holidays_legal['date'].dt.weekday\n",
    "\n",
    "# Initialize DataFrame\n",
    "df_bridge_days = pd.DataFrame()\n",
    "\n",
    "# Loop over all Thursday bank holidays and add Brückentage (Fridays)\n",
    "for datetime in df_holidays_legal.loc[df_holidays_legal['weekday'] == 3, 'date']:\n",
    "\n",
    "    # Fetch region\n",
    "    bridge_day_region = df_holidays_legal.loc[df_holidays_legal['date'] == datetime, 'holiday_region'][0]\n",
    "    \n",
    "    # Add a day\n",
    "    datetime_tomorrow = datetime + relativedelta(days=+1)\n",
    "    \n",
    "    # Assemble DataFrame\n",
    "    df_temp = pd.DataFrame([(datetime_tomorrow, 'Brückentag', bridge_day_region)],\n",
    "                                 columns=['date', 'name', 'holiday_region'])\n",
    "\n",
    "    #print(datetime_tomorrow, bridge_day_region)\n",
    "    \n",
    "    # Append\n",
    "    df_bridge_days = df_bridge_days.append(df_temp)\n",
    "    \n",
    "# Loop over all Thursday bank holidays and add Brückentage (Mondays)\n",
    "for datetime in df_holidays_legal.loc[df_holidays_legal['weekday'] == 1, 'date']:\n",
    "\n",
    "    # Fetch region\n",
    "    bridge_day_region = df_holidays_legal.loc[df_holidays_legal['date'] == datetime, 'holiday_region'][0]\n",
    "    \n",
    "    # Add a day\n",
    "    datetime_tomorrow = datetime + relativedelta(days=-1)\n",
    "    \n",
    "    # Assemble DataFrame\n",
    "    df_temp = pd.DataFrame([(datetime_tomorrow, 'Brückentag', bridge_day_region)],\n",
    "                                 columns=['date', 'name', 'holiday_region'])\n",
    "\n",
    "    #print(datetime_tomorrow, bridge_day_region)\n",
    "    \n",
    "    # Append\n",
    "    df_bridge_days = df_bridge_days.append(df_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>holiday_region</th>\n",
       "      <th>name</th>\n",
       "      <th>weekday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>alle BL</td>\n",
       "      <td>Neujahr</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-01-06</td>\n",
       "      <td>BW, BY, ST</td>\n",
       "      <td>Heilige Drei Könige</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-03-25</td>\n",
       "      <td>alle BL</td>\n",
       "      <td>Karfreitag</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-03-27</td>\n",
       "      <td>BB</td>\n",
       "      <td>Ostersonntag</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-03-28</td>\n",
       "      <td>alle BL</td>\n",
       "      <td>Ostermontag</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date holiday_region                 name  weekday\n",
       "0 2016-01-01        alle BL              Neujahr      4.0\n",
       "1 2016-01-06     BW, BY, ST  Heilige Drei Könige      2.0\n",
       "2 2016-03-25        alle BL           Karfreitag      4.0\n",
       "3 2016-03-27             BB         Ostersonntag      6.0\n",
       "4 2016-03-28        alle BL          Ostermontag      0.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_holidays = pd.concat([df_holidays_legal,df_holidays_not_legal,df_bridge_days])\n",
    "df_holidays = df_holidays.reset_index(drop=True)\n",
    "df_holidays.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only keep holidays in NRW.\n",
    "df_holidays = df_holidays[df_holidays.holiday_region.str.contains('NW')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# serialize output\n",
    "df_holidays.to_pickle('df_holidays.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}