explore.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "ee6c4641-271d-48a7-a59f-2351dc36b350",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "from sklearn.decomposition import NMF\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "03aceb3d-8732-40a2-8560-b64156f704e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./train_public/1h/L03.B02_1H.csv',\n",
       " './train_public/1h/L06.B01_1H.csv',\n",
       " './train_public/1h/L09.B01_1H.csv',\n",
       " './train_public/1h/L10.B01_1H.csv',\n",
       " './train_public/1h/L14.B01_1H.csv',\n",
       " './train_public/1h/L14.B02_1H.csv',\n",
       " './train_public/1h/L14.B03_1H.csv',\n",
       " './train_public/1h/L14.B04_1H.csv',\n",
       " './train_public/1h/L14.B05_1H.csv']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_1h = './train_public/1h'\n",
    "file_30min = './train_public/30min'\n",
    "file_15min = './train_public/15min'\n",
    "file_5min = './train_public/5min'\n",
    "\n",
    "csvfiles_1h = [file_1h + '/' + file for file in os.listdir(file_1h)]\n",
    "csvfiles_1h.sort()\n",
    "csvfiles_1h"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d56a9f4-d47f-4701-b0a1-48b9486f224d",
   "metadata": {},
   "source": [
    "<img src='figures/data_description.png' width=500, height=450>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dffa42cd-dbaa-44ca-8467-881469cc3de8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l03b02_1h (8400, 2)\n",
      "l06b01_1h (7674, 3)\n",
      "l09b01_1h (8287, 3)\n",
      "l10b01_1h (13867, 2)\n",
      "l14b01_1h (8760, 2)\n",
      "l14b02_1h (8760, 2)\n",
      "l14b03_1h (8760, 2)\n",
      "l14b04_1h (8760, 2)\n",
      "l14b05_1h (8760, 2)\n"
     ]
    }
   ],
   "source": [
    "l03b02_1h = pd.read_csv(csvfiles_1h[0])\n",
    "l06b01_1h = pd.read_csv(csvfiles_1h[1])\n",
    "l09b01_1h = pd.read_csv(csvfiles_1h[2])\n",
    "l10b01_1h = pd.read_csv(csvfiles_1h[3])\n",
    "l14b01_1h = pd.read_csv(csvfiles_1h[4])\n",
    "l14b02_1h = pd.read_csv(csvfiles_1h[5])\n",
    "l14b03_1h = pd.read_csv(csvfiles_1h[6])\n",
    "l14b04_1h = pd.read_csv(csvfiles_1h[7])\n",
    "l14b05_1h = pd.read_csv(csvfiles_1h[8])\n",
    "\n",
    "data_1h = {'l03b02_1h': l03b02_1h,\n",
    "           'l06b01_1h': l06b01_1h,  \n",
    "           'l09b01_1h': l09b01_1h,\n",
    "           'l10b01_1h': l10b01_1h, \n",
    "           'l14b01_1h': l14b01_1h, \n",
    "           'l14b02_1h': l14b02_1h, \n",
    "           'l14b03_1h': l14b03_1h, \n",
    "           'l14b04_1h': l14b04_1h, \n",
    "           'l14b05_1h': l14b05_1h\n",
    "    }\n",
    "\n",
    "l03b02_1h.head()\n",
    "l06b01_1h.head()\n",
    "\n",
    "for key, value in data_1h.items():\n",
    "    print(key, value.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c48db83d-1a6b-4c63-87e4-bfd441ab11df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to disaggregate heating and cooling using NMF\n",
    "def disaggregate_energy(file_path, weather_file_path, output_file_path):\n",
    "    # Load the building energy data and weather data\n",
    "    energy_data = pd.read_csv(file_path)\n",
    "    weather_data = pd.read_csv(weather_file_path)\n",
    "\n",
    "    # Merge datasets on timestamp\n",
    "    data = pd.merge(energy_data, weather_data, on='timestamp')\n",
    "\n",
    "    # Select relevant features for NMF (including weather data)\n",
    "    features = ['main_meter(kW)', 'air_temperature_at_2m(deg_C)', 'relative_humidity_at_2m(%)']\n",
    "\n",
    "    # Normalize the data to ensure non-negativity\n",
    "    scaler = MinMaxScaler()\n",
    "    X_normalized = scaler.fit_transform(data[features])\n",
    "\n",
    "    # Apply NMF to the combined dataset\n",
    "    nmf_model = NMF(n_components=2, init='random', random_state=42, max_iter=1000)\n",
    "    W = nmf_model.fit_transform(X_normalized)\n",
    "    H = nmf_model.components_\n",
    "\n",
    "    # Denormalize the results back to the original scale\n",
    "    W_denormalized = scaler.inverse_transform(W)\n",
    "\n",
    "    # Extract the heating and cooling loads (assuming the first component corresponds to heating)\n",
    "    heating_and_cooling_load = W_denormalized[:, 0]  # Assuming the first component is heating/cooling\n",
    "\n",
    "    # Prepare the output dataframe\n",
    "    output_data = pd.DataFrame({\n",
    "        'timestamp': energy_data['timestamp'],\n",
    "        'heating_and_cooling_load(kW)': heating_and_cooling_load\n",
    "    })\n",
    "\n",
    "    # Save the disaggregated data to a new CSV file\n",
    "    output_data.to_csv(output_file_path, index=False)\n",
    "\n",
    "# Directory paths for input and output\n",
    "input_dir = 'path_to_input_directory'  # Replace with the path to your input directory\n",
    "output_dir = 'path_to_output_directory'  # Replace with the path to your output directory\n",
    "weather_dir = 'path_to_weather_directory'  # Replace with the path to your weather data directory\n",
    "\n",
    "# Process each building's data\n",
    "for i in range(1, 10):  # Assuming the files are named in a sequence, e.g., building1.csv, building2.csv, ...\n",
    "    input_file = os.path.join(input_dir, f'building{i}.csv')\n",
    "    weather_file = os.path.join(weather_dir, f'weather_building{i}.csv')  # Assuming each building has its own weather file\n",
    "    output_file = os.path.join(output_dir, f'building{i}_heating_cooling.csv')\n",
    "\n",
    "    # Disaggregate and save the results\n",
    "    disaggregate_energy(input_file, weather_file, output_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "471546b7-7b59-4a63-b7be-348726bf6773",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>air_temperature_at_2m(deg_C)</th>\n",
       "      <th>relative_humidity_at_2m(%)</th>\n",
       "      <th>direct_solar_radiation(W/m^2)</th>\n",
       "      <th>diffuse_solar_radiation(W/m^2)</th>\n",
       "      <th>wind_speed_at_10m(km/h)</th>\n",
       "      <th>wind_direction_at_10m(deg)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-12-01 00:00:00+00:00</td>\n",
       "      <td>3.444376</td>\n",
       "      <td>97.016089</td>\n",
       "      <td>0.026658</td>\n",
       "      <td>-0.054859</td>\n",
       "      <td>15.924282</td>\n",
       "      <td>175.974536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-12-01 01:00:00+00:00</td>\n",
       "      <td>3.662187</td>\n",
       "      <td>96.964139</td>\n",
       "      <td>-0.022419</td>\n",
       "      <td>0.039787</td>\n",
       "      <td>13.008325</td>\n",
       "      <td>174.044967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-12-01 02:00:00+00:00</td>\n",
       "      <td>3.849796</td>\n",
       "      <td>97.042752</td>\n",
       "      <td>0.018184</td>\n",
       "      <td>-0.019089</td>\n",
       "      <td>9.631573</td>\n",
       "      <td>167.017767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-12-01 03:00:00+00:00</td>\n",
       "      <td>4.047407</td>\n",
       "      <td>97.044695</td>\n",
       "      <td>0.054901</td>\n",
       "      <td>-0.016765</td>\n",
       "      <td>6.825520</td>\n",
       "      <td>148.045236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-12-01 04:00:00+00:00</td>\n",
       "      <td>4.184971</td>\n",
       "      <td>97.035953</td>\n",
       "      <td>-0.022905</td>\n",
       "      <td>-0.012460</td>\n",
       "      <td>6.036958</td>\n",
       "      <td>114.996164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   timestamp  air_temperature_at_2m(deg_C)  \\\n",
       "0  2020-12-01 00:00:00+00:00                      3.444376   \n",
       "1  2020-12-01 01:00:00+00:00                      3.662187   \n",
       "2  2020-12-01 02:00:00+00:00                      3.849796   \n",
       "3  2020-12-01 03:00:00+00:00                      4.047407   \n",
       "4  2020-12-01 04:00:00+00:00                      4.184971   \n",
       "\n",
       "   relative_humidity_at_2m(%)  direct_solar_radiation(W/m^2)  \\\n",
       "0                   97.016089                       0.026658   \n",
       "1                   96.964139                      -0.022419   \n",
       "2                   97.042752                       0.018184   \n",
       "3                   97.044695                       0.054901   \n",
       "4                   97.035953                      -0.022905   \n",
       "\n",
       "   diffuse_solar_radiation(W/m^2)  wind_speed_at_10m(km/h)  \\\n",
       "0                       -0.054859                15.924282   \n",
       "1                        0.039787                13.008325   \n",
       "2                       -0.019089                 9.631573   \n",
       "3                       -0.016765                 6.825520   \n",
       "4                       -0.012460                 6.036958   \n",
       "\n",
       "   wind_direction_at_10m(deg)  \n",
       "0                  175.974536  \n",
       "1                  174.044967  \n",
       "2                  167.017767  \n",
       "3                  148.045236  \n",
       "4                  114.996164  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l03_weather = pd.read_csv('train_public/weather/L03_weather_train.csv')\n",
    "l06_weather = pd.read_csv('train_public/weather/L06_weather_train.csv')\n",
    "l09_weather = pd.read_csv('train_public/weather/L09_weather_train.csv')\n",
    "l10_weather = pd.read_csv('train_public/weather/L10_weather_train.csv')\n",
    "l14_weather = pd.read_csv('train_public/weather/L14_weather_train.csv')\n",
    "\n",
    "data_weather = {'l03': l03_weather,\n",
    "                'l06': l06_weather,\n",
    "                'l09': l09_weather,\n",
    "                'l10': l10_weather,\n",
    "                'l14': l14_weather\n",
    "}\n",
    "\n",
    "l03_weather.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f6b01569-bbdd-4c40-ad73-36ed8043d077",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>air_temperature_at_2m(deg_C)</th>\n",
       "      <th>relative_humidity_at_2m(%)</th>\n",
       "      <th>direct_solar_radiation(W/m^2)</th>\n",
       "      <th>diffuse_solar_radiation(W/m^2)</th>\n",
       "      <th>wind_speed_at_10m(km/h)</th>\n",
       "      <th>wind_direction_at_10m(deg)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8395</th>\n",
       "      <td>2021-11-15 19:00:00+00:00</td>\n",
       "      <td>6.910766</td>\n",
       "      <td>81.985465</td>\n",
       "      <td>-0.036131</td>\n",
       "      <td>0.038543</td>\n",
       "      <td>6.930305</td>\n",
       "      <td>137.044424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>2021-11-15 20:00:00+00:00</td>\n",
       "      <td>6.753975</td>\n",
       "      <td>81.976869</td>\n",
       "      <td>0.015393</td>\n",
       "      <td>0.038366</td>\n",
       "      <td>7.456723</td>\n",
       "      <td>144.981432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8397</th>\n",
       "      <td>2021-11-15 21:00:00+00:00</td>\n",
       "      <td>6.582176</td>\n",
       "      <td>82.985219</td>\n",
       "      <td>0.014573</td>\n",
       "      <td>0.001191</td>\n",
       "      <td>7.893027</td>\n",
       "      <td>149.998625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8398</th>\n",
       "      <td>2021-11-15 22:00:00+00:00</td>\n",
       "      <td>6.631711</td>\n",
       "      <td>83.958208</td>\n",
       "      <td>-0.020878</td>\n",
       "      <td>-0.013877</td>\n",
       "      <td>6.360064</td>\n",
       "      <td>142.022950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8399</th>\n",
       "      <td>2021-11-15 23:00:00+00:00</td>\n",
       "      <td>6.427195</td>\n",
       "      <td>84.972944</td>\n",
       "      <td>0.038590</td>\n",
       "      <td>-0.000835</td>\n",
       "      <td>6.216857</td>\n",
       "      <td>143.977515</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      timestamp  air_temperature_at_2m(deg_C)  \\\n",
       "8395  2021-11-15 19:00:00+00:00                      6.910766   \n",
       "8396  2021-11-15 20:00:00+00:00                      6.753975   \n",
       "8397  2021-11-15 21:00:00+00:00                      6.582176   \n",
       "8398  2021-11-15 22:00:00+00:00                      6.631711   \n",
       "8399  2021-11-15 23:00:00+00:00                      6.427195   \n",
       "\n",
       "      relative_humidity_at_2m(%)  direct_solar_radiation(W/m^2)  \\\n",
       "8395                   81.985465                      -0.036131   \n",
       "8396                   81.976869                       0.015393   \n",
       "8397                   82.985219                       0.014573   \n",
       "8398                   83.958208                      -0.020878   \n",
       "8399                   84.972944                       0.038590   \n",
       "\n",
       "      diffuse_solar_radiation(W/m^2)  wind_speed_at_10m(km/h)  \\\n",
       "8395                        0.038543                 6.930305   \n",
       "8396                        0.038366                 7.456723   \n",
       "8397                        0.001191                 7.893027   \n",
       "8398                       -0.013877                 6.360064   \n",
       "8399                       -0.000835                 6.216857   \n",
       "\n",
       "      wind_direction_at_10m(deg)  \n",
       "8395                  137.044424  \n",
       "8396                  144.981432  \n",
       "8397                  149.998625  \n",
       "8398                  142.022950  \n",
       "8399                  143.977515  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l03_weather.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ccc8d912-80a1-4a28-803c-2df68de5e1a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l03 (8400, 7)\n",
      "l06 (7704, 7)\n",
      "l09 (8760, 7)\n",
      "l10 (13872, 7)\n",
      "l14 (8760, 7)\n"
     ]
    }
   ],
   "source": [
    "for key, value in data_weather.items():\n",
    "    print(key, value.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "2d6c97d4-3919-484e-8660-f96e46c802ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l03b02_1h (8400, 2)\n",
      "l06b01_1h (7674, 3)\n",
      "l09b01_1h (8287, 3)\n",
      "l10b01_1h (13867, 2)\n",
      "l14b01_1h (8760, 2)\n",
      "l14b02_1h (8760, 2)\n",
      "l14b03_1h (8760, 2)\n",
      "l14b04_1h (8760, 2)\n",
      "l14b05_1h (8760, 2)\n"
     ]
    }
   ],
   "source": [
    "for key, value in data_1h.items():\n",
    "    print(key, value.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9049584f-9cfe-4edf-abe0-267ee4e9e299",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned data for l03 saved to: train_public/weather/L03_weather_train_l03_cleaned.csv\n",
      "Cleaned data for l06 saved to: train_public/weather/L06_weather_train_l06_cleaned.csv\n",
      "Cleaned data for l09 saved to: train_public/weather/L09_weather_train_l09_cleaned.csv\n",
      "Cleaned data for l10 saved to: train_public/weather/L10_weather_train_l10_cleaned.csv\n",
      "Cleaned data for l14 saved to: train_public/weather/L14_weather_train_l14_cleaned.csv\n"
     ]
    }
   ],
   "source": [
    "# Function to clean weather data based on location\n",
    "def clean_weather_data_by_location(file_path, location):\n",
    "    # Load the weather data\n",
    "    weather_data = pd.read_csv(file_path)\n",
    "\n",
    "    # Location-specific temperature ranges\n",
    "    temp_ranges = {\n",
    "        'l03': (-10, 35),  # Denmark\n",
    "        'l06': (-30, 30),  # Norway\n",
    "        'l09': (-5, 45),   # Australia\n",
    "        'l10': (-30, 30),  # Norway\n",
    "        'l14': (-30, 30)   # Norway\n",
    "    }\n",
    "\n",
    "    # Get the appropriate temperature range for the location\n",
    "    temp_min, temp_max = temp_ranges[location]\n",
    "\n",
    "    # Clean the temperature data\n",
    "    weather_data['air_temperature_at_2m(deg_C)'] = weather_data['air_temperature_at_2m(deg_C)'].clip(temp_min, temp_max)\n",
    "\n",
    "    # Set negative values in solar radiation columns to 0\n",
    "    weather_data['direct_solar_radiation(W/m^2)'] = weather_data['direct_solar_radiation(W/m^2)'].clip(lower=0)\n",
    "    weather_data['diffuse_solar_radiation(W/m^2)'] = weather_data['diffuse_solar_radiation(W/m^2)'].clip(lower=0)\n",
    "\n",
    "    # Clip humidity between 0% and 100%\n",
    "    weather_data['relative_humidity_at_2m(%)'] = weather_data['relative_humidity_at_2m(%)'].clip(0, 100)\n",
    "\n",
    "    # Set negative wind speeds to 0\n",
    "    weather_data['wind_speed_at_10m(km/h)'] = weather_data['wind_speed_at_10m(km/h)'].clip(lower=0)\n",
    "\n",
    "    # Optionally, you could also set a maximum wind speed threshold\n",
    "    weather_data['wind_speed_at_10m(km/h)'] = weather_data['wind_speed_at_10m(km/h)'].clip(0, 150)\n",
    "\n",
    "    # Save the cleaned data back to the CSV file (or a new file)\n",
    "    cleaned_file_path = file_path.replace('.csv', f'_{location}_cleaned.csv')\n",
    "    weather_data.to_csv(cleaned_file_path, index=False)\n",
    "\n",
    "    return cleaned_file_path\n",
    "\n",
    "# Example of applying the cleaning function to all weather datasets\n",
    "weather_files = {\n",
    "    'l03': 'train_public/weather/L03_weather_train.csv',\n",
    "    'l06': 'train_public/weather/L06_weather_train.csv',\n",
    "    'l09': 'train_public/weather/L09_weather_train.csv',\n",
    "    'l10': 'train_public/weather/L10_weather_train.csv',\n",
    "    'l14': 'train_public/weather/L14_weather_train.csv'\n",
    "}\n",
    "\n",
    "for location, weather_file in weather_files.items():\n",
    "    cleaned_file = clean_weather_data_by_location(weather_file, location)\n",
    "    print(f\"Cleaned data for {location} saved to: {cleaned_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d8382bb6-2089-4bbc-a2fd-b6225af48595",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "236\n"
     ]
    }
   ],
   "source": [
    "j = 0\n",
    "for i in l06b01_1h.iloc[:, 1]:\n",
    "    if i < 0:\n",
    "        j = j + 1\n",
    "print(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "3b088157-573b-45bf-9890-d7b50aa9e4fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>main_meter(kW)</th>\n",
       "      <th>PV_battery_system(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3100</th>\n",
       "      <td>2021-08-08 04:00:00+00:00</td>\n",
       "      <td>11.628814</td>\n",
       "      <td>0.805000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3101</th>\n",
       "      <td>2021-08-08 05:00:00+00:00</td>\n",
       "      <td>8.690321</td>\n",
       "      <td>4.924546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3102</th>\n",
       "      <td>2021-08-08 06:00:00+00:00</td>\n",
       "      <td>-1.262319</td>\n",
       "      <td>14.790002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3103</th>\n",
       "      <td>2021-08-08 07:00:00+00:00</td>\n",
       "      <td>-10.246774</td>\n",
       "      <td>23.699652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3104</th>\n",
       "      <td>2021-08-08 08:00:00+00:00</td>\n",
       "      <td>-20.815252</td>\n",
       "      <td>33.356487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105</th>\n",
       "      <td>2021-08-08 09:00:00+00:00</td>\n",
       "      <td>-15.234246</td>\n",
       "      <td>32.414921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3106</th>\n",
       "      <td>2021-08-08 10:00:00+00:00</td>\n",
       "      <td>-22.268257</td>\n",
       "      <td>37.618954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3107</th>\n",
       "      <td>2021-08-08 11:00:00+00:00</td>\n",
       "      <td>-17.697014</td>\n",
       "      <td>32.018768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3108</th>\n",
       "      <td>2021-08-08 12:00:00+00:00</td>\n",
       "      <td>-10.589552</td>\n",
       "      <td>25.286966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3109</th>\n",
       "      <td>2021-08-08 13:00:00+00:00</td>\n",
       "      <td>-8.533872</td>\n",
       "      <td>22.153507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3110</th>\n",
       "      <td>2021-08-08 14:00:00+00:00</td>\n",
       "      <td>-10.858211</td>\n",
       "      <td>24.438427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3111</th>\n",
       "      <td>2021-08-08 15:00:00+00:00</td>\n",
       "      <td>-9.828356</td>\n",
       "      <td>23.968946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3112</th>\n",
       "      <td>2021-08-08 16:00:00+00:00</td>\n",
       "      <td>6.759090</td>\n",
       "      <td>6.170178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3113</th>\n",
       "      <td>2021-08-08 17:00:00+00:00</td>\n",
       "      <td>5.354546</td>\n",
       "      <td>5.889473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3114</th>\n",
       "      <td>2021-08-08 18:00:00+00:00</td>\n",
       "      <td>11.998304</td>\n",
       "      <td>1.365143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3115</th>\n",
       "      <td>2021-08-08 19:00:00+00:00</td>\n",
       "      <td>18.047459</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3116</th>\n",
       "      <td>2021-08-08 20:00:00+00:00</td>\n",
       "      <td>21.044832</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3117</th>\n",
       "      <td>2021-08-08 21:00:00+00:00</td>\n",
       "      <td>21.171665</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3118</th>\n",
       "      <td>2021-08-08 22:00:00+00:00</td>\n",
       "      <td>20.337284</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3119</th>\n",
       "      <td>2021-08-08 23:00:00+00:00</td>\n",
       "      <td>22.891668</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3120</th>\n",
       "      <td>2021-08-09 00:00:00+00:00</td>\n",
       "      <td>20.427118</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3121</th>\n",
       "      <td>2021-08-09 01:00:00+00:00</td>\n",
       "      <td>19.039997</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3122</th>\n",
       "      <td>2021-08-09 02:00:00+00:00</td>\n",
       "      <td>21.277967</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3123</th>\n",
       "      <td>2021-08-09 03:00:00+00:00</td>\n",
       "      <td>21.674995</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3124</th>\n",
       "      <td>2021-08-09 04:00:00+00:00</td>\n",
       "      <td>83.435585</td>\n",
       "      <td>0.488947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3125</th>\n",
       "      <td>2021-08-09 05:00:00+00:00</td>\n",
       "      <td>96.971664</td>\n",
       "      <td>1.627407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3126</th>\n",
       "      <td>2021-08-09 06:00:00+00:00</td>\n",
       "      <td>87.340004</td>\n",
       "      <td>11.505895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3127</th>\n",
       "      <td>2021-08-09 07:00:00+00:00</td>\n",
       "      <td>77.604996</td>\n",
       "      <td>23.388247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3128</th>\n",
       "      <td>2021-08-09 08:00:00+00:00</td>\n",
       "      <td>76.218658</td>\n",
       "      <td>20.163506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3129</th>\n",
       "      <td>2021-08-09 09:00:00+00:00</td>\n",
       "      <td>83.481667</td>\n",
       "      <td>15.757545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3130</th>\n",
       "      <td>2021-08-09 10:00:00+00:00</td>\n",
       "      <td>74.648331</td>\n",
       "      <td>23.474913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3131</th>\n",
       "      <td>2021-08-09 11:00:00+00:00</td>\n",
       "      <td>78.389839</td>\n",
       "      <td>19.967192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3132</th>\n",
       "      <td>2021-08-09 12:00:00+00:00</td>\n",
       "      <td>62.268337</td>\n",
       "      <td>37.961224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3133</th>\n",
       "      <td>2021-08-09 13:00:00+00:00</td>\n",
       "      <td>68.743332</td>\n",
       "      <td>27.601574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3134</th>\n",
       "      <td>2021-08-09 14:00:00+00:00</td>\n",
       "      <td>73.708328</td>\n",
       "      <td>22.550537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3135</th>\n",
       "      <td>2021-08-09 15:00:00+00:00</td>\n",
       "      <td>80.922050</td>\n",
       "      <td>15.729647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3136</th>\n",
       "      <td>2021-08-09 16:00:00+00:00</td>\n",
       "      <td>85.331665</td>\n",
       "      <td>9.037019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3137</th>\n",
       "      <td>2021-08-09 17:00:00+00:00</td>\n",
       "      <td>90.678345</td>\n",
       "      <td>4.887544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3138</th>\n",
       "      <td>2021-08-09 18:00:00+00:00</td>\n",
       "      <td>99.184990</td>\n",
       "      <td>1.502745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3139</th>\n",
       "      <td>2021-08-09 19:00:00+00:00</td>\n",
       "      <td>35.718636</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3140</th>\n",
       "      <td>2021-08-09 20:00:00+00:00</td>\n",
       "      <td>28.623728</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3141</th>\n",
       "      <td>2021-08-09 21:00:00+00:00</td>\n",
       "      <td>25.941668</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3142</th>\n",
       "      <td>2021-08-09 22:00:00+00:00</td>\n",
       "      <td>25.088137</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3143</th>\n",
       "      <td>2021-08-09 23:00:00+00:00</td>\n",
       "      <td>22.948334</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3144</th>\n",
       "      <td>2021-08-10 00:00:00+00:00</td>\n",
       "      <td>22.167801</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3145</th>\n",
       "      <td>2021-08-10 01:00:00+00:00</td>\n",
       "      <td>20.994919</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3146</th>\n",
       "      <td>2021-08-10 02:00:00+00:00</td>\n",
       "      <td>21.071667</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3147</th>\n",
       "      <td>2021-08-10 03:00:00+00:00</td>\n",
       "      <td>17.677969</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3148</th>\n",
       "      <td>2021-08-10 04:00:00+00:00</td>\n",
       "      <td>59.040001</td>\n",
       "      <td>1.591861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3149</th>\n",
       "      <td>2021-08-10 05:00:00+00:00</td>\n",
       "      <td>87.138985</td>\n",
       "      <td>6.727544</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      timestamp  main_meter(kW)  PV_battery_system(kW)\n",
       "3100  2021-08-08 04:00:00+00:00       11.628814               0.805000\n",
       "3101  2021-08-08 05:00:00+00:00        8.690321               4.924546\n",
       "3102  2021-08-08 06:00:00+00:00       -1.262319              14.790002\n",
       "3103  2021-08-08 07:00:00+00:00      -10.246774              23.699652\n",
       "3104  2021-08-08 08:00:00+00:00      -20.815252              33.356487\n",
       "3105  2021-08-08 09:00:00+00:00      -15.234246              32.414921\n",
       "3106  2021-08-08 10:00:00+00:00      -22.268257              37.618954\n",
       "3107  2021-08-08 11:00:00+00:00      -17.697014              32.018768\n",
       "3108  2021-08-08 12:00:00+00:00      -10.589552              25.286966\n",
       "3109  2021-08-08 13:00:00+00:00       -8.533872              22.153507\n",
       "3110  2021-08-08 14:00:00+00:00      -10.858211              24.438427\n",
       "3111  2021-08-08 15:00:00+00:00       -9.828356              23.968946\n",
       "3112  2021-08-08 16:00:00+00:00        6.759090               6.170178\n",
       "3113  2021-08-08 17:00:00+00:00        5.354546               5.889473\n",
       "3114  2021-08-08 18:00:00+00:00       11.998304               1.365143\n",
       "3115  2021-08-08 19:00:00+00:00       18.047459                    NaN\n",
       "3116  2021-08-08 20:00:00+00:00       21.044832                    NaN\n",
       "3117  2021-08-08 21:00:00+00:00       21.171665                    NaN\n",
       "3118  2021-08-08 22:00:00+00:00       20.337284                    NaN\n",
       "3119  2021-08-08 23:00:00+00:00       22.891668                    NaN\n",
       "3120  2021-08-09 00:00:00+00:00       20.427118                    NaN\n",
       "3121  2021-08-09 01:00:00+00:00       19.039997               0.000000\n",
       "3122  2021-08-09 02:00:00+00:00       21.277967                    NaN\n",
       "3123  2021-08-09 03:00:00+00:00       21.674995                    NaN\n",
       "3124  2021-08-09 04:00:00+00:00       83.435585               0.488947\n",
       "3125  2021-08-09 05:00:00+00:00       96.971664               1.627407\n",
       "3126  2021-08-09 06:00:00+00:00       87.340004              11.505895\n",
       "3127  2021-08-09 07:00:00+00:00       77.604996              23.388247\n",
       "3128  2021-08-09 08:00:00+00:00       76.218658              20.163506\n",
       "3129  2021-08-09 09:00:00+00:00       83.481667              15.757545\n",
       "3130  2021-08-09 10:00:00+00:00       74.648331              23.474913\n",
       "3131  2021-08-09 11:00:00+00:00       78.389839              19.967192\n",
       "3132  2021-08-09 12:00:00+00:00       62.268337              37.961224\n",
       "3133  2021-08-09 13:00:00+00:00       68.743332              27.601574\n",
       "3134  2021-08-09 14:00:00+00:00       73.708328              22.550537\n",
       "3135  2021-08-09 15:00:00+00:00       80.922050              15.729647\n",
       "3136  2021-08-09 16:00:00+00:00       85.331665               9.037019\n",
       "3137  2021-08-09 17:00:00+00:00       90.678345               4.887544\n",
       "3138  2021-08-09 18:00:00+00:00       99.184990               1.502745\n",
       "3139  2021-08-09 19:00:00+00:00       35.718636                    NaN\n",
       "3140  2021-08-09 20:00:00+00:00       28.623728                    NaN\n",
       "3141  2021-08-09 21:00:00+00:00       25.941668                    NaN\n",
       "3142  2021-08-09 22:00:00+00:00       25.088137                    NaN\n",
       "3143  2021-08-09 23:00:00+00:00       22.948334                    NaN\n",
       "3144  2021-08-10 00:00:00+00:00       22.167801                    NaN\n",
       "3145  2021-08-10 01:00:00+00:00       20.994919                    NaN\n",
       "3146  2021-08-10 02:00:00+00:00       21.071667                    NaN\n",
       "3147  2021-08-10 03:00:00+00:00       17.677969                    NaN\n",
       "3148  2021-08-10 04:00:00+00:00       59.040001               1.591861\n",
       "3149  2021-08-10 05:00:00+00:00       87.138985               6.727544"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l06b01_1h[3100:3150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "221c954d-39a1-4662-bb11-6d768e97385d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>main_meter(kW)</th>\n",
       "      <th>PV_battery_system(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8237</th>\n",
       "      <td>2021-12-29 22:00:00+00:00</td>\n",
       "      <td>-110.583333</td>\n",
       "      <td>188.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8238</th>\n",
       "      <td>2021-12-29 23:00:00+00:00</td>\n",
       "      <td>-97.583333</td>\n",
       "      <td>161.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8239</th>\n",
       "      <td>2021-12-30 00:00:00+00:00</td>\n",
       "      <td>-167.333333</td>\n",
       "      <td>240.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8240</th>\n",
       "      <td>2021-12-30 01:00:00+00:00</td>\n",
       "      <td>-218.692308</td>\n",
       "      <td>295.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8241</th>\n",
       "      <td>2021-12-30 02:00:00+00:00</td>\n",
       "      <td>-216.083333</td>\n",
       "      <td>293.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8242</th>\n",
       "      <td>2021-12-30 03:00:00+00:00</td>\n",
       "      <td>-205.166667</td>\n",
       "      <td>281.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8243</th>\n",
       "      <td>2021-12-30 04:00:00+00:00</td>\n",
       "      <td>-178.083333</td>\n",
       "      <td>255.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8244</th>\n",
       "      <td>2021-12-30 05:00:00+00:00</td>\n",
       "      <td>-132.833333</td>\n",
       "      <td>211.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8245</th>\n",
       "      <td>2021-12-30 06:00:00+00:00</td>\n",
       "      <td>-78.750000</td>\n",
       "      <td>152.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8246</th>\n",
       "      <td>2021-12-30 07:00:00+00:00</td>\n",
       "      <td>-28.000000</td>\n",
       "      <td>101.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8247</th>\n",
       "      <td>2021-12-30 08:00:00+00:00</td>\n",
       "      <td>-6.166667</td>\n",
       "      <td>83.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8248</th>\n",
       "      <td>2021-12-30 09:00:00+00:00</td>\n",
       "      <td>75.250000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8249</th>\n",
       "      <td>2021-12-30 10:00:00+00:00</td>\n",
       "      <td>59.750000</td>\n",
       "      <td>18.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8250</th>\n",
       "      <td>2021-12-30 11:00:00+00:00</td>\n",
       "      <td>145.833333</td>\n",
       "      <td>-69.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8251</th>\n",
       "      <td>2021-12-30 12:00:00+00:00</td>\n",
       "      <td>153.250000</td>\n",
       "      <td>-75.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8252</th>\n",
       "      <td>2021-12-30 13:00:00+00:00</td>\n",
       "      <td>102.833333</td>\n",
       "      <td>-30.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8253</th>\n",
       "      <td>2021-12-30 14:00:00+00:00</td>\n",
       "      <td>78.333333</td>\n",
       "      <td>-1.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8254</th>\n",
       "      <td>2021-12-30 15:00:00+00:00</td>\n",
       "      <td>77.916667</td>\n",
       "      <td>-1.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8255</th>\n",
       "      <td>2021-12-30 16:00:00+00:00</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>-1.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8256</th>\n",
       "      <td>2021-12-30 17:00:00+00:00</td>\n",
       "      <td>77.750000</td>\n",
       "      <td>-1.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8257</th>\n",
       "      <td>2021-12-30 18:00:00+00:00</td>\n",
       "      <td>77.833333</td>\n",
       "      <td>-1.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8258</th>\n",
       "      <td>2021-12-30 19:00:00+00:00</td>\n",
       "      <td>66.250000</td>\n",
       "      <td>7.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8259</th>\n",
       "      <td>2021-12-30 20:00:00+00:00</td>\n",
       "      <td>29.083333</td>\n",
       "      <td>44.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8260</th>\n",
       "      <td>2021-12-30 21:00:00+00:00</td>\n",
       "      <td>-41.500000</td>\n",
       "      <td>117.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8261</th>\n",
       "      <td>2021-12-30 22:00:00+00:00</td>\n",
       "      <td>-118.833333</td>\n",
       "      <td>193.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8262</th>\n",
       "      <td>2021-12-30 23:00:00+00:00</td>\n",
       "      <td>-166.083333</td>\n",
       "      <td>242.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8263</th>\n",
       "      <td>2021-12-31 00:00:00+00:00</td>\n",
       "      <td>-160.750000</td>\n",
       "      <td>238.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8264</th>\n",
       "      <td>2021-12-31 01:00:00+00:00</td>\n",
       "      <td>-137.461538</td>\n",
       "      <td>198.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8265</th>\n",
       "      <td>2021-12-31 02:00:00+00:00</td>\n",
       "      <td>-185.583333</td>\n",
       "      <td>279.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8266</th>\n",
       "      <td>2021-12-31 03:00:00+00:00</td>\n",
       "      <td>-187.250000</td>\n",
       "      <td>234.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8267</th>\n",
       "      <td>2021-12-31 04:00:00+00:00</td>\n",
       "      <td>-176.166667</td>\n",
       "      <td>256.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8268</th>\n",
       "      <td>2021-12-31 05:00:00+00:00</td>\n",
       "      <td>-129.750000</td>\n",
       "      <td>209.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8269</th>\n",
       "      <td>2021-12-31 06:00:00+00:00</td>\n",
       "      <td>-78.833333</td>\n",
       "      <td>162.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8270</th>\n",
       "      <td>2021-12-31 07:00:00+00:00</td>\n",
       "      <td>16.833333</td>\n",
       "      <td>63.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8271</th>\n",
       "      <td>2021-12-31 08:00:00+00:00</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>75.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8272</th>\n",
       "      <td>2021-12-31 09:00:00+00:00</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>13.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8273</th>\n",
       "      <td>2021-12-31 10:00:00+00:00</td>\n",
       "      <td>27.500000</td>\n",
       "      <td>50.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8274</th>\n",
       "      <td>2021-12-31 11:00:00+00:00</td>\n",
       "      <td>90.166667</td>\n",
       "      <td>-11.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8275</th>\n",
       "      <td>2021-12-31 12:00:00+00:00</td>\n",
       "      <td>82.500000</td>\n",
       "      <td>-2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8276</th>\n",
       "      <td>2021-12-31 13:00:00+00:00</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8277</th>\n",
       "      <td>2021-12-31 14:00:00+00:00</td>\n",
       "      <td>80.083333</td>\n",
       "      <td>-1.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8278</th>\n",
       "      <td>2021-12-31 15:00:00+00:00</td>\n",
       "      <td>79.333333</td>\n",
       "      <td>-2.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8279</th>\n",
       "      <td>2021-12-31 16:00:00+00:00</td>\n",
       "      <td>81.083333</td>\n",
       "      <td>-1.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8280</th>\n",
       "      <td>2021-12-31 17:00:00+00:00</td>\n",
       "      <td>79.916667</td>\n",
       "      <td>-1.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8281</th>\n",
       "      <td>2021-12-31 18:00:00+00:00</td>\n",
       "      <td>80.166667</td>\n",
       "      <td>-1.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8282</th>\n",
       "      <td>2021-12-31 19:00:00+00:00</td>\n",
       "      <td>62.250000</td>\n",
       "      <td>12.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8283</th>\n",
       "      <td>2021-12-31 20:00:00+00:00</td>\n",
       "      <td>15.833333</td>\n",
       "      <td>60.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8284</th>\n",
       "      <td>2021-12-31 21:00:00+00:00</td>\n",
       "      <td>-21.833333</td>\n",
       "      <td>97.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8285</th>\n",
       "      <td>2021-12-31 22:00:00+00:00</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>82.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8286</th>\n",
       "      <td>2021-12-31 23:00:00+00:00</td>\n",
       "      <td>-32.333333</td>\n",
       "      <td>110.166667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      timestamp  main_meter(kW)  PV_battery_system(kW)\n",
       "8237  2021-12-29 22:00:00+00:00     -110.583333             188.416667\n",
       "8238  2021-12-29 23:00:00+00:00      -97.583333             161.250000\n",
       "8239  2021-12-30 00:00:00+00:00     -167.333333             240.250000\n",
       "8240  2021-12-30 01:00:00+00:00     -218.692308             295.846154\n",
       "8241  2021-12-30 02:00:00+00:00     -216.083333             293.833333\n",
       "8242  2021-12-30 03:00:00+00:00     -205.166667             281.833333\n",
       "8243  2021-12-30 04:00:00+00:00     -178.083333             255.916667\n",
       "8244  2021-12-30 05:00:00+00:00     -132.833333             211.000000\n",
       "8245  2021-12-30 06:00:00+00:00      -78.750000             152.583333\n",
       "8246  2021-12-30 07:00:00+00:00      -28.000000             101.666667\n",
       "8247  2021-12-30 08:00:00+00:00       -6.166667              83.666667\n",
       "8248  2021-12-30 09:00:00+00:00       75.250000               4.000000\n",
       "8249  2021-12-30 10:00:00+00:00       59.750000              18.583333\n",
       "8250  2021-12-30 11:00:00+00:00      145.833333             -69.083333\n",
       "8251  2021-12-30 12:00:00+00:00      153.250000             -75.416667\n",
       "8252  2021-12-30 13:00:00+00:00      102.833333             -30.666667\n",
       "8253  2021-12-30 14:00:00+00:00       78.333333              -1.583333\n",
       "8254  2021-12-30 15:00:00+00:00       77.916667              -1.583333\n",
       "8255  2021-12-30 16:00:00+00:00       77.000000              -1.416667\n",
       "8256  2021-12-30 17:00:00+00:00       77.750000              -1.666667\n",
       "8257  2021-12-30 18:00:00+00:00       77.833333              -1.250000\n",
       "8258  2021-12-30 19:00:00+00:00       66.250000               7.666667\n",
       "8259  2021-12-30 20:00:00+00:00       29.083333              44.750000\n",
       "8260  2021-12-30 21:00:00+00:00      -41.500000             117.916667\n",
       "8261  2021-12-30 22:00:00+00:00     -118.833333             193.083333\n",
       "8262  2021-12-30 23:00:00+00:00     -166.083333             242.333333\n",
       "8263  2021-12-31 00:00:00+00:00     -160.750000             238.166667\n",
       "8264  2021-12-31 01:00:00+00:00     -137.461538             198.000000\n",
       "8265  2021-12-31 02:00:00+00:00     -185.583333             279.166667\n",
       "8266  2021-12-31 03:00:00+00:00     -187.250000             234.333333\n",
       "8267  2021-12-31 04:00:00+00:00     -176.166667             256.166667\n",
       "8268  2021-12-31 05:00:00+00:00     -129.750000             209.250000\n",
       "8269  2021-12-31 06:00:00+00:00      -78.833333             162.083333\n",
       "8270  2021-12-31 07:00:00+00:00       16.833333              63.166667\n",
       "8271  2021-12-31 08:00:00+00:00        4.000000              75.666667\n",
       "8272  2021-12-31 09:00:00+00:00       67.000000              13.250000\n",
       "8273  2021-12-31 10:00:00+00:00       27.500000              50.750000\n",
       "8274  2021-12-31 11:00:00+00:00       90.166667             -11.583333\n",
       "8275  2021-12-31 12:00:00+00:00       82.500000              -2.000000\n",
       "8276  2021-12-31 13:00:00+00:00       81.000000              -2.000000\n",
       "8277  2021-12-31 14:00:00+00:00       80.083333              -1.916667\n",
       "8278  2021-12-31 15:00:00+00:00       79.333333              -2.083333\n",
       "8279  2021-12-31 16:00:00+00:00       81.083333              -1.333333\n",
       "8280  2021-12-31 17:00:00+00:00       79.916667              -1.333333\n",
       "8281  2021-12-31 18:00:00+00:00       80.166667              -1.333333\n",
       "8282  2021-12-31 19:00:00+00:00       62.250000              12.083333\n",
       "8283  2021-12-31 20:00:00+00:00       15.833333              60.666667\n",
       "8284  2021-12-31 21:00:00+00:00      -21.833333              97.666667\n",
       "8285  2021-12-31 22:00:00+00:00       -2.000000              82.666667\n",
       "8286  2021-12-31 23:00:00+00:00      -32.333333             110.166667"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l09b01_1h.tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "7dd5d8a5-1a1f-42cf-b72c-d9b520af6625",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3375\n"
     ]
    }
   ],
   "source": [
    "j = 0\n",
    "for i in l09b01_1h.iloc[:, 2]:\n",
    "    if i < 0:\n",
    "        j = j + 1\n",
    "print(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "c25def65-77c9-449f-8a11-999e82be5fe7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "k = 0\n",
    "for i, j in zip(l09b01_1h.iloc[:, 1], l09b01_1h.iloc[:, 2]):\n",
    "    if i < 0 and j < 0:\n",
    "        k = k + 1\n",
    "print(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "ba242bb3-d14c-4e87-9922-ecbc7ea6c7c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>air_temperature_at_2m(deg_C)</th>\n",
       "      <th>relative_humidity_at_2m(%)</th>\n",
       "      <th>direct_solar_radiation(W/m^2)</th>\n",
       "      <th>diffuse_solar_radiation(W/m^2)</th>\n",
       "      <th>wind_speed_at_10m(km/h)</th>\n",
       "      <th>wind_direction_at_10m(deg)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-12-01 00:00:00+00:00</td>\n",
       "      <td>3.444376</td>\n",
       "      <td>97.016089</td>\n",
       "      <td>0.026658</td>\n",
       "      <td>-0.054859</td>\n",
       "      <td>15.924282</td>\n",
       "      <td>175.974536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-12-01 01:00:00+00:00</td>\n",
       "      <td>3.662187</td>\n",
       "      <td>96.964139</td>\n",
       "      <td>-0.022419</td>\n",
       "      <td>0.039787</td>\n",
       "      <td>13.008325</td>\n",
       "      <td>174.044967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-12-01 02:00:00+00:00</td>\n",
       "      <td>3.849796</td>\n",
       "      <td>97.042752</td>\n",
       "      <td>0.018184</td>\n",
       "      <td>-0.019089</td>\n",
       "      <td>9.631573</td>\n",
       "      <td>167.017767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-12-01 03:00:00+00:00</td>\n",
       "      <td>4.047407</td>\n",
       "      <td>97.044695</td>\n",
       "      <td>0.054901</td>\n",
       "      <td>-0.016765</td>\n",
       "      <td>6.825520</td>\n",
       "      <td>148.045236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-12-01 04:00:00+00:00</td>\n",
       "      <td>4.184971</td>\n",
       "      <td>97.035953</td>\n",
       "      <td>-0.022905</td>\n",
       "      <td>-0.012460</td>\n",
       "      <td>6.036958</td>\n",
       "      <td>114.996164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   timestamp  air_temperature_at_2m(deg_C)  \\\n",
       "0  2020-12-01 00:00:00+00:00                      3.444376   \n",
       "1  2020-12-01 01:00:00+00:00                      3.662187   \n",
       "2  2020-12-01 02:00:00+00:00                      3.849796   \n",
       "3  2020-12-01 03:00:00+00:00                      4.047407   \n",
       "4  2020-12-01 04:00:00+00:00                      4.184971   \n",
       "\n",
       "   relative_humidity_at_2m(%)  direct_solar_radiation(W/m^2)  \\\n",
       "0                   97.016089                       0.026658   \n",
       "1                   96.964139                      -0.022419   \n",
       "2                   97.042752                       0.018184   \n",
       "3                   97.044695                       0.054901   \n",
       "4                   97.035953                      -0.022905   \n",
       "\n",
       "   diffuse_solar_radiation(W/m^2)  wind_speed_at_10m(km/h)  \\\n",
       "0                       -0.054859                15.924282   \n",
       "1                        0.039787                13.008325   \n",
       "2                       -0.019089                 9.631573   \n",
       "3                       -0.016765                 6.825520   \n",
       "4                       -0.012460                 6.036958   \n",
       "\n",
       "   wind_direction_at_10m(deg)  \n",
       "0                  175.974536  \n",
       "1                  174.044967  \n",
       "2                  167.017767  \n",
       "3                  148.045236  \n",
       "4                  114.996164  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l03_weather.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "01f41c87-5d35-424a-bcbc-2ad3a856c02f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>main_meter(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-12-01 00:00:00+00:00</td>\n",
       "      <td>69.778595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-12-01 01:00:00+00:00</td>\n",
       "      <td>52.449440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-12-01 02:00:00+00:00</td>\n",
       "      <td>72.574860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-12-01 03:00:00+00:00</td>\n",
       "      <td>67.813957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-12-01 04:00:00+00:00</td>\n",
       "      <td>65.743805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8395</th>\n",
       "      <td>2021-11-15 19:00:00+00:00</td>\n",
       "      <td>148.864655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>2021-11-15 20:00:00+00:00</td>\n",
       "      <td>151.322693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8397</th>\n",
       "      <td>2021-11-15 21:00:00+00:00</td>\n",
       "      <td>120.674629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8398</th>\n",
       "      <td>2021-11-15 22:00:00+00:00</td>\n",
       "      <td>113.046677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8399</th>\n",
       "      <td>2021-11-15 23:00:00+00:00</td>\n",
       "      <td>63.877048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8400 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      timestamp  main_meter(kW)\n",
       "0     2020-12-01 00:00:00+00:00       69.778595\n",
       "1     2020-12-01 01:00:00+00:00       52.449440\n",
       "2     2020-12-01 02:00:00+00:00       72.574860\n",
       "3     2020-12-01 03:00:00+00:00       67.813957\n",
       "4     2020-12-01 04:00:00+00:00       65.743805\n",
       "...                         ...             ...\n",
       "8395  2021-11-15 19:00:00+00:00      148.864655\n",
       "8396  2021-11-15 20:00:00+00:00      151.322693\n",
       "8397  2021-11-15 21:00:00+00:00      120.674629\n",
       "8398  2021-11-15 22:00:00+00:00      113.046677\n",
       "8399  2021-11-15 23:00:00+00:00       63.877048\n",
       "\n",
       "[8400 rows x 2 columns]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l03b02_1h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "8cbcea91-9563-42c3-afe8-f53945367a73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>main_meter(kW)</th>\n",
       "      <th>PV_battery_system(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-01 00:00:00+00:00</td>\n",
       "      <td>62.873325</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-01 01:00:00+00:00</td>\n",
       "      <td>66.489815</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-04-01 02:00:00+00:00</td>\n",
       "      <td>65.625000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-04-01 03:00:00+00:00</td>\n",
       "      <td>67.636673</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-04-01 04:00:00+00:00</td>\n",
       "      <td>65.049164</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   timestamp  main_meter(kW)  PV_battery_system(kW)\n",
       "0  2021-04-01 00:00:00+00:00       62.873325                    NaN\n",
       "1  2021-04-01 01:00:00+00:00       66.489815                    0.0\n",
       "2  2021-04-01 02:00:00+00:00       65.625000                    NaN\n",
       "3  2021-04-01 03:00:00+00:00       67.636673                    NaN\n",
       "4  2021-04-01 04:00:00+00:00       65.049164                    NaN"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l06b01_1h.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "40f7ef84-c6ce-4c0f-bf9f-77f1803624d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned data saved to: ./train_public/1h/L03.B02_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L06.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L09.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L10.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B02_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B03_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B04_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B05_1H_cleaned.csv\n"
     ]
    }
   ],
   "source": [
    "# Function to clean a building energy consumption dataset\n",
    "def clean_building_energy_data(file_path):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "\n",
    "    # 1. Check for daylight saving time or duplicated timestamps\n",
    "    duplicated_timestamps = data['timestamp'][data['timestamp'].duplicated()]\n",
    "    if not duplicated_timestamps.empty:\n",
    "        print(f\"Duplicate or suspicious timestamps found in {file_path}:\")\n",
    "        print(duplicated_timestamps)\n",
    "\n",
    "    # 2. Split the \"timestamp\" column into \"date\" and \"time\"\n",
    "    data['timestamp'] = pd.to_datetime(data['timestamp'])\n",
    "    data['date'] = data['timestamp'].dt.date\n",
    "    data['time'] = data['timestamp'].dt.time\n",
    "\n",
    "    # Drop the old \"timestamp\" column\n",
    "    data = data.drop(columns=['timestamp'])\n",
    "\n",
    "    # 3. Calculate \"energy_consumption(kW)\"\n",
    "    if 'PV_battery_system(kW)' in data.columns:\n",
    "        data['energy_consumption(kW)'] = data['main_meter(kW)'] + data['PV_battery_system(kW)']\n",
    "        data = data.drop(columns=['main_meter(kW)', 'PV_battery_system(kW)'])\n",
    "    else:\n",
    "        data = data.rename(columns={'main_meter(kW)': 'energy_consumption(kW)'})\n",
    "\n",
    "    # Save the cleaned data back to a new CSV file\n",
    "    cleaned_file_path = file_path.replace('.csv', '_cleaned.csv')\n",
    "    data.to_csv(cleaned_file_path, index=False)\n",
    "\n",
    "    return cleaned_file_path\n",
    "\n",
    "# List of building energy files\n",
    "building_files = [\n",
    " './train_public/1h/L03.B02_1H.csv',\n",
    " './train_public/1h/L06.B01_1H.csv',\n",
    " './train_public/1h/L09.B01_1H.csv',\n",
    " './train_public/1h/L10.B01_1H.csv',\n",
    " './train_public/1h/L14.B01_1H.csv',\n",
    " './train_public/1h/L14.B02_1H.csv',\n",
    " './train_public/1h/L14.B03_1H.csv',\n",
    " './train_public/1h/L14.B04_1H.csv',\n",
    " './train_public/1h/L14.B05_1H.csv'\n",
    "]\n",
    "\n",
    "# Apply the cleaning function to each building dataset\n",
    "for building_file in building_files:\n",
    "    cleaned_file = clean_building_energy_data(building_file)\n",
    "    print(f\"Cleaned data saved to: {cleaned_file}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d452ac7-f2d1-45b4-ba23-4dcb337129f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "'./train_public/1h/L03.B02_1H.csv',\n",
    " './train_public/1h/L06.B01_1H.csv',\n",
    " './train_public/1h/L09.B01_1H.csv',\n",
    " './train_public/1h/L10.B01_1H.csv',\n",
    " './train_public/1h/L14.B01_1H.csv',\n",
    " './train_public/1h/L14.B02_1H.csv',\n",
    " './train_public/1h/L14.B03_1H.csv',\n",
    " './train_public/1h/L14.B04_1H.csv',\n",
    " './train_public/1h/L14.B05_1H.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "18b786cc-9e76-4cf3-bd01-c2ba15beba1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned data saved to: ./train_public/1h/L03.B02_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L06.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L09.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L10.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B01_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B02_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B03_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B04_1H_cleaned.csv\n",
      "Cleaned data saved to: ./train_public/1h/L14.B05_1H_cleaned.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "# Function to clean a building energy consumption dataset\n",
    "def clean_energy_data(file_path):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "\n",
    "    # 1. Check for daylight saving time duplicates in the \"timestamp\" column\n",
    "    if data['timestamp'].duplicated().any():\n",
    "        duplicates = data[data['timestamp'].duplicated()]\n",
    "        print(f\"Found duplicate timestamps in {file_path}:\")\n",
    "        print(duplicates)\n",
    "    \n",
    "    # 2. Split \"timestamp\" into \"date\" and \"time\"\n",
    "    data['timestamp'] = pd.to_datetime(data['timestamp'])\n",
    "    data['date'] = data['timestamp'].dt.date\n",
    "    data['time'] = data['timestamp'].dt.time\n",
    "\n",
    "    # Drop the original \"timestamp\" column\n",
    "    data = data.drop(columns=['timestamp'])\n",
    "\n",
    "    # 3. Convert \"main_meter(kW)\" and \"PV_battery_system(kW)\" to numeric and handle missing values\n",
    "    data['main_meter(kW)'] = pd.to_numeric(data['main_meter(kW)'], errors='coerce').fillna(0)\n",
    "\n",
    "    if 'PV_battery_system(kW)' in data.columns:\n",
    "        data['PV_battery_system(kW)'] = pd.to_numeric(data['PV_battery_system(kW)'], errors='coerce').fillna(0)\n",
    "\n",
    "    # 4. Handle \"energy_consumption(kW)\" calculation and column renaming\n",
    "    if 'PV_battery_system(kW)' in data.columns:\n",
    "        data['energy_consumption(kW)'] = data['main_meter(kW)'] + data['PV_battery_system(kW)']\n",
    "        data = data.drop(columns=['main_meter(kW)', 'PV_battery_system(kW)'])\n",
    "    else:\n",
    "        data = data.rename(columns={'main_meter(kW)': 'energy_consumption(kW)'})\n",
    "\n",
    "    # Reorder columns to have \"date\", \"time\", and \"energy_consumption(kW)\"\n",
    "    data = data[['date', 'time', 'energy_consumption(kW)']]\n",
    "\n",
    "    # Save the cleaned data to a new CSV file\n",
    "    cleaned_file_path = file_path.replace('.csv', '_cleaned.csv')\n",
    "    data.to_csv(cleaned_file_path, index=False)\n",
    "\n",
    "    return cleaned_file_path\n",
    "\n",
    "# Directory paths for input and output\n",
    "input_dir = ''  # Replace with the path to your input directory\n",
    "\n",
    "# List of input files\n",
    "input_files = [\n",
    " './train_public/1h/L03.B02_1H.csv',\n",
    " './train_public/1h/L06.B01_1H.csv',\n",
    " './train_public/1h/L09.B01_1H.csv',\n",
    " './train_public/1h/L10.B01_1H.csv',\n",
    " './train_public/1h/L14.B01_1H.csv',\n",
    " './train_public/1h/L14.B02_1H.csv',\n",
    " './train_public/1h/L14.B03_1H.csv',\n",
    " './train_public/1h/L14.B04_1H.csv',\n",
    " './train_public/1h/L14.B05_1H.csv'\n",
    "]\n",
    "\n",
    "# Clean each file\n",
    "for input_file in input_files:\n",
    "    file_path = os.path.join(input_dir, input_file)\n",
    "    cleaned_file = clean_energy_data(file_path)\n",
    "    print(f\"Cleaned data saved to: {cleaned_file}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "3ddc94d0-ec58-4137-a8b2-23f528982bcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>energy_consumption(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-01</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>62.873325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-01</td>\n",
       "      <td>01:00:00</td>\n",
       "      <td>66.489815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-04-01</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>65.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-04-01</td>\n",
       "      <td>03:00:00</td>\n",
       "      <td>67.636673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-04-01</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>65.049164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date      time  energy_consumption(kW)\n",
       "0  2021-04-01  00:00:00               62.873325\n",
       "1  2021-04-01  01:00:00               66.489815\n",
       "2  2021-04-01  02:00:00               65.625000\n",
       "3  2021-04-01  03:00:00               67.636673\n",
       "4  2021-04-01  04:00:00               65.049164"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l06b01_1h_cleaned = pd.read_csv('train_public/1h_cleaned/L06.B01_1H_cleaned.csv')\n",
    "l06b01_1h_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "59d6459e-9cd9-4d36-9eba-dcc89bd61ca1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>main_meter(kW)</th>\n",
       "      <th>PV_battery_system(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-01 00:00:00+00:00</td>\n",
       "      <td>62.873325</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-01 01:00:00+00:00</td>\n",
       "      <td>66.489815</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-04-01 02:00:00+00:00</td>\n",
       "      <td>65.625000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-04-01 03:00:00+00:00</td>\n",
       "      <td>67.636673</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-04-01 04:00:00+00:00</td>\n",
       "      <td>65.049164</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   timestamp  main_meter(kW)  PV_battery_system(kW)\n",
       "0  2021-04-01 00:00:00+00:00       62.873325                    NaN\n",
       "1  2021-04-01 01:00:00+00:00       66.489815                    0.0\n",
       "2  2021-04-01 02:00:00+00:00       65.625000                    NaN\n",
       "3  2021-04-01 03:00:00+00:00       67.636673                    NaN\n",
       "4  2021-04-01 04:00:00+00:00       65.049164                    NaN"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l06b01_1h.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "4497474e-c3b6-4dd1-a0d3-a9db403cd068",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No daylight saving recording in train_public/weather/L03_weather_train.csv\n",
      "Cleaned data for l03 saved to: train_public/weather/L03_weather_train_cleaned.csv\n",
      "No daylight saving recording in train_public/weather/L06_weather_train.csv\n",
      "Cleaned data for l06 saved to: train_public/weather/L06_weather_train_cleaned.csv\n",
      "No daylight saving recording in train_public/weather/L09_weather_train.csv\n",
      "Cleaned data for l09 saved to: train_public/weather/L09_weather_train_cleaned.csv\n",
      "No daylight saving recording in train_public/weather/L10_weather_train.csv\n",
      "Cleaned data for l10 saved to: train_public/weather/L10_weather_train_cleaned.csv\n",
      "No daylight saving recording in train_public/weather/L14_weather_train.csv\n",
      "Cleaned data for l14 saved to: train_public/weather/L14_weather_train_cleaned.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Function to clean weather data\n",
    "def clean_weather_data(file_path, location):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "\n",
    "    # 1. Check for daylight saving time duplicates in the \"timestamp\" column\n",
    "    if data['timestamp'].duplicated().any():\n",
    "        duplicates = data[data['timestamp'].duplicated()]\n",
    "        print(f\"Found duplicate timestamps in {file_path}:\")\n",
    "        print(duplicates)\n",
    "    else:\n",
    "        print(f\"No daylight saving recording in {file_path}\")\n",
    "\n",
    "    # 2. Split \"timestamp\" into \"date\" and \"time\"\n",
    "    data['timestamp'] = pd.to_datetime(data['timestamp'])\n",
    "    data['date'] = data['timestamp'].dt.date\n",
    "    data['time'] = data['timestamp'].dt.time\n",
    "\n",
    "    # Drop the original \"timestamp\" column\n",
    "    data = data.drop(columns=['timestamp'])\n",
    "\n",
    "    # 3. Revise all negative values in \"direct_solar_radiation(W/m^2)\" and \"diffuse_solar_radiation(W/m^2)\" into 0\n",
    "    data['direct_solar_radiation(W/m^2)'] = data['direct_solar_radiation(W/m^2)'].clip(lower=0)\n",
    "    data['diffuse_solar_radiation(W/m^2)'] = data['diffuse_solar_radiation(W/m^2)'].clip(lower=0)\n",
    "\n",
    "    # 4. Location-specific reasonable value checks\n",
    "\n",
    "    # Temperature ranges for different locations\n",
    "    temp_ranges = {\n",
    "        'l03': (-10, 35),  # Denmark\n",
    "        'l06': (-30, 30),  # Norway\n",
    "        'l09': (-5, 45),   # Australia\n",
    "        'l10': (-30, 30),  # Norway\n",
    "        'l14': (-30, 30)   # Norway\n",
    "    }\n",
    "    temp_min, temp_max = temp_ranges[location]\n",
    "    data['air_temperature_at_2m(deg_C)'] = data['air_temperature_at_2m(deg_C)'].clip(temp_min, temp_max)\n",
    "\n",
    "    # Humidity should be between 0% and 100%\n",
    "    data['relative_humidity_at_2m(%)'] = data['relative_humidity_at_2m(%)'].clip(0, 100)\n",
    "\n",
    "    # Wind speed should not be negative; clip extreme wind speeds at 150 km/h\n",
    "    data['wind_speed_at_10m(km/h)'] = data['wind_speed_at_10m(km/h)'].clip(0, 150)\n",
    "\n",
    "    # 5. Convert columns to numeric and handle missing values\n",
    "    columns_to_check = [\n",
    "        'air_temperature_at_2m(deg_C)',\n",
    "        'relative_humidity_at_2m(%)',\n",
    "        'direct_solar_radiation(W/m^2)',\n",
    "        'diffuse_solar_radiation(W/m^2)',\n",
    "        'wind_speed_at_10m(km/h)',\n",
    "        'wind_direction_at_10m(deg)'\n",
    "    ]\n",
    "\n",
    "    for column in columns_to_check:\n",
    "        data[column] = pd.to_numeric(data[column], errors='coerce')\n",
    "\n",
    "        # Fill missing values with the average of the adjacent values\n",
    "        data[column] = data[column].interpolate(method='linear', limit_direction='both')\n",
    "\n",
    "    # 6. Correct wind direction values\n",
    "    data['wind_direction_at_10m(deg)'] = data['wind_direction_at_10m(deg)'].mod(360)\n",
    "\n",
    "    # 7. Correct unrealistic solar radiation values\n",
    "    data['direct_solar_radiation(W/m^2)'] = data['direct_solar_radiation(W/m^2)'].clip(upper=1000)\n",
    "    data['diffuse_solar_radiation(W/m^2)'] = data['diffuse_solar_radiation(W/m^2)'].clip(upper=1000)\n",
    "\n",
    "    # Save the cleaned data back to a new CSV file\n",
    "    cleaned_file_path = file_path.replace('.csv', '_cleaned.csv')\n",
    "    data.to_csv(cleaned_file_path, index=False)\n",
    "\n",
    "    return cleaned_file_path\n",
    "\n",
    "# Directory paths for input and output\n",
    "input_dir = 'train_public/weather/'  # Replace with the path to your weather data directory\n",
    "\n",
    "# Location-specific file names\n",
    "weather_files = {\n",
    "    'l03': 'L03_weather_train.csv',\n",
    "    'l06': 'L06_weather_train.csv',\n",
    "    'l09': 'L09_weather_train.csv',\n",
    "    'l10': 'L10_weather_train.csv',\n",
    "    'l14': 'L14_weather_train.csv'\n",
    "}\n",
    "\n",
    "# Clean each weather file\n",
    "for location, weather_file in weather_files.items():\n",
    "    file_path = os.path.join(input_dir, weather_file)\n",
    "    cleaned_file = clean_weather_data(file_path, location)\n",
    "    print(f\"Cleaned data for {location} saved to: {cleaned_file}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "c857ad3a-7b05-4f20-9f56-008d12355cb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l09b01_30min (16554, 3)\n",
      "l10b01_30min (27731, 2)\n",
      "l09b01_15min (33044, 3)\n",
      "l10b01_15min (55460, 2)\n",
      "l09b01_5min (98978, 3)\n"
     ]
    }
   ],
   "source": [
    "# 30min data\n",
    "l09b01_30min = pd.read_csv('train_public/30min/L09.B01_30min.csv')\n",
    "l10b01_30min = pd.read_csv('train_public/30min/L10.B01_30min.csv')\n",
    "\n",
    "# 15min data\n",
    "l09b01_15min = pd.read_csv('train_public/15min/L09.B01_15min.csv')\n",
    "l10b01_15min = pd.read_csv('train_public/15min/L10.B01_15min.csv')\n",
    "\n",
    "# 5min data\n",
    "l09b01_5min = pd.read_csv('train_public/5min/L09.B01_5min.csv')\n",
    "\n",
    "print('l09b01_30min', l09b01_30min.shape)\n",
    "print(\"l10b01_30min\", l10b01_30min.shape)\n",
    "print('l09b01_15min', l09b01_15min.shape)\n",
    "print('l10b01_15min', l10b01_15min.shape)\n",
    "print('l09b01_5min', l09b01_5min.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "dda29eed-d244-47cf-8524-4c456a9ef066",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       timestamp  main_meter(kW)  PV_battery_system(kW)\n",
      "98968  2021-12-31 23:10:00+00:00           -42.0                  114.0\n",
      "98969  2021-12-31 23:15:00+00:00           -70.0                  152.0\n",
      "98970  2021-12-31 23:20:00+00:00          -147.0                  228.0\n",
      "98971  2021-12-31 23:25:00+00:00           -17.0                   97.0\n",
      "98972  2021-12-31 23:30:00+00:00            12.0                   62.0\n",
      "98973  2021-12-31 23:35:00+00:00            12.0                   62.0\n",
      "98974  2021-12-31 23:40:00+00:00            13.0                   65.0\n",
      "98975  2021-12-31 23:45:00+00:00           -14.0                   97.0\n",
      "98976  2021-12-31 23:50:00+00:00           -30.0                  114.0\n",
      "98977  2021-12-31 23:55:00+00:00           -32.0                  112.0\n",
      "                       timestamp  main_meter(kW)  PV_battery_system(kW)\n",
      "33034  2021-12-31 21:30:00+00:00      -20.333333              87.000000\n",
      "33035  2021-12-31 21:45:00+00:00      -13.000000              96.666667\n",
      "33036  2021-12-31 22:00:00+00:00      -25.000000             110.000000\n",
      "33037  2021-12-31 22:15:00+00:00        2.000000              79.666667\n",
      "33038  2021-12-31 22:30:00+00:00       39.000000              36.333333\n",
      "33039  2021-12-31 22:45:00+00:00      -24.000000             104.666667\n",
      "33040  2021-12-31 23:00:00+00:00      -38.333333             111.000000\n",
      "33041  2021-12-31 23:15:00+00:00      -78.000000             159.000000\n",
      "33042  2021-12-31 23:30:00+00:00       12.333333              63.000000\n",
      "33043  2021-12-31 23:45:00+00:00      -25.333333             107.666667\n",
      "                       timestamp  main_meter(kW)  PV_battery_system(kW)\n",
      "16549  2021-12-31 21:30:00+00:00      -16.666667              91.833333\n",
      "16550  2021-12-31 22:00:00+00:00      -11.500000              94.833333\n",
      "16551  2021-12-31 22:30:00+00:00        7.500000              70.500000\n",
      "16552  2021-12-31 23:00:00+00:00      -58.166667             135.000000\n",
      "16553  2021-12-31 23:30:00+00:00       -6.500000              85.333333\n",
      "                      timestamp  main_meter(kW)  PV_battery_system(kW)\n",
      "8282  2021-12-31 19:00:00+00:00       62.250000              12.083333\n",
      "8283  2021-12-31 20:00:00+00:00       15.833333              60.666667\n",
      "8284  2021-12-31 21:00:00+00:00      -21.833333              97.666667\n",
      "8285  2021-12-31 22:00:00+00:00       -2.000000              82.666667\n",
      "8286  2021-12-31 23:00:00+00:00      -32.333333             110.166667\n"
     ]
    }
   ],
   "source": [
    "print(l09b01_5min.tail(10))\n",
    "print(l09b01_15min.tail(10))\n",
    "print(l09b01_30min.tail())\n",
    "print(l09b01_1h.tail())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "6ae2b361-e4e7-4fc8-aa6d-64d0ba7e8845",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No daylight saving recording in train_public/<1h/L09.B01_30min.csv\n",
      "Cleaned data saved to: train_public/<1h/L09.B01_30min_cleaned.csv\n",
      "No daylight saving recording in train_public/<1h/l10.B01_30min.csv\n",
      "Cleaned data saved to: train_public/<1h/l10.B01_30min_cleaned.csv\n",
      "No daylight saving recording in train_public/<1h/L09.B01_15min.csv\n",
      "Cleaned data saved to: train_public/<1h/L09.B01_15min_cleaned.csv\n",
      "No daylight saving recording in train_public/<1h/L10.B01_15min.csv\n",
      "Cleaned data saved to: train_public/<1h/L10.B01_15min_cleaned.csv\n",
      "No daylight saving recording in train_public/<1h/L09.B01_5min.csv\n",
      "Cleaned data saved to: train_public/<1h/L09.B01_5min_cleaned.csv\n"
     ]
    }
   ],
   "source": [
    "# Function to clean a building energy consumption dataset\n",
    "def clean_energy_data(file_path):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "\n",
    "    # 1. Check for daylight saving time duplicates in the \"timestamp\" column\n",
    "    if data['timestamp'].duplicated().any():\n",
    "        duplicates = data[data['timestamp'].duplicated()]\n",
    "        print(f\"Found duplicate timestamps in {file_path}:\")\n",
    "        print(duplicates)\n",
    "    else:\n",
    "        print(f\"No daylight saving recording in {file_path}\")\n",
    "\n",
    "    # 2. Split \"timestamp\" into \"date\" and \"time\"\n",
    "    data['timestamp'] = pd.to_datetime(data['timestamp'])\n",
    "    data['date'] = data['timestamp'].dt.date\n",
    "    data['time'] = data['timestamp'].dt.time\n",
    "\n",
    "    # Drop the original \"timestamp\" column\n",
    "    data = data.drop(columns=['timestamp'])\n",
    "\n",
    "    # 3. Convert \"main_meter(kW)\" and \"PV_battery_system(kW)\" to numeric and handle missing values\n",
    "    data['main_meter(kW)'] = pd.to_numeric(data['main_meter(kW)'], errors='coerce').fillna(0)\n",
    "\n",
    "    if 'PV_battery_system(kW)' in data.columns:\n",
    "        data['PV_battery_system(kW)'] = pd.to_numeric(data['PV_battery_system(kW)'], errors='coerce').fillna(0)\n",
    "\n",
    "    # 4. Handle \"energy_consumption(kW)\" calculation and column renaming\n",
    "    if 'PV_battery_system(kW)' in data.columns:\n",
    "        data['energy_consumption(kW)'] = data['main_meter(kW)'] + data['PV_battery_system(kW)']\n",
    "        data = data.drop(columns=['main_meter(kW)', 'PV_battery_system(kW)'])\n",
    "    else:\n",
    "        data = data.rename(columns={'main_meter(kW)': 'energy_consumption(kW)'})\n",
    "\n",
    "    # Reorder columns to have \"date\", \"time\", and \"energy_consumption(kW)\"\n",
    "    data = data[['date', 'time', 'energy_consumption(kW)']]\n",
    "\n",
    "    # Save the cleaned data to a new CSV file\n",
    "    cleaned_file_path = file_path.replace('.csv', '_cleaned.csv')\n",
    "    data.to_csv(cleaned_file_path, index=False)\n",
    "\n",
    "    return cleaned_file_path\n",
    "\n",
    "# Directory paths for input and output\n",
    "input_dir = 'train_public/<1h/'  # Replace with the path to your input directory\n",
    "\n",
    "# List of input files\n",
    "input_files = [\n",
    "    'L09.B01_30min.csv', \n",
    "    'l10.B01_30min.csv', \n",
    "    'L09.B01_15min.csv', \n",
    "    'L10.B01_15min.csv', \n",
    "    'L09.B01_5min.csv'\n",
    "]\n",
    "\n",
    "# Clean each file\n",
    "for input_file in input_files:\n",
    "    file_path = os.path.join(input_dir, input_file)\n",
    "    cleaned_file = clean_energy_data(file_path)\n",
    "    print(f\"Cleaned data saved to: {cleaned_file}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7a8259e-cce9-4740-afc7-f5e9f64d0b29",
   "metadata": {},
   "source": [
    "### Possible additional check\n",
    "1. Timestamp Continuity Check:\n",
    "Purpose: Ensure that there are no missing or irregular time intervals in the dataset. For example, in a dataset with a 30-minute resolution, the difference between consecutive timestamps should always be 30 minutes.\n",
    "Fix: If gaps or irregularities are detected, you may want to interpolate or add the missing timestamps with appropriate values (e.g., by interpolating the energy consumption).\n",
    "2. Outlier Detection in Energy Consumption:\n",
    "Purpose: Identify and address extreme outliers in the energy consumption data, which could indicate sensor errors or other anomalies.\n",
    "Fix: You can use statistical methods (e.g., z-scores, interquartile range) to detect outliers and decide whether to correct, remove, or further investigate them.\n",
    "3. Logical Consistency Checks:\n",
    "Purpose: Ensure that energy consumption values are within a realistic range for the building and time of year. For instance, energy consumption values should generally be non-negative and should not exceed plausible limits based on the building's size and usage patterns.\n",
    "Fix: Clip or flag values that fall outside of expected ranges for further investigation.\n",
    "4. Seasonal/Time-of-Day Analysis:\n",
    "Purpose: Perform a sanity check by comparing energy consumption patterns across different times of the day, days of the week, or seasons. For example, a significant energy spike during the night might indicate a potential issue.\n",
    "Fix: Investigate and potentially adjust any unexpected patterns.\n",
    "5. Correlation Analysis:\n",
    "Purpose: Check the correlation between the main_meter(kW) and PV_battery_system(kW) columns (if present). In a well-functioning system, these might have a predictable relationship, especially if the PV system is directly offsetting some of the main meter load.\n",
    "Fix: If the correlation is unexpectedly low or high, it might warrant further investigation or corrections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "ba3cc96c-7c80-4147-85db-cc7c3f866613",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Irregular timestamps found in train_public/cleaned/L03.B02_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2020-12-01  00:00:00             NaT\n",
      "1  2020-12-01  01:00:00 0 days 01:00:00\n",
      "2  2020-12-01  02:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L03.B02_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "1016  2021-01-12  08:00:00              211.758621  3.370834\n",
      "1041  2021-01-13  09:00:00              211.166672  3.356213\n",
      "1088  2021-01-15  08:00:00              210.816086  3.347554\n",
      "1160  2021-01-18  08:00:00              220.204437  3.579439\n",
      "1352  2021-01-26  08:00:00              214.899231  3.448404\n",
      "...          ...       ...                     ...       ...\n",
      "8275  2021-11-10  19:00:00              197.809525  3.026301\n",
      "8287  2021-11-11  07:00:00              238.903503  4.041292\n",
      "8383  2021-11-15  07:00:00              226.410721  3.732730\n",
      "8384  2021-11-15  08:00:00              231.380951  3.855491\n",
      "8385  2021-11-15  09:00:00              229.672043  3.813282\n",
      "\n",
      "[68 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L06.B01_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2021-04-01  00:00:00             NaT\n",
      "1  2021-04-01  01:00:00 0 days 01:00:00\n",
      "2  2021-04-01  02:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "5113  2021-10-31  01:00:00 0 days 01:00:00\n",
      "5114  2021-10-31  02:00:00 0 days 01:00:00\n",
      "5115  2021-11-01  09:00:00 1 days 07:00:00\n",
      "5116  2021-11-01  10:00:00 0 days 01:00:00\n",
      "5117  2021-11-01  11:00:00 0 days 01:00:00\n",
      "No outliers detected in train_public/cleaned/L06.B01_1H_cleaned.csv\n",
      "Irregular timestamps found in train_public/cleaned/L09.B01_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2021-01-01  00:00:00             NaT\n",
      "1  2021-01-01  01:00:00 0 days 01:00:00\n",
      "2  2021-01-01  02:00:00 0 days 01:00:00\n",
      "           date      time           delta\n",
      "178  2021-01-08  10:00:00 0 days 01:00:00\n",
      "179  2021-01-08  11:00:00 0 days 01:00:00\n",
      "180  2021-01-08  22:00:00 0 days 11:00:00\n",
      "181  2021-01-08  23:00:00 0 days 01:00:00\n",
      "182  2021-01-09  00:00:00 0 days 01:00:00\n",
      "           date      time           delta\n",
      "885  2021-02-07  07:00:00 0 days 01:00:00\n",
      "886  2021-02-07  08:00:00 0 days 01:00:00\n",
      "887  2021-02-07  10:00:00 0 days 02:00:00\n",
      "888  2021-02-07  11:00:00 0 days 01:00:00\n",
      "889  2021-02-07  12:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "1238  2021-02-22  01:00:00 0 days 01:00:00\n",
      "1239  2021-02-22  02:00:00 0 days 01:00:00\n",
      "1240  2021-02-22  06:00:00 0 days 04:00:00\n",
      "1241  2021-02-22  07:00:00 0 days 01:00:00\n",
      "1242  2021-02-22  08:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "1246  2021-02-22  12:00:00 0 days 01:00:00\n",
      "1247  2021-02-22  13:00:00 0 days 01:00:00\n",
      "1248  2021-03-01  00:00:00 6 days 11:00:00\n",
      "1249  2021-03-01  01:00:00 0 days 01:00:00\n",
      "1250  2021-03-01  02:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "1869  2021-03-26  21:00:00 0 days 01:00:00\n",
      "1870  2021-03-26  22:00:00 0 days 01:00:00\n",
      "1871  2021-03-27  00:00:00 0 days 02:00:00\n",
      "1872  2021-03-27  01:00:00 0 days 01:00:00\n",
      "1873  2021-03-27  02:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "1931  2021-03-29  12:00:00 0 days 01:00:00\n",
      "1932  2021-03-29  13:00:00 0 days 01:00:00\n",
      "1933  2021-03-29  20:00:00 0 days 07:00:00\n",
      "1934  2021-03-29  21:00:00 0 days 01:00:00\n",
      "1935  2021-03-29  22:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "2679  2021-04-29  22:00:00 0 days 01:00:00\n",
      "2680  2021-04-29  23:00:00 0 days 01:00:00\n",
      "2681  2021-05-03  03:00:00 3 days 04:00:00\n",
      "2682  2021-05-03  04:00:00 0 days 01:00:00\n",
      "2683  2021-05-03  05:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "2937  2021-05-13  19:00:00 0 days 01:00:00\n",
      "2938  2021-05-13  20:00:00 0 days 01:00:00\n",
      "2939  2021-05-14  18:00:00 0 days 22:00:00\n",
      "2940  2021-05-14  19:00:00 0 days 01:00:00\n",
      "2941  2021-05-14  20:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "2982  2021-05-16  13:00:00 0 days 01:00:00\n",
      "2983  2021-05-16  14:00:00 0 days 01:00:00\n",
      "2984  2021-05-16  21:00:00 0 days 07:00:00\n",
      "2985  2021-05-16  22:00:00 0 days 01:00:00\n",
      "2986  2021-05-16  23:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "4111  2021-07-02  20:00:00 0 days 01:00:00\n",
      "4112  2021-07-02  21:00:00 0 days 01:00:00\n",
      "4113  2021-07-03  00:00:00 0 days 03:00:00\n",
      "4114  2021-07-03  01:00:00 0 days 01:00:00\n",
      "4115  2021-07-03  02:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "4381  2021-07-14  04:00:00 0 days 01:00:00\n",
      "4382  2021-07-14  05:00:00 0 days 01:00:00\n",
      "4383  2021-07-19  00:00:00 4 days 19:00:00\n",
      "4384  2021-07-19  01:00:00 0 days 01:00:00\n",
      "4385  2021-07-19  02:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "4667  2021-07-30  20:00:00 0 days 01:00:00\n",
      "4668  2021-07-30  21:00:00 0 days 01:00:00\n",
      "4669  2021-07-31  02:00:00 0 days 05:00:00\n",
      "4670  2021-07-31  03:00:00 0 days 01:00:00\n",
      "4671  2021-07-31  04:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "4894  2021-08-09  11:00:00 0 days 01:00:00\n",
      "4895  2021-08-09  12:00:00 0 days 01:00:00\n",
      "4896  2021-08-09  23:00:00 0 days 11:00:00\n",
      "4897  2021-08-10  00:00:00 0 days 01:00:00\n",
      "4898  2021-08-10  01:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "5382  2021-08-30  05:00:00 0 days 01:00:00\n",
      "5383  2021-08-30  06:00:00 0 days 01:00:00\n",
      "5384  2021-09-01  23:00:00 2 days 17:00:00\n",
      "5385  2021-09-02  00:00:00 0 days 01:00:00\n",
      "5386  2021-09-02  01:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "6490  2021-10-18  01:00:00 0 days 01:00:00\n",
      "6491  2021-10-18  02:00:00 0 days 01:00:00\n",
      "6492  2021-10-18  05:00:00 0 days 03:00:00\n",
      "6493  2021-10-18  06:00:00 0 days 01:00:00\n",
      "6494  2021-10-18  07:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L09.B01_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "94    2021-01-04  22:00:00              328.916667  3.037795\n",
      "95    2021-01-04  23:00:00              339.583333  3.204839\n",
      "96    2021-01-05  00:00:00              345.666667  3.300105\n",
      "97    2021-01-05  01:00:00              332.750000  3.097827\n",
      "99    2021-01-05  03:00:00              340.416667  3.217889\n",
      "...          ...       ...                     ...       ...\n",
      "8047  2021-12-22  00:00:00              335.000000  3.133062\n",
      "8048  2021-12-22  01:00:00              360.692308  3.535411\n",
      "8069  2021-12-22  22:00:00              327.333333  3.013000\n",
      "8095  2021-12-24  00:00:00              329.583333  3.048236\n",
      "8096  2021-12-24  01:00:00              327.846154  3.021031\n",
      "\n",
      "[204 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L10.B01_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2020-01-01  00:00:00             NaT\n",
      "1  2020-01-01  01:00:00 0 days 01:00:00\n",
      "2  2020-01-01  02:00:00 0 days 01:00:00\n",
      "            date      time           delta\n",
      "4199  2020-06-23  23:00:00 0 days 01:00:00\n",
      "4200  2020-06-24  00:00:00 0 days 01:00:00\n",
      "4201  2020-06-24  06:00:00 0 days 06:00:00\n",
      "4202  2020-06-24  07:00:00 0 days 01:00:00\n",
      "4203  2020-06-24  08:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L10.B01_1H_cleaned.csv:\n",
      "             date      time  energy_consumption(kW)   z_score\n",
      "3897   2020-06-11  09:00:00              551.055901  3.389851\n",
      "3898   2020-06-11  10:00:00              583.004845  3.703047\n",
      "3899   2020-06-11  11:00:00              551.705078  3.396215\n",
      "3920   2020-06-12  08:00:00              541.553207  3.296697\n",
      "3921   2020-06-12  09:00:00              560.742851  3.484812\n",
      "...           ...       ...                     ...       ...\n",
      "13448  2021-07-14  13:00:00              530.813354  3.191414\n",
      "13731  2021-07-26  08:00:00              551.563282  3.394825\n",
      "13732  2021-07-26  09:00:00              620.004166  4.065750\n",
      "13733  2021-07-26  10:00:00              591.954758  3.790782\n",
      "13734  2021-07-26  11:00:00              546.500015  3.345190\n",
      "\n",
      "[236 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L14.B01_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2018-01-01  00:00:00             NaT\n",
      "1  2018-01-01  01:00:00 0 days 01:00:00\n",
      "2  2018-01-01  02:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L14.B01_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "199   2018-01-09  07:00:00                   232.0  4.036932\n",
      "200   2018-01-09  08:00:00                   231.0  4.014037\n",
      "201   2018-01-09  09:00:00                   238.0  4.174301\n",
      "202   2018-01-09  10:00:00                   240.0  4.220091\n",
      "203   2018-01-09  11:00:00                   250.0  4.449040\n",
      "...          ...       ...                     ...       ...\n",
      "8337  2018-12-14  09:00:00                   210.0  3.533244\n",
      "8338  2018-12-14  10:00:00                   210.0  3.533244\n",
      "8339  2018-12-14  11:00:00                   208.0  3.487454\n",
      "8340  2018-12-14  12:00:00                   204.0  3.395874\n",
      "8341  2018-12-14  13:00:00                   196.0  3.212715\n",
      "\n",
      "[140 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L14.B02_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2018-01-01  00:00:00             NaT\n",
      "1  2018-01-01  01:00:00 0 days 01:00:00\n",
      "2  2018-01-01  02:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L14.B02_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "7     2018-01-01  07:00:00                    88.0  3.104002\n",
      "8     2018-01-01  08:00:00                    93.0  3.413675\n",
      "9     2018-01-01  09:00:00                   106.0  4.218824\n",
      "10    2018-01-01  10:00:00                    90.0  3.227871\n",
      "11    2018-01-01  11:00:00                    88.0  3.104002\n",
      "...          ...       ...                     ...       ...\n",
      "1829  2018-03-18  05:00:00                    90.0  3.227871\n",
      "1831  2018-03-18  07:00:00                    92.0  3.351740\n",
      "1832  2018-03-18  08:00:00                    87.0  3.042068\n",
      "6462  2018-09-27  06:00:00                    88.0  3.104002\n",
      "8334  2018-12-14  06:00:00                    93.0  3.413675\n",
      "\n",
      "[145 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L14.B03_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2018-01-01  00:00:00             NaT\n",
      "1  2018-01-01  01:00:00 0 days 01:00:00\n",
      "2  2018-01-01  02:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L14.B03_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "31    2018-01-02  07:00:00                   186.0  3.130075\n",
      "32    2018-01-02  08:00:00                   191.0  3.246813\n",
      "33    2018-01-02  09:00:00                   189.0  3.200118\n",
      "34    2018-01-02  10:00:00                   190.0  3.223465\n",
      "35    2018-01-02  11:00:00                   192.0  3.270160\n",
      "...          ...       ...                     ...       ...\n",
      "8434  2018-12-18  10:00:00                   196.0  3.363550\n",
      "8435  2018-12-18  11:00:00                   187.0  3.153423\n",
      "8437  2018-12-18  13:00:00                   185.0  3.106728\n",
      "8455  2018-12-19  07:00:00                   191.0  3.246813\n",
      "8456  2018-12-19  08:00:00                   189.0  3.200118\n",
      "\n",
      "[124 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L14.B04_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2018-01-01  00:00:00             NaT\n",
      "1  2018-01-01  01:00:00 0 days 01:00:00\n",
      "2  2018-01-01  02:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L14.B04_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "56    2018-01-03  08:00:00                   716.0  3.012012\n",
      "202   2018-01-09  10:00:00                   824.0  3.773544\n",
      "203   2018-01-09  11:00:00                   810.0  3.674827\n",
      "204   2018-01-09  12:00:00                   786.0  3.505598\n",
      "205   2018-01-09  13:00:00                   776.0  3.435085\n",
      "206   2018-01-09  14:00:00                   770.0  3.392778\n",
      "207   2018-01-09  15:00:00                   759.0  3.315214\n",
      "220   2018-01-10  04:00:00                   734.0  3.138934\n",
      "221   2018-01-10  05:00:00                   765.0  3.357522\n",
      "222   2018-01-10  06:00:00                   823.0  3.766493\n",
      "223   2018-01-10  07:00:00                   830.0  3.815851\n",
      "224   2018-01-10  08:00:00                   829.0  3.808800\n",
      "225   2018-01-10  09:00:00                   805.0  3.639571\n",
      "226   2018-01-10  10:00:00                   790.0  3.533802\n",
      "227   2018-01-10  11:00:00                   780.0  3.463290\n",
      "228   2018-01-10  12:00:00                   759.0  3.315214\n",
      "229   2018-01-10  13:00:00                   716.0  3.012012\n",
      "515   2018-01-22  11:00:00                   760.0  3.322266\n",
      "516   2018-01-22  12:00:00                   753.0  3.272907\n",
      "1350  2018-02-26  06:00:00                   737.0  3.160088\n",
      "1351  2018-02-26  07:00:00                   728.0  3.096627\n",
      "1423  2018-03-01  07:00:00                   723.0  3.061370\n",
      "8573  2018-12-24  05:00:00                   723.0  3.061370\n",
      "8574  2018-12-24  06:00:00                   724.0  3.068422\n",
      "8575  2018-12-24  07:00:00                   717.0  3.019063\n",
      "8576  2018-12-24  08:00:00                   734.0  3.138934\n",
      "Irregular timestamps found in train_public/cleaned/L14.B05_1H_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2018-01-01  00:00:00             NaT\n",
      "1  2018-01-01  01:00:00 0 days 01:00:00\n",
      "2  2018-01-01  02:00:00 0 days 01:00:00\n",
      "Outliers detected in train_public/cleaned/L14.B05_1H_cleaned.csv:\n",
      "            date      time  energy_consumption(kW)   z_score\n",
      "29    2018-01-02  05:00:00                   476.0  3.032655\n",
      "52    2018-01-03  04:00:00                   475.0  3.024335\n",
      "202   2018-01-09  10:00:00                   517.0  3.373784\n",
      "203   2018-01-09  11:00:00                   509.0  3.307223\n",
      "204   2018-01-09  12:00:00                   489.0  3.140818\n",
      "205   2018-01-09  13:00:00                   488.0  3.132498\n",
      "206   2018-01-09  14:00:00                   499.0  3.224020\n",
      "207   2018-01-09  15:00:00                   517.0  3.373784\n",
      "208   2018-01-09  16:00:00                   476.0  3.032655\n",
      "219   2018-01-10  03:00:00                   476.0  3.032655\n",
      "220   2018-01-10  04:00:00                   556.0  3.698273\n",
      "221   2018-01-10  05:00:00                   547.0  3.623391\n",
      "222   2018-01-10  06:00:00                   566.0  3.781475\n",
      "223   2018-01-10  07:00:00                   534.0  3.515228\n",
      "224   2018-01-10  08:00:00                   528.0  3.465307\n",
      "225   2018-01-10  09:00:00                   509.0  3.307223\n",
      "226   2018-01-10  10:00:00                   489.0  3.140818\n",
      "227   2018-01-10  11:00:00                   479.0  3.057616\n",
      "1158  2018-02-18  06:00:00                   485.0  3.107537\n",
      "1349  2018-02-26  05:00:00                   489.0  3.140818\n",
      "1350  2018-02-26  06:00:00                   496.0  3.199060\n",
      "1445  2018-03-02  05:00:00                   478.0  3.049296\n",
      "8382  2018-12-16  06:00:00                   475.0  3.024335\n",
      "8550  2018-12-23  06:00:00                   506.0  3.282262\n",
      "8572  2018-12-24  04:00:00                   475.0  3.024335\n",
      "8573  2018-12-24  05:00:00                   536.0  3.531869\n",
      "8574  2018-12-24  06:00:00                   525.0  3.440346\n",
      "8575  2018-12-24  07:00:00                   516.0  3.365464\n",
      "8576  2018-12-24  08:00:00                   535.0  3.523548\n",
      "8577  2018-12-24  09:00:00                   495.0  3.190739\n",
      "8578  2018-12-24  10:00:00                   496.0  3.199060\n",
      "8580  2018-12-24  12:00:00                   496.0  3.199060\n",
      "8581  2018-12-24  13:00:00                   485.0  3.107537\n",
      "8582  2018-12-24  14:00:00                   475.0  3.024335\n",
      "Irregular timestamps found in train_public/cleaned/L09.B01_30min_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2021-01-01  00:00:00             NaT\n",
      "1  2021-01-01  00:30:00 0 days 00:30:00\n",
      "2  2021-01-01  01:00:00 0 days 00:30:00\n",
      "           date      time           delta\n",
      "358  2021-01-08  11:00:00 0 days 00:30:00\n",
      "359  2021-01-08  11:30:00 0 days 00:30:00\n",
      "360  2021-01-08  22:30:00 0 days 11:00:00\n",
      "361  2021-01-08  23:00:00 0 days 00:30:00\n",
      "362  2021-01-08  23:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "1771  2021-02-07  08:00:00 0 days 00:30:00\n",
      "1772  2021-02-07  08:30:00 0 days 00:30:00\n",
      "1773  2021-02-07  10:00:00 0 days 01:30:00\n",
      "1774  2021-02-07  10:30:00 0 days 00:30:00\n",
      "1775  2021-02-07  11:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "2073  2021-02-13  16:00:00 0 days 00:30:00\n",
      "2074  2021-02-13  16:30:00 0 days 00:30:00\n",
      "2075  2021-02-13  17:30:00 0 days 01:00:00\n",
      "2076  2021-02-13  18:00:00 0 days 00:30:00\n",
      "2077  2021-02-13  18:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "2475  2021-02-22  01:30:00 0 days 00:30:00\n",
      "2476  2021-02-22  02:00:00 0 days 00:30:00\n",
      "2477  2021-02-22  06:00:00 0 days 04:00:00\n",
      "2478  2021-02-22  06:30:00 0 days 00:30:00\n",
      "2479  2021-02-22  07:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "2490  2021-02-22  12:30:00 0 days 00:30:00\n",
      "2491  2021-02-22  13:00:00 0 days 00:30:00\n",
      "2492  2021-03-01  00:00:00 6 days 11:00:00\n",
      "2493  2021-03-01  00:30:00 0 days 00:30:00\n",
      "2494  2021-03-01  01:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "3646  2021-03-25  01:00:00 0 days 00:30:00\n",
      "3647  2021-03-25  01:30:00 0 days 00:30:00\n",
      "3648  2021-03-25  02:30:00 0 days 01:00:00\n",
      "3649  2021-03-25  03:00:00 0 days 00:30:00\n",
      "3650  2021-03-25  03:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "3697  2021-03-26  03:00:00 0 days 00:30:00\n",
      "3698  2021-03-26  03:30:00 0 days 00:30:00\n",
      "3699  2021-03-26  04:30:00 0 days 01:00:00\n",
      "3700  2021-03-26  05:00:00 0 days 00:30:00\n",
      "3701  2021-03-26  05:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "3734  2021-03-26  22:00:00 0 days 00:30:00\n",
      "3735  2021-03-26  22:30:00 0 days 00:30:00\n",
      "3736  2021-03-27  00:30:00 0 days 02:00:00\n",
      "3737  2021-03-27  01:00:00 0 days 00:30:00\n",
      "3738  2021-03-27  01:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "3857  2021-03-29  13:00:00 0 days 00:30:00\n",
      "3858  2021-03-29  13:30:00 0 days 00:30:00\n",
      "3859  2021-03-29  20:30:00 0 days 07:00:00\n",
      "3860  2021-03-29  21:00:00 0 days 00:30:00\n",
      "3861  2021-03-29  21:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "4437  2021-04-10  21:30:00 0 days 00:30:00\n",
      "4438  2021-04-10  22:00:00 0 days 00:30:00\n",
      "4439  2021-04-10  23:00:00 0 days 01:00:00\n",
      "4440  2021-04-10  23:30:00 0 days 00:30:00\n",
      "4441  2021-04-11  00:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "5351  2021-04-29  23:00:00 0 days 00:30:00\n",
      "5352  2021-04-29  23:30:00 0 days 00:30:00\n",
      "5353  2021-05-03  03:00:00 3 days 03:30:00\n",
      "5354  2021-05-03  03:30:00 0 days 00:30:00\n",
      "5355  2021-05-03  04:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "5866  2021-05-13  19:30:00 0 days 00:30:00\n",
      "5867  2021-05-13  20:00:00 0 days 00:30:00\n",
      "5868  2021-05-14  18:00:00 0 days 22:00:00\n",
      "5869  2021-05-14  18:30:00 0 days 00:30:00\n",
      "5870  2021-05-14  19:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "5956  2021-05-16  14:00:00 0 days 00:30:00\n",
      "5957  2021-05-16  14:30:00 0 days 00:30:00\n",
      "5958  2021-05-16  21:00:00 0 days 06:30:00\n",
      "5959  2021-05-16  21:30:00 0 days 00:30:00\n",
      "5960  2021-05-16  22:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "8213  2021-07-02  20:30:00 0 days 00:30:00\n",
      "8214  2021-07-02  21:00:00 0 days 00:30:00\n",
      "8215  2021-07-03  00:30:00 0 days 03:30:00\n",
      "8216  2021-07-03  01:00:00 0 days 00:30:00\n",
      "8217  2021-07-03  01:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "8751  2021-07-14  04:30:00 0 days 00:30:00\n",
      "8752  2021-07-14  05:00:00 0 days 00:30:00\n",
      "8753  2021-07-19  00:30:00 4 days 19:30:00\n",
      "8754  2021-07-19  01:00:00 0 days 00:30:00\n",
      "8755  2021-07-19  01:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "9321  2021-07-30  20:30:00 0 days 00:30:00\n",
      "9322  2021-07-30  21:00:00 0 days 00:30:00\n",
      "9323  2021-07-31  02:30:00 0 days 05:30:00\n",
      "9324  2021-07-31  03:00:00 0 days 00:30:00\n",
      "9325  2021-07-31  03:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "9617  2021-08-06  05:30:00 0 days 00:30:00\n",
      "9618  2021-08-06  06:00:00 0 days 00:30:00\n",
      "9619  2021-08-06  07:30:00 0 days 01:30:00\n",
      "9620  2021-08-06  08:00:00 0 days 00:30:00\n",
      "9621  2021-08-06  08:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "9772  2021-08-09  12:00:00 0 days 00:30:00\n",
      "9773  2021-08-09  12:30:00 0 days 00:30:00\n",
      "9774  2021-08-09  23:00:00 0 days 10:30:00\n",
      "9775  2021-08-09  23:30:00 0 days 00:30:00\n",
      "9776  2021-08-10  00:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "10748  2021-08-30  06:00:00 0 days 00:30:00\n",
      "10749  2021-08-30  06:30:00 0 days 00:30:00\n",
      "10750  2021-09-01  23:30:00 2 days 17:00:00\n",
      "10751  2021-09-02  00:00:00 0 days 00:30:00\n",
      "10752  2021-09-02  00:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "12963  2021-10-18  02:00:00 0 days 00:30:00\n",
      "12964  2021-10-18  02:30:00 0 days 00:30:00\n",
      "12965  2021-10-18  05:30:00 0 days 03:00:00\n",
      "12966  2021-10-18  06:00:00 0 days 00:30:00\n",
      "12967  2021-10-18  06:30:00 0 days 00:30:00\n",
      "Outliers detected in train_public/cleaned/L09.B01_30min_cleaned.csv:\n",
      "             date      time  energy_consumption(kW)   z_score\n",
      "189    2021-01-04  22:30:00              333.166667  3.074135\n",
      "190    2021-01-04  23:00:00              341.666667  3.205922\n",
      "191    2021-01-04  23:30:00              337.500000  3.141320\n",
      "192    2021-01-05  00:00:00              353.666667  3.391975\n",
      "193    2021-01-05  00:30:00              337.666667  3.143904\n",
      "...           ...       ...                     ...       ...\n",
      "16120  2021-12-22  23:00:00              356.000000  3.428152\n",
      "16126  2021-12-23  02:00:00              343.166667  3.229179\n",
      "16168  2021-12-23  23:00:00              350.000000  3.335125\n",
      "16171  2021-12-24  00:30:00              341.000000  3.195586\n",
      "16172  2021-12-24  01:00:00              330.166667  3.027621\n",
      "\n",
      "[409 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L10.B01_30min_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2020-01-01  00:00:00             NaT\n",
      "1  2020-01-01  00:30:00 0 days 00:30:00\n",
      "2  2020-01-01  01:00:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "7992  2020-06-15  12:00:00 0 days 00:30:00\n",
      "7993  2020-06-15  12:30:00 0 days 00:30:00\n",
      "7994  2020-06-15  13:30:00 0 days 01:00:00\n",
      "7995  2020-06-15  14:00:00 0 days 00:30:00\n",
      "7996  2020-06-15  14:30:00 0 days 00:30:00\n",
      "            date      time           delta\n",
      "8398  2020-06-23  23:30:00 0 days 00:30:00\n",
      "8399  2020-06-24  00:00:00 0 days 00:30:00\n",
      "8400  2020-06-24  06:30:00 0 days 06:30:00\n",
      "8401  2020-06-24  07:00:00 0 days 00:30:00\n",
      "8402  2020-06-24  07:30:00 0 days 00:30:00\n",
      "Outliers detected in train_public/cleaned/L10.B01_30min_cleaned.csv:\n",
      "             date      time  energy_consumption(kW)   z_score\n",
      "7794   2020-06-11  09:00:00              539.732140  3.258336\n",
      "7795   2020-06-11  09:30:00              563.312515  3.487884\n",
      "7796   2020-06-11  10:00:00              583.087494  3.680387\n",
      "7797   2020-06-11  10:30:00              583.199997  3.681483\n",
      "7798   2020-06-11  11:00:00              577.301132  3.624059\n",
      "...           ...       ...                     ...       ...\n",
      "27461  2021-07-26  09:00:00              618.369644  4.023848\n",
      "27462  2021-07-26  09:30:00              621.414780  4.053492\n",
      "27463  2021-07-26  10:00:00              593.309528  3.779896\n",
      "27464  2021-07-26  10:30:00              590.600006  3.753519\n",
      "27465  2021-07-26  11:00:00              591.460999  3.761901\n",
      "\n",
      "[485 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L09.B01_15min_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2021-01-01  00:00:00             NaT\n",
      "1  2021-01-01  00:15:00 0 days 00:15:00\n",
      "2  2021-01-01  00:30:00 0 days 00:15:00\n",
      "           date      time           delta\n",
      "717  2021-01-08  11:15:00 0 days 00:15:00\n",
      "718  2021-01-08  11:30:00 0 days 00:15:00\n",
      "719  2021-01-08  22:30:00 0 days 11:00:00\n",
      "720  2021-01-08  22:45:00 0 days 00:15:00\n",
      "721  2021-01-08  23:00:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "3543  2021-02-07  08:30:00 0 days 00:15:00\n",
      "3544  2021-02-07  08:45:00 0 days 00:15:00\n",
      "3545  2021-02-07  10:00:00 0 days 01:15:00\n",
      "3546  2021-02-07  10:15:00 0 days 00:15:00\n",
      "3547  2021-02-07  10:30:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "4146  2021-02-13  16:15:00 0 days 00:15:00\n",
      "4147  2021-02-13  16:30:00 0 days 00:15:00\n",
      "4148  2021-02-13  17:30:00 0 days 01:00:00\n",
      "4149  2021-02-13  17:45:00 0 days 00:15:00\n",
      "4150  2021-02-13  18:00:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "4949  2021-02-22  01:45:00 0 days 00:15:00\n",
      "4950  2021-02-22  02:00:00 0 days 00:15:00\n",
      "4951  2021-02-22  06:00:00 0 days 04:00:00\n",
      "4952  2021-02-22  06:15:00 0 days 00:15:00\n",
      "4953  2021-02-22  06:30:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "4979  2021-02-22  13:00:00 0 days 00:15:00\n",
      "4980  2021-02-22  13:15:00 0 days 00:15:00\n",
      "4981  2021-03-01  00:15:00 6 days 11:00:00\n",
      "4982  2021-03-01  00:30:00 0 days 00:15:00\n",
      "4983  2021-03-01  00:45:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "7215  2021-03-24  06:45:00 0 days 00:15:00\n",
      "7216  2021-03-24  07:00:00 0 days 00:15:00\n",
      "7217  2021-03-24  07:45:00 0 days 00:45:00\n",
      "7218  2021-03-24  08:00:00 0 days 00:15:00\n",
      "7219  2021-03-24  08:15:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "7287  2021-03-25  01:15:00 0 days 00:15:00\n",
      "7288  2021-03-25  01:30:00 0 days 00:15:00\n",
      "7289  2021-03-25  02:30:00 0 days 01:00:00\n",
      "7290  2021-03-25  02:45:00 0 days 00:15:00\n",
      "7291  2021-03-25  03:00:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "7389  2021-03-26  03:30:00 0 days 00:15:00\n",
      "7390  2021-03-26  03:45:00 0 days 00:15:00\n",
      "7391  2021-03-26  04:30:00 0 days 00:45:00\n",
      "7392  2021-03-26  04:45:00 0 days 00:15:00\n",
      "7393  2021-03-26  05:00:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "7462  2021-03-26  22:15:00 0 days 00:15:00\n",
      "7463  2021-03-26  22:30:00 0 days 00:15:00\n",
      "7464  2021-03-27  00:45:00 0 days 02:15:00\n",
      "7465  2021-03-27  01:00:00 0 days 00:15:00\n",
      "7466  2021-03-27  01:15:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "7707  2021-03-29  13:30:00 0 days 00:15:00\n",
      "7708  2021-03-29  13:45:00 0 days 00:15:00\n",
      "7709  2021-03-29  20:45:00 0 days 07:00:00\n",
      "7710  2021-03-29  21:00:00 0 days 00:15:00\n",
      "7711  2021-03-29  21:15:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "8236  2021-04-04  08:30:00 0 days 00:15:00\n",
      "8237  2021-04-04  08:45:00 0 days 00:15:00\n",
      "8238  2021-04-04  09:15:00 0 days 00:30:00\n",
      "8239  2021-04-04  09:30:00 0 days 00:15:00\n",
      "8240  2021-04-04  09:45:00 0 days 00:15:00\n",
      "            date      time           delta\n",
      "8865  2021-04-10  22:00:00 0 days 00:15:00\n",
      "8866  2021-04-10  22:15:00 0 days 00:15:00\n",
      "8867  2021-04-10  23:15:00 0 days 01:00:00\n",
      "8868  2021-04-10  23:30:00 0 days 00:15:00\n",
      "8869  2021-04-10  23:45:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "10692  2021-04-29  23:30:00 0 days 00:15:00\n",
      "10693  2021-04-29  23:45:00 0 days 00:15:00\n",
      "10694  2021-05-03  03:15:00 3 days 03:30:00\n",
      "10695  2021-05-03  03:30:00 0 days 00:15:00\n",
      "10696  2021-05-03  03:45:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "11720  2021-05-13  19:45:00 0 days 00:15:00\n",
      "11721  2021-05-13  20:00:00 0 days 00:15:00\n",
      "11722  2021-05-14  18:00:00 0 days 22:00:00\n",
      "11723  2021-05-14  18:15:00 0 days 00:15:00\n",
      "11724  2021-05-14  18:30:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "11900  2021-05-16  14:30:00 0 days 00:15:00\n",
      "11901  2021-05-16  14:45:00 0 days 00:15:00\n",
      "11902  2021-05-16  21:15:00 0 days 06:30:00\n",
      "11903  2021-05-16  21:30:00 0 days 00:15:00\n",
      "11904  2021-05-16  21:45:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "16412  2021-07-02  20:45:00 0 days 00:15:00\n",
      "16413  2021-07-02  21:00:00 0 days 00:15:00\n",
      "16414  2021-07-03  00:45:00 0 days 03:45:00\n",
      "16415  2021-07-03  01:00:00 0 days 00:15:00\n",
      "16416  2021-07-03  01:15:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "17487  2021-07-14  05:00:00 0 days 00:15:00\n",
      "17488  2021-07-14  05:15:00 0 days 00:15:00\n",
      "17489  2021-07-19  00:45:00 4 days 19:30:00\n",
      "17490  2021-07-19  01:00:00 0 days 00:15:00\n",
      "17491  2021-07-19  01:15:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "18625  2021-07-30  20:45:00 0 days 00:15:00\n",
      "18626  2021-07-30  21:00:00 0 days 00:15:00\n",
      "18627  2021-07-31  02:45:00 0 days 05:45:00\n",
      "18628  2021-07-31  03:00:00 0 days 00:15:00\n",
      "18629  2021-07-31  03:15:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "19215  2021-08-06  05:45:00 0 days 00:15:00\n",
      "19216  2021-08-06  06:00:00 0 days 00:15:00\n",
      "19217  2021-08-06  07:45:00 0 days 01:45:00\n",
      "19218  2021-08-06  08:00:00 0 days 00:15:00\n",
      "19219  2021-08-06  08:15:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "19481  2021-08-09  01:45:00 0 days 00:15:00\n",
      "19482  2021-08-09  02:00:00 0 days 00:15:00\n",
      "19483  2021-08-09  02:30:00 0 days 00:30:00\n",
      "19484  2021-08-09  02:45:00 0 days 00:15:00\n",
      "19485  2021-08-09  03:00:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "19523  2021-08-09  12:30:00 0 days 00:15:00\n",
      "19524  2021-08-09  12:45:00 0 days 00:15:00\n",
      "19525  2021-08-09  23:00:00 0 days 10:15:00\n",
      "19526  2021-08-09  23:15:00 0 days 00:15:00\n",
      "19527  2021-08-09  23:30:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "21474  2021-08-30  06:15:00 0 days 00:15:00\n",
      "21475  2021-08-30  06:30:00 0 days 00:15:00\n",
      "21476  2021-09-01  23:45:00 2 days 17:15:00\n",
      "21477  2021-09-02  00:00:00 0 days 00:15:00\n",
      "21478  2021-09-02  00:15:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "24960  2021-10-08  06:45:00 0 days 00:15:00\n",
      "24961  2021-10-08  07:00:00 0 days 00:15:00\n",
      "24962  2021-10-08  07:30:00 0 days 00:30:00\n",
      "24963  2021-10-08  08:00:00 0 days 00:30:00\n",
      "24964  2021-10-08  08:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24961  2021-10-08  07:00:00 0 days 00:15:00\n",
      "24962  2021-10-08  07:30:00 0 days 00:30:00\n",
      "24963  2021-10-08  08:00:00 0 days 00:30:00\n",
      "24964  2021-10-08  08:30:00 0 days 00:30:00\n",
      "24965  2021-10-08  09:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24962  2021-10-08  07:30:00 0 days 00:30:00\n",
      "24963  2021-10-08  08:00:00 0 days 00:30:00\n",
      "24964  2021-10-08  08:30:00 0 days 00:30:00\n",
      "24965  2021-10-08  09:00:00 0 days 00:30:00\n",
      "24966  2021-10-08  09:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24963  2021-10-08  08:00:00 0 days 00:30:00\n",
      "24964  2021-10-08  08:30:00 0 days 00:30:00\n",
      "24965  2021-10-08  09:00:00 0 days 00:30:00\n",
      "24966  2021-10-08  09:30:00 0 days 00:30:00\n",
      "24967  2021-10-08  10:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24964  2021-10-08  08:30:00 0 days 00:30:00\n",
      "24965  2021-10-08  09:00:00 0 days 00:30:00\n",
      "24966  2021-10-08  09:30:00 0 days 00:30:00\n",
      "24967  2021-10-08  10:00:00 0 days 00:30:00\n",
      "24968  2021-10-08  10:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24965  2021-10-08  09:00:00 0 days 00:30:00\n",
      "24966  2021-10-08  09:30:00 0 days 00:30:00\n",
      "24967  2021-10-08  10:00:00 0 days 00:30:00\n",
      "24968  2021-10-08  10:30:00 0 days 00:30:00\n",
      "24969  2021-10-08  11:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24966  2021-10-08  09:30:00 0 days 00:30:00\n",
      "24967  2021-10-08  10:00:00 0 days 00:30:00\n",
      "24968  2021-10-08  10:30:00 0 days 00:30:00\n",
      "24969  2021-10-08  11:00:00 0 days 00:30:00\n",
      "24970  2021-10-08  11:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24967  2021-10-08  10:00:00 0 days 00:30:00\n",
      "24968  2021-10-08  10:30:00 0 days 00:30:00\n",
      "24969  2021-10-08  11:00:00 0 days 00:30:00\n",
      "24970  2021-10-08  11:30:00 0 days 00:30:00\n",
      "24971  2021-10-08  12:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24968  2021-10-08  10:30:00 0 days 00:30:00\n",
      "24969  2021-10-08  11:00:00 0 days 00:30:00\n",
      "24970  2021-10-08  11:30:00 0 days 00:30:00\n",
      "24971  2021-10-08  12:00:00 0 days 00:30:00\n",
      "24972  2021-10-08  12:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24969  2021-10-08  11:00:00 0 days 00:30:00\n",
      "24970  2021-10-08  11:30:00 0 days 00:30:00\n",
      "24971  2021-10-08  12:00:00 0 days 00:30:00\n",
      "24972  2021-10-08  12:30:00 0 days 00:30:00\n",
      "24973  2021-10-08  13:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24970  2021-10-08  11:30:00 0 days 00:30:00\n",
      "24971  2021-10-08  12:00:00 0 days 00:30:00\n",
      "24972  2021-10-08  12:30:00 0 days 00:30:00\n",
      "24973  2021-10-08  13:00:00 0 days 00:30:00\n",
      "24974  2021-10-08  13:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24971  2021-10-08  12:00:00 0 days 00:30:00\n",
      "24972  2021-10-08  12:30:00 0 days 00:30:00\n",
      "24973  2021-10-08  13:00:00 0 days 00:30:00\n",
      "24974  2021-10-08  13:30:00 0 days 00:30:00\n",
      "24975  2021-10-08  14:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24972  2021-10-08  12:30:00 0 days 00:30:00\n",
      "24973  2021-10-08  13:00:00 0 days 00:30:00\n",
      "24974  2021-10-08  13:30:00 0 days 00:30:00\n",
      "24975  2021-10-08  14:00:00 0 days 00:30:00\n",
      "24976  2021-10-08  14:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24973  2021-10-08  13:00:00 0 days 00:30:00\n",
      "24974  2021-10-08  13:30:00 0 days 00:30:00\n",
      "24975  2021-10-08  14:00:00 0 days 00:30:00\n",
      "24976  2021-10-08  14:30:00 0 days 00:30:00\n",
      "24977  2021-10-08  15:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24974  2021-10-08  13:30:00 0 days 00:30:00\n",
      "24975  2021-10-08  14:00:00 0 days 00:30:00\n",
      "24976  2021-10-08  14:30:00 0 days 00:30:00\n",
      "24977  2021-10-08  15:00:00 0 days 00:30:00\n",
      "24978  2021-10-08  15:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24975  2021-10-08  14:00:00 0 days 00:30:00\n",
      "24976  2021-10-08  14:30:00 0 days 00:30:00\n",
      "24977  2021-10-08  15:00:00 0 days 00:30:00\n",
      "24978  2021-10-08  15:30:00 0 days 00:30:00\n",
      "24979  2021-10-08  16:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24976  2021-10-08  14:30:00 0 days 00:30:00\n",
      "24977  2021-10-08  15:00:00 0 days 00:30:00\n",
      "24978  2021-10-08  15:30:00 0 days 00:30:00\n",
      "24979  2021-10-08  16:00:00 0 days 00:30:00\n",
      "24980  2021-10-08  16:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24977  2021-10-08  15:00:00 0 days 00:30:00\n",
      "24978  2021-10-08  15:30:00 0 days 00:30:00\n",
      "24979  2021-10-08  16:00:00 0 days 00:30:00\n",
      "24980  2021-10-08  16:30:00 0 days 00:30:00\n",
      "24981  2021-10-08  17:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24978  2021-10-08  15:30:00 0 days 00:30:00\n",
      "24979  2021-10-08  16:00:00 0 days 00:30:00\n",
      "24980  2021-10-08  16:30:00 0 days 00:30:00\n",
      "24981  2021-10-08  17:00:00 0 days 00:30:00\n",
      "24982  2021-10-08  17:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24979  2021-10-08  16:00:00 0 days 00:30:00\n",
      "24980  2021-10-08  16:30:00 0 days 00:30:00\n",
      "24981  2021-10-08  17:00:00 0 days 00:30:00\n",
      "24982  2021-10-08  17:30:00 0 days 00:30:00\n",
      "24983  2021-10-08  18:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24980  2021-10-08  16:30:00 0 days 00:30:00\n",
      "24981  2021-10-08  17:00:00 0 days 00:30:00\n",
      "24982  2021-10-08  17:30:00 0 days 00:30:00\n",
      "24983  2021-10-08  18:00:00 0 days 00:30:00\n",
      "24984  2021-10-08  18:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24981  2021-10-08  17:00:00 0 days 00:30:00\n",
      "24982  2021-10-08  17:30:00 0 days 00:30:00\n",
      "24983  2021-10-08  18:00:00 0 days 00:30:00\n",
      "24984  2021-10-08  18:30:00 0 days 00:30:00\n",
      "24985  2021-10-08  19:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24982  2021-10-08  17:30:00 0 days 00:30:00\n",
      "24983  2021-10-08  18:00:00 0 days 00:30:00\n",
      "24984  2021-10-08  18:30:00 0 days 00:30:00\n",
      "24985  2021-10-08  19:00:00 0 days 00:30:00\n",
      "24986  2021-10-08  19:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24983  2021-10-08  18:00:00 0 days 00:30:00\n",
      "24984  2021-10-08  18:30:00 0 days 00:30:00\n",
      "24985  2021-10-08  19:00:00 0 days 00:30:00\n",
      "24986  2021-10-08  19:30:00 0 days 00:30:00\n",
      "24987  2021-10-08  20:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24984  2021-10-08  18:30:00 0 days 00:30:00\n",
      "24985  2021-10-08  19:00:00 0 days 00:30:00\n",
      "24986  2021-10-08  19:30:00 0 days 00:30:00\n",
      "24987  2021-10-08  20:00:00 0 days 00:30:00\n",
      "24988  2021-10-08  20:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24985  2021-10-08  19:00:00 0 days 00:30:00\n",
      "24986  2021-10-08  19:30:00 0 days 00:30:00\n",
      "24987  2021-10-08  20:00:00 0 days 00:30:00\n",
      "24988  2021-10-08  20:30:00 0 days 00:30:00\n",
      "24989  2021-10-08  21:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24986  2021-10-08  19:30:00 0 days 00:30:00\n",
      "24987  2021-10-08  20:00:00 0 days 00:30:00\n",
      "24988  2021-10-08  20:30:00 0 days 00:30:00\n",
      "24989  2021-10-08  21:00:00 0 days 00:30:00\n",
      "24990  2021-10-08  21:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24987  2021-10-08  20:00:00 0 days 00:30:00\n",
      "24988  2021-10-08  20:30:00 0 days 00:30:00\n",
      "24989  2021-10-08  21:00:00 0 days 00:30:00\n",
      "24990  2021-10-08  21:30:00 0 days 00:30:00\n",
      "24991  2021-10-08  22:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24988  2021-10-08  20:30:00 0 days 00:30:00\n",
      "24989  2021-10-08  21:00:00 0 days 00:30:00\n",
      "24990  2021-10-08  21:30:00 0 days 00:30:00\n",
      "24991  2021-10-08  22:00:00 0 days 00:30:00\n",
      "24992  2021-10-08  22:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24989  2021-10-08  21:00:00 0 days 00:30:00\n",
      "24990  2021-10-08  21:30:00 0 days 00:30:00\n",
      "24991  2021-10-08  22:00:00 0 days 00:30:00\n",
      "24992  2021-10-08  22:30:00 0 days 00:30:00\n",
      "24993  2021-10-08  23:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24990  2021-10-08  21:30:00 0 days 00:30:00\n",
      "24991  2021-10-08  22:00:00 0 days 00:30:00\n",
      "24992  2021-10-08  22:30:00 0 days 00:30:00\n",
      "24993  2021-10-08  23:00:00 0 days 00:30:00\n",
      "24994  2021-10-08  23:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24991  2021-10-08  22:00:00 0 days 00:30:00\n",
      "24992  2021-10-08  22:30:00 0 days 00:30:00\n",
      "24993  2021-10-08  23:00:00 0 days 00:30:00\n",
      "24994  2021-10-08  23:30:00 0 days 00:30:00\n",
      "24995  2021-10-09  00:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24992  2021-10-08  22:30:00 0 days 00:30:00\n",
      "24993  2021-10-08  23:00:00 0 days 00:30:00\n",
      "24994  2021-10-08  23:30:00 0 days 00:30:00\n",
      "24995  2021-10-09  00:00:00 0 days 00:30:00\n",
      "24996  2021-10-09  00:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24993  2021-10-08  23:00:00 0 days 00:30:00\n",
      "24994  2021-10-08  23:30:00 0 days 00:30:00\n",
      "24995  2021-10-09  00:00:00 0 days 00:30:00\n",
      "24996  2021-10-09  00:30:00 0 days 00:30:00\n",
      "24997  2021-10-09  01:00:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24994  2021-10-08  23:30:00 0 days 00:30:00\n",
      "24995  2021-10-09  00:00:00 0 days 00:30:00\n",
      "24996  2021-10-09  00:30:00 0 days 00:30:00\n",
      "24997  2021-10-09  01:00:00 0 days 00:30:00\n",
      "24998  2021-10-09  01:30:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "24995  2021-10-09  00:00:00 0 days 00:30:00\n",
      "24996  2021-10-09  00:30:00 0 days 00:30:00\n",
      "24997  2021-10-09  01:00:00 0 days 00:30:00\n",
      "24998  2021-10-09  01:30:00 0 days 00:30:00\n",
      "24999  2021-10-09  01:45:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "24996  2021-10-09  00:30:00 0 days 00:30:00\n",
      "24997  2021-10-09  01:00:00 0 days 00:30:00\n",
      "24998  2021-10-09  01:30:00 0 days 00:30:00\n",
      "24999  2021-10-09  01:45:00 0 days 00:15:00\n",
      "25000  2021-10-09  02:00:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "25865  2021-10-18  02:15:00 0 days 00:15:00\n",
      "25866  2021-10-18  02:30:00 0 days 00:15:00\n",
      "25867  2021-10-18  05:45:00 0 days 03:15:00\n",
      "25868  2021-10-18  06:00:00 0 days 00:15:00\n",
      "25869  2021-10-18  06:15:00 0 days 00:15:00\n",
      "Outliers detected in train_public/cleaned/L09.B01_15min_cleaned.csv:\n",
      "             date      time  energy_consumption(kW)   z_score\n",
      "377    2021-01-04  22:15:00              331.000000  3.015645\n",
      "378    2021-01-04  22:30:00              343.666667  3.210088\n",
      "380    2021-01-04  23:00:00              342.666667  3.194737\n",
      "381    2021-01-04  23:15:00              340.666667  3.164036\n",
      "382    2021-01-04  23:30:00              336.666667  3.102633\n",
      "...           ...       ...                     ...       ...\n",
      "32272  2021-12-23  23:00:00              386.333333  3.865052\n",
      "32276  2021-12-24  00:00:00              330.333333  3.005411\n",
      "32278  2021-12-24  00:30:00              357.000000  3.414764\n",
      "32281  2021-12-24  01:15:00              335.666667  3.087282\n",
      "32682  2021-12-28  05:30:00              -78.666667  3.273033\n",
      "\n",
      "[814 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L10.B01_15min_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2020-01-01  00:00:00             NaT\n",
      "1  2020-01-01  00:15:00 0 days 00:15:00\n",
      "2  2020-01-01  00:30:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "15985  2020-06-15  12:15:00 0 days 00:15:00\n",
      "15986  2020-06-15  12:30:00 0 days 00:15:00\n",
      "15987  2020-06-15  13:30:00 0 days 01:00:00\n",
      "15988  2020-06-15  13:45:00 0 days 00:15:00\n",
      "15989  2020-06-15  14:00:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "16731  2020-06-23  07:30:00 0 days 00:15:00\n",
      "16732  2020-06-23  07:45:00 0 days 00:15:00\n",
      "16733  2020-06-23  08:15:00 0 days 00:30:00\n",
      "16734  2020-06-23  08:30:00 0 days 00:15:00\n",
      "16735  2020-06-23  08:45:00 0 days 00:15:00\n",
      "             date      time           delta\n",
      "16796  2020-06-24  00:00:00 0 days 00:15:00\n",
      "16797  2020-06-24  00:15:00 0 days 00:15:00\n",
      "16798  2020-06-24  06:30:00 0 days 06:15:00\n",
      "16799  2020-06-24  06:45:00 0 days 00:15:00\n",
      "16800  2020-06-24  07:00:00 0 days 00:15:00\n",
      "Outliers detected in train_public/cleaned/L10.B01_15min_cleaned.csv:\n",
      "             date      time  energy_consumption(kW)   z_score\n",
      "15588  2020-06-11  09:00:00              523.828568  3.091454\n",
      "15589  2020-06-11  09:15:00              555.635719  3.399741\n",
      "15590  2020-06-11  09:30:00              561.184525  3.453522\n",
      "15591  2020-06-11  09:45:00              568.625000  3.525638\n",
      "15592  2020-06-11  10:00:00              574.460716  3.582200\n",
      "...           ...       ...                     ...       ...\n",
      "54926  2021-07-26  10:30:00              594.516670  3.776590\n",
      "54927  2021-07-26  10:45:00              586.207153  3.696051\n",
      "54928  2021-07-26  11:00:00              603.250000  3.861237\n",
      "54929  2021-07-26  11:15:00              580.916672  3.644774\n",
      "54930  2021-07-26  11:30:00              536.971420  3.218839\n",
      "\n",
      "[954 rows x 4 columns]\n",
      "Irregular timestamps found in train_public/cleaned/L09.B01_5min_cleaned.csv:\n",
      "         date      time           delta\n",
      "0  2021-01-01  00:00:00             NaT\n",
      "1  2021-01-01  00:05:00 0 days 00:05:00\n",
      "2  2021-01-01  00:10:00 0 days 00:05:00\n",
      "            date      time           delta\n",
      "2153  2021-01-08  11:25:00 0 days 00:05:00\n",
      "2154  2021-01-08  11:30:00 0 days 00:05:00\n",
      "2155  2021-01-08  22:40:00 0 days 11:10:00\n",
      "2156  2021-01-08  22:45:00 0 days 00:05:00\n",
      "2157  2021-01-08  22:50:00 0 days 00:05:00\n",
      "            date      time           delta\n",
      "8205  2021-01-29  22:50:00 0 days 00:05:00\n",
      "8206  2021-01-29  22:55:00 0 days 00:05:00\n",
      "8207  2021-01-29  23:10:00 0 days 00:15:00\n",
      "8208  2021-01-29  23:25:00 0 days 00:15:00\n",
      "8209  2021-01-29  23:30:00 0 days 00:05:00\n",
      "            date      time           delta\n",
      "8206  2021-01-29  22:55:00 0 days 00:05:00\n",
      "8207  2021-01-29  23:10:00 0 days 00:15:00\n",
      "8208  2021-01-29  23:25:00 0 days 00:15:00\n",
      "8209  2021-01-29  23:30:00 0 days 00:05:00\n",
      "8210  2021-01-29  23:35:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "10625  2021-02-07  08:50:00 0 days 00:05:00\n",
      "10626  2021-02-07  08:55:00 0 days 00:05:00\n",
      "10627  2021-02-07  10:00:00 0 days 01:05:00\n",
      "10628  2021-02-07  10:05:00 0 days 00:05:00\n",
      "10629  2021-02-07  10:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "11143  2021-02-09  05:00:00 0 days 00:05:00\n",
      "11144  2021-02-09  05:05:00 0 days 00:05:00\n",
      "11145  2021-02-09  05:15:00 0 days 00:10:00\n",
      "11146  2021-02-09  05:20:00 0 days 00:05:00\n",
      "11147  2021-02-09  05:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "12433  2021-02-13  16:35:00 0 days 00:05:00\n",
      "12434  2021-02-13  16:40:00 0 days 00:05:00\n",
      "12435  2021-02-13  17:30:00 0 days 00:50:00\n",
      "12436  2021-02-13  17:35:00 0 days 00:05:00\n",
      "12437  2021-02-13  17:40:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "13334  2021-02-16  20:25:00 0 days 00:05:00\n",
      "13335  2021-02-16  20:30:00 0 days 00:05:00\n",
      "13336  2021-02-16  20:40:00 0 days 00:10:00\n",
      "13337  2021-02-16  20:45:00 0 days 00:05:00\n",
      "13338  2021-02-16  20:50:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "14839  2021-02-22  01:55:00 0 days 00:05:00\n",
      "14840  2021-02-22  02:00:00 0 days 00:05:00\n",
      "14841  2021-02-22  06:05:00 0 days 04:05:00\n",
      "14842  2021-02-22  06:10:00 0 days 00:05:00\n",
      "14843  2021-02-22  06:15:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "14928  2021-02-22  13:20:00 0 days 00:05:00\n",
      "14929  2021-02-22  13:25:00 0 days 00:05:00\n",
      "14930  2021-03-01  00:20:00 6 days 10:55:00\n",
      "14931  2021-03-01  00:25:00 0 days 00:05:00\n",
      "14932  2021-03-01  00:30:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "15568  2021-03-03  05:30:00 0 days 00:05:00\n",
      "15569  2021-03-03  05:35:00 0 days 00:05:00\n",
      "15570  2021-03-03  05:45:00 0 days 00:10:00\n",
      "15571  2021-03-03  05:50:00 0 days 00:05:00\n",
      "15572  2021-03-03  05:55:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "16859  2021-03-07  17:10:00 0 days 00:05:00\n",
      "16860  2021-03-07  17:15:00 0 days 00:05:00\n",
      "16861  2021-03-07  17:25:00 0 days 00:10:00\n",
      "16862  2021-03-07  17:30:00 0 days 00:05:00\n",
      "16863  2021-03-07  17:35:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "17021  2021-03-08  06:45:00 0 days 00:05:00\n",
      "17022  2021-03-08  06:50:00 0 days 00:05:00\n",
      "17023  2021-03-08  07:00:00 0 days 00:10:00\n",
      "17024  2021-03-08  07:05:00 0 days 00:05:00\n",
      "17025  2021-03-08  07:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "21632  2021-03-24  07:05:00 0 days 00:05:00\n",
      "21633  2021-03-24  07:10:00 0 days 00:05:00\n",
      "21634  2021-03-24  07:55:00 0 days 00:45:00\n",
      "21635  2021-03-24  08:00:00 0 days 00:05:00\n",
      "21636  2021-03-24  08:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "21846  2021-03-25  01:35:00 0 days 00:05:00\n",
      "21847  2021-03-25  01:40:00 0 days 00:05:00\n",
      "21848  2021-03-25  02:35:00 0 days 00:55:00\n",
      "21849  2021-03-25  02:40:00 0 days 00:05:00\n",
      "21850  2021-03-25  02:45:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "22151  2021-03-26  03:50:00 0 days 00:05:00\n",
      "22152  2021-03-26  03:55:00 0 days 00:05:00\n",
      "22153  2021-03-26  04:40:00 0 days 00:45:00\n",
      "22154  2021-03-26  04:45:00 0 days 00:05:00\n",
      "22155  2021-03-26  04:50:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "22367  2021-03-26  22:30:00 0 days 00:05:00\n",
      "22368  2021-03-26  22:35:00 0 days 00:05:00\n",
      "22369  2021-03-27  00:55:00 0 days 02:20:00\n",
      "22370  2021-03-27  01:00:00 0 days 00:05:00\n",
      "22371  2021-03-27  01:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "23100  2021-03-29  13:50:00 0 days 00:05:00\n",
      "23101  2021-03-29  13:55:00 0 days 00:05:00\n",
      "23102  2021-03-29  20:50:00 0 days 06:55:00\n",
      "23103  2021-03-29  20:55:00 0 days 00:05:00\n",
      "23104  2021-03-29  21:00:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "23137  2021-03-29  23:45:00 0 days 00:05:00\n",
      "23138  2021-03-29  23:50:00 0 days 00:05:00\n",
      "23139  2021-03-30  00:00:00 0 days 00:10:00\n",
      "23140  2021-03-30  00:05:00 0 days 00:05:00\n",
      "23141  2021-03-30  00:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "24685  2021-04-04  08:50:00 0 days 00:05:00\n",
      "24686  2021-04-04  08:55:00 0 days 00:05:00\n",
      "24687  2021-04-04  09:20:00 0 days 00:25:00\n",
      "24688  2021-04-04  09:25:00 0 days 00:05:00\n",
      "24689  2021-04-04  09:30:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "26571  2021-04-10  22:20:00 0 days 00:05:00\n",
      "26572  2021-04-10  22:25:00 0 days 00:05:00\n",
      "26573  2021-04-10  23:15:00 0 days 00:50:00\n",
      "26574  2021-04-10  23:35:00 0 days 00:20:00\n",
      "26575  2021-04-10  23:40:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "26572  2021-04-10  22:25:00 0 days 00:05:00\n",
      "26573  2021-04-10  23:15:00 0 days 00:50:00\n",
      "26574  2021-04-10  23:35:00 0 days 00:20:00\n",
      "26575  2021-04-10  23:40:00 0 days 00:05:00\n",
      "26576  2021-04-10  23:50:00 0 days 00:10:00\n",
      "             date      time           delta\n",
      "26574  2021-04-10  23:35:00 0 days 00:20:00\n",
      "26575  2021-04-10  23:40:00 0 days 00:05:00\n",
      "26576  2021-04-10  23:50:00 0 days 00:10:00\n",
      "26577  2021-04-10  23:55:00 0 days 00:05:00\n",
      "26578  2021-04-11  00:00:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27201  2021-04-13  03:55:00 0 days 00:05:00\n",
      "27202  2021-04-13  04:00:00 0 days 00:05:00\n",
      "27203  2021-04-13  04:25:00 0 days 00:25:00\n",
      "27204  2021-04-13  04:30:00 0 days 00:05:00\n",
      "27205  2021-04-13  04:35:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27246  2021-04-13  08:00:00 0 days 00:05:00\n",
      "27247  2021-04-13  08:05:00 0 days 00:05:00\n",
      "27248  2021-04-13  08:15:00 0 days 00:10:00\n",
      "27249  2021-04-13  08:20:00 0 days 00:05:00\n",
      "27250  2021-04-13  08:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27341  2021-04-13  16:00:00 0 days 00:05:00\n",
      "27342  2021-04-13  16:05:00 0 days 00:05:00\n",
      "27343  2021-04-13  16:15:00 0 days 00:10:00\n",
      "27344  2021-04-13  16:20:00 0 days 00:05:00\n",
      "27345  2021-04-13  16:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27364  2021-04-13  18:00:00 0 days 00:05:00\n",
      "27365  2021-04-13  18:05:00 0 days 00:05:00\n",
      "27366  2021-04-13  18:20:00 0 days 00:15:00\n",
      "27367  2021-04-13  18:25:00 0 days 00:05:00\n",
      "27368  2021-04-13  18:30:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27410  2021-04-13  22:00:00 0 days 00:05:00\n",
      "27411  2021-04-13  22:05:00 0 days 00:05:00\n",
      "27412  2021-04-13  22:15:00 0 days 00:10:00\n",
      "27413  2021-04-13  22:20:00 0 days 00:05:00\n",
      "27414  2021-04-13  22:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27841  2021-04-15  10:00:00 0 days 00:05:00\n",
      "27842  2021-04-15  10:05:00 0 days 00:05:00\n",
      "27843  2021-04-15  10:15:00 0 days 00:10:00\n",
      "27844  2021-04-15  10:20:00 0 days 00:05:00\n",
      "27845  2021-04-15  10:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "27990  2021-04-15  22:30:00 0 days 00:05:00\n",
      "27991  2021-04-15  22:35:00 0 days 00:05:00\n",
      "27992  2021-04-15  22:55:00 0 days 00:20:00\n",
      "27993  2021-04-15  23:00:00 0 days 00:05:00\n",
      "27994  2021-04-15  23:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "32034  2021-04-29  23:45:00 0 days 00:05:00\n",
      "32035  2021-04-29  23:50:00 0 days 00:05:00\n",
      "32036  2021-05-03  03:15:00 3 days 03:25:00\n",
      "32037  2021-05-03  03:20:00 0 days 00:05:00\n",
      "32038  2021-05-03  03:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "33701  2021-05-08  22:00:00 0 days 00:05:00\n",
      "33702  2021-05-08  22:05:00 0 days 00:05:00\n",
      "33703  2021-05-08  22:15:00 0 days 00:10:00\n",
      "33704  2021-05-08  22:20:00 0 days 00:05:00\n",
      "33705  2021-05-08  22:30:00 0 days 00:10:00\n",
      "             date      time           delta\n",
      "33703  2021-05-08  22:15:00 0 days 00:10:00\n",
      "33704  2021-05-08  22:20:00 0 days 00:05:00\n",
      "33705  2021-05-08  22:30:00 0 days 00:10:00\n",
      "33706  2021-05-08  22:35:00 0 days 00:05:00\n",
      "33707  2021-05-08  22:40:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "35114  2021-05-13  19:55:00 0 days 00:05:00\n",
      "35115  2021-05-13  20:00:00 0 days 00:05:00\n",
      "35116  2021-05-14  18:00:00 0 days 22:00:00\n",
      "35117  2021-05-14  18:05:00 0 days 00:05:00\n",
      "35118  2021-05-14  18:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "35652  2021-05-16  14:40:00 0 days 00:05:00\n",
      "35653  2021-05-16  14:45:00 0 days 00:05:00\n",
      "35654  2021-05-16  21:25:00 0 days 06:40:00\n",
      "35655  2021-05-16  21:30:00 0 days 00:05:00\n",
      "35656  2021-05-16  21:35:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "36886  2021-05-21  04:05:00 0 days 00:05:00\n",
      "36887  2021-05-21  04:10:00 0 days 00:05:00\n",
      "36888  2021-05-21  04:20:00 0 days 00:10:00\n",
      "36889  2021-05-21  04:25:00 0 days 00:05:00\n",
      "36890  2021-05-21  04:30:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "43993  2021-06-14  20:25:00 0 days 00:05:00\n",
      "43994  2021-06-14  20:30:00 0 days 00:05:00\n",
      "43995  2021-06-14  20:40:00 0 days 00:10:00\n",
      "43996  2021-06-14  20:45:00 0 days 00:05:00\n",
      "43997  2021-06-14  20:50:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "49184  2021-07-02  21:05:00 0 days 00:05:00\n",
      "49185  2021-07-02  21:10:00 0 days 00:05:00\n",
      "49186  2021-07-03  00:55:00 0 days 03:45:00\n",
      "49187  2021-07-03  01:00:00 0 days 00:05:00\n",
      "49188  2021-07-03  01:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "50339  2021-07-07  01:00:00 0 days 00:05:00\n",
      "50340  2021-07-07  01:05:00 0 days 00:05:00\n",
      "50341  2021-07-07  01:15:00 0 days 00:10:00\n",
      "50342  2021-07-07  01:20:00 0 days 00:05:00\n",
      "50343  2021-07-07  01:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "52404  2021-07-14  05:10:00 0 days 00:05:00\n",
      "52405  2021-07-14  05:15:00 0 days 00:05:00\n",
      "52406  2021-07-19  00:55:00 4 days 19:40:00\n",
      "52407  2021-07-19  01:00:00 0 days 00:05:00\n",
      "52408  2021-07-19  01:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "55816  2021-07-30  21:05:00 0 days 00:05:00\n",
      "55817  2021-07-30  21:10:00 0 days 00:05:00\n",
      "55818  2021-07-31  02:50:00 0 days 05:40:00\n",
      "55819  2021-07-31  02:55:00 0 days 00:05:00\n",
      "55820  2021-07-31  03:00:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "57416  2021-08-05  16:00:00 0 days 00:05:00\n",
      "57417  2021-08-05  16:05:00 0 days 00:05:00\n",
      "57418  2021-08-05  16:15:00 0 days 00:10:00\n",
      "57419  2021-08-05  16:20:00 0 days 00:05:00\n",
      "57420  2021-08-05  16:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "57583  2021-08-06  06:00:00 0 days 00:05:00\n",
      "57584  2021-08-06  06:05:00 0 days 00:05:00\n",
      "57585  2021-08-06  07:45:00 0 days 01:40:00\n",
      "57586  2021-08-06  07:50:00 0 days 00:05:00\n",
      "57587  2021-08-06  07:55:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "58381  2021-08-09  02:05:00 0 days 00:05:00\n",
      "58382  2021-08-09  02:10:00 0 days 00:05:00\n",
      "58383  2021-08-09  02:35:00 0 days 00:25:00\n",
      "58384  2021-08-09  02:40:00 0 days 00:05:00\n",
      "58385  2021-08-09  02:45:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "58396  2021-08-09  03:40:00 0 days 00:05:00\n",
      "58397  2021-08-09  03:45:00 0 days 00:05:00\n",
      "58398  2021-08-09  04:00:00 0 days 00:15:00\n",
      "58399  2021-08-09  04:05:00 0 days 00:05:00\n",
      "58400  2021-08-09  04:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "58503  2021-08-09  12:45:00 0 days 00:05:00\n",
      "58504  2021-08-09  12:50:00 0 days 00:05:00\n",
      "58505  2021-08-09  23:00:00 0 days 10:10:00\n",
      "58506  2021-08-09  23:05:00 0 days 00:05:00\n",
      "58507  2021-08-09  23:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "64356  2021-08-30  06:35:00 0 days 00:05:00\n",
      "64357  2021-08-30  06:40:00 0 days 00:05:00\n",
      "64358  2021-09-01  23:55:00 2 days 17:15:00\n",
      "64359  2021-09-02  00:00:00 0 days 00:05:00\n",
      "64360  2021-09-02  00:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "74811  2021-10-08  07:00:00 0 days 00:05:00\n",
      "74812  2021-10-08  07:05:00 0 days 00:05:00\n",
      "74813  2021-10-08  07:35:00 0 days 00:30:00\n",
      "74814  2021-10-08  08:05:00 0 days 00:30:00\n",
      "74815  2021-10-08  08:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74812  2021-10-08  07:05:00 0 days 00:05:00\n",
      "74813  2021-10-08  07:35:00 0 days 00:30:00\n",
      "74814  2021-10-08  08:05:00 0 days 00:30:00\n",
      "74815  2021-10-08  08:35:00 0 days 00:30:00\n",
      "74816  2021-10-08  09:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74813  2021-10-08  07:35:00 0 days 00:30:00\n",
      "74814  2021-10-08  08:05:00 0 days 00:30:00\n",
      "74815  2021-10-08  08:35:00 0 days 00:30:00\n",
      "74816  2021-10-08  09:05:00 0 days 00:30:00\n",
      "74817  2021-10-08  09:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74814  2021-10-08  08:05:00 0 days 00:30:00\n",
      "74815  2021-10-08  08:35:00 0 days 00:30:00\n",
      "74816  2021-10-08  09:05:00 0 days 00:30:00\n",
      "74817  2021-10-08  09:35:00 0 days 00:30:00\n",
      "74818  2021-10-08  10:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74815  2021-10-08  08:35:00 0 days 00:30:00\n",
      "74816  2021-10-08  09:05:00 0 days 00:30:00\n",
      "74817  2021-10-08  09:35:00 0 days 00:30:00\n",
      "74818  2021-10-08  10:05:00 0 days 00:30:00\n",
      "74819  2021-10-08  10:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74816  2021-10-08  09:05:00 0 days 00:30:00\n",
      "74817  2021-10-08  09:35:00 0 days 00:30:00\n",
      "74818  2021-10-08  10:05:00 0 days 00:30:00\n",
      "74819  2021-10-08  10:35:00 0 days 00:30:00\n",
      "74820  2021-10-08  11:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74817  2021-10-08  09:35:00 0 days 00:30:00\n",
      "74818  2021-10-08  10:05:00 0 days 00:30:00\n",
      "74819  2021-10-08  10:35:00 0 days 00:30:00\n",
      "74820  2021-10-08  11:05:00 0 days 00:30:00\n",
      "74821  2021-10-08  11:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74818  2021-10-08  10:05:00 0 days 00:30:00\n",
      "74819  2021-10-08  10:35:00 0 days 00:30:00\n",
      "74820  2021-10-08  11:05:00 0 days 00:30:00\n",
      "74821  2021-10-08  11:35:00 0 days 00:30:00\n",
      "74822  2021-10-08  12:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74819  2021-10-08  10:35:00 0 days 00:30:00\n",
      "74820  2021-10-08  11:05:00 0 days 00:30:00\n",
      "74821  2021-10-08  11:35:00 0 days 00:30:00\n",
      "74822  2021-10-08  12:05:00 0 days 00:30:00\n",
      "74823  2021-10-08  12:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74820  2021-10-08  11:05:00 0 days 00:30:00\n",
      "74821  2021-10-08  11:35:00 0 days 00:30:00\n",
      "74822  2021-10-08  12:05:00 0 days 00:30:00\n",
      "74823  2021-10-08  12:35:00 0 days 00:30:00\n",
      "74824  2021-10-08  13:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74821  2021-10-08  11:35:00 0 days 00:30:00\n",
      "74822  2021-10-08  12:05:00 0 days 00:30:00\n",
      "74823  2021-10-08  12:35:00 0 days 00:30:00\n",
      "74824  2021-10-08  13:05:00 0 days 00:30:00\n",
      "74825  2021-10-08  13:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74822  2021-10-08  12:05:00 0 days 00:30:00\n",
      "74823  2021-10-08  12:35:00 0 days 00:30:00\n",
      "74824  2021-10-08  13:05:00 0 days 00:30:00\n",
      "74825  2021-10-08  13:35:00 0 days 00:30:00\n",
      "74826  2021-10-08  14:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74823  2021-10-08  12:35:00 0 days 00:30:00\n",
      "74824  2021-10-08  13:05:00 0 days 00:30:00\n",
      "74825  2021-10-08  13:35:00 0 days 00:30:00\n",
      "74826  2021-10-08  14:05:00 0 days 00:30:00\n",
      "74827  2021-10-08  14:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74824  2021-10-08  13:05:00 0 days 00:30:00\n",
      "74825  2021-10-08  13:35:00 0 days 00:30:00\n",
      "74826  2021-10-08  14:05:00 0 days 00:30:00\n",
      "74827  2021-10-08  14:35:00 0 days 00:30:00\n",
      "74828  2021-10-08  15:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74825  2021-10-08  13:35:00 0 days 00:30:00\n",
      "74826  2021-10-08  14:05:00 0 days 00:30:00\n",
      "74827  2021-10-08  14:35:00 0 days 00:30:00\n",
      "74828  2021-10-08  15:05:00 0 days 00:30:00\n",
      "74829  2021-10-08  15:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74826  2021-10-08  14:05:00 0 days 00:30:00\n",
      "74827  2021-10-08  14:35:00 0 days 00:30:00\n",
      "74828  2021-10-08  15:05:00 0 days 00:30:00\n",
      "74829  2021-10-08  15:35:00 0 days 00:30:00\n",
      "74830  2021-10-08  16:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74827  2021-10-08  14:35:00 0 days 00:30:00\n",
      "74828  2021-10-08  15:05:00 0 days 00:30:00\n",
      "74829  2021-10-08  15:35:00 0 days 00:30:00\n",
      "74830  2021-10-08  16:05:00 0 days 00:30:00\n",
      "74831  2021-10-08  16:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74828  2021-10-08  15:05:00 0 days 00:30:00\n",
      "74829  2021-10-08  15:35:00 0 days 00:30:00\n",
      "74830  2021-10-08  16:05:00 0 days 00:30:00\n",
      "74831  2021-10-08  16:35:00 0 days 00:30:00\n",
      "74832  2021-10-08  17:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74829  2021-10-08  15:35:00 0 days 00:30:00\n",
      "74830  2021-10-08  16:05:00 0 days 00:30:00\n",
      "74831  2021-10-08  16:35:00 0 days 00:30:00\n",
      "74832  2021-10-08  17:05:00 0 days 00:30:00\n",
      "74833  2021-10-08  17:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74830  2021-10-08  16:05:00 0 days 00:30:00\n",
      "74831  2021-10-08  16:35:00 0 days 00:30:00\n",
      "74832  2021-10-08  17:05:00 0 days 00:30:00\n",
      "74833  2021-10-08  17:35:00 0 days 00:30:00\n",
      "74834  2021-10-08  18:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74831  2021-10-08  16:35:00 0 days 00:30:00\n",
      "74832  2021-10-08  17:05:00 0 days 00:30:00\n",
      "74833  2021-10-08  17:35:00 0 days 00:30:00\n",
      "74834  2021-10-08  18:05:00 0 days 00:30:00\n",
      "74835  2021-10-08  18:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74832  2021-10-08  17:05:00 0 days 00:30:00\n",
      "74833  2021-10-08  17:35:00 0 days 00:30:00\n",
      "74834  2021-10-08  18:05:00 0 days 00:30:00\n",
      "74835  2021-10-08  18:35:00 0 days 00:30:00\n",
      "74836  2021-10-08  19:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74833  2021-10-08  17:35:00 0 days 00:30:00\n",
      "74834  2021-10-08  18:05:00 0 days 00:30:00\n",
      "74835  2021-10-08  18:35:00 0 days 00:30:00\n",
      "74836  2021-10-08  19:05:00 0 days 00:30:00\n",
      "74837  2021-10-08  19:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74834  2021-10-08  18:05:00 0 days 00:30:00\n",
      "74835  2021-10-08  18:35:00 0 days 00:30:00\n",
      "74836  2021-10-08  19:05:00 0 days 00:30:00\n",
      "74837  2021-10-08  19:35:00 0 days 00:30:00\n",
      "74838  2021-10-08  20:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74835  2021-10-08  18:35:00 0 days 00:30:00\n",
      "74836  2021-10-08  19:05:00 0 days 00:30:00\n",
      "74837  2021-10-08  19:35:00 0 days 00:30:00\n",
      "74838  2021-10-08  20:05:00 0 days 00:30:00\n",
      "74839  2021-10-08  20:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74836  2021-10-08  19:05:00 0 days 00:30:00\n",
      "74837  2021-10-08  19:35:00 0 days 00:30:00\n",
      "74838  2021-10-08  20:05:00 0 days 00:30:00\n",
      "74839  2021-10-08  20:35:00 0 days 00:30:00\n",
      "74840  2021-10-08  21:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74837  2021-10-08  19:35:00 0 days 00:30:00\n",
      "74838  2021-10-08  20:05:00 0 days 00:30:00\n",
      "74839  2021-10-08  20:35:00 0 days 00:30:00\n",
      "74840  2021-10-08  21:05:00 0 days 00:30:00\n",
      "74841  2021-10-08  21:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74838  2021-10-08  20:05:00 0 days 00:30:00\n",
      "74839  2021-10-08  20:35:00 0 days 00:30:00\n",
      "74840  2021-10-08  21:05:00 0 days 00:30:00\n",
      "74841  2021-10-08  21:35:00 0 days 00:30:00\n",
      "74842  2021-10-08  22:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74839  2021-10-08  20:35:00 0 days 00:30:00\n",
      "74840  2021-10-08  21:05:00 0 days 00:30:00\n",
      "74841  2021-10-08  21:35:00 0 days 00:30:00\n",
      "74842  2021-10-08  22:05:00 0 days 00:30:00\n",
      "74843  2021-10-08  22:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74840  2021-10-08  21:05:00 0 days 00:30:00\n",
      "74841  2021-10-08  21:35:00 0 days 00:30:00\n",
      "74842  2021-10-08  22:05:00 0 days 00:30:00\n",
      "74843  2021-10-08  22:35:00 0 days 00:30:00\n",
      "74844  2021-10-08  23:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74841  2021-10-08  21:35:00 0 days 00:30:00\n",
      "74842  2021-10-08  22:05:00 0 days 00:30:00\n",
      "74843  2021-10-08  22:35:00 0 days 00:30:00\n",
      "74844  2021-10-08  23:05:00 0 days 00:30:00\n",
      "74845  2021-10-08  23:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74842  2021-10-08  22:05:00 0 days 00:30:00\n",
      "74843  2021-10-08  22:35:00 0 days 00:30:00\n",
      "74844  2021-10-08  23:05:00 0 days 00:30:00\n",
      "74845  2021-10-08  23:35:00 0 days 00:30:00\n",
      "74846  2021-10-09  00:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74843  2021-10-08  22:35:00 0 days 00:30:00\n",
      "74844  2021-10-08  23:05:00 0 days 00:30:00\n",
      "74845  2021-10-08  23:35:00 0 days 00:30:00\n",
      "74846  2021-10-09  00:05:00 0 days 00:30:00\n",
      "74847  2021-10-09  00:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74844  2021-10-08  23:05:00 0 days 00:30:00\n",
      "74845  2021-10-08  23:35:00 0 days 00:30:00\n",
      "74846  2021-10-09  00:05:00 0 days 00:30:00\n",
      "74847  2021-10-09  00:35:00 0 days 00:30:00\n",
      "74848  2021-10-09  01:05:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74845  2021-10-08  23:35:00 0 days 00:30:00\n",
      "74846  2021-10-09  00:05:00 0 days 00:30:00\n",
      "74847  2021-10-09  00:35:00 0 days 00:30:00\n",
      "74848  2021-10-09  01:05:00 0 days 00:30:00\n",
      "74849  2021-10-09  01:35:00 0 days 00:30:00\n",
      "             date      time           delta\n",
      "74846  2021-10-09  00:05:00 0 days 00:30:00\n",
      "74847  2021-10-09  00:35:00 0 days 00:30:00\n",
      "74848  2021-10-09  01:05:00 0 days 00:30:00\n",
      "74849  2021-10-09  01:35:00 0 days 00:30:00\n",
      "74850  2021-10-09  01:45:00 0 days 00:10:00\n",
      "             date      time           delta\n",
      "74847  2021-10-09  00:35:00 0 days 00:30:00\n",
      "74848  2021-10-09  01:05:00 0 days 00:30:00\n",
      "74849  2021-10-09  01:35:00 0 days 00:30:00\n",
      "74850  2021-10-09  01:45:00 0 days 00:10:00\n",
      "74851  2021-10-09  01:50:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "74848  2021-10-09  01:05:00 0 days 00:30:00\n",
      "74849  2021-10-09  01:35:00 0 days 00:30:00\n",
      "74850  2021-10-09  01:45:00 0 days 00:10:00\n",
      "74851  2021-10-09  01:50:00 0 days 00:05:00\n",
      "74852  2021-10-09  01:55:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "75753  2021-10-12  05:00:00 0 days 00:05:00\n",
      "75754  2021-10-12  05:05:00 0 days 00:05:00\n",
      "75755  2021-10-12  05:15:00 0 days 00:10:00\n",
      "75756  2021-10-12  05:20:00 0 days 00:05:00\n",
      "75757  2021-10-12  05:25:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "77449  2021-10-18  02:25:00 0 days 00:05:00\n",
      "77450  2021-10-18  02:30:00 0 days 00:05:00\n",
      "77451  2021-10-18  05:50:00 0 days 03:20:00\n",
      "77452  2021-10-18  05:55:00 0 days 00:05:00\n",
      "77453  2021-10-18  06:00:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "81721  2021-11-02  01:40:00 0 days 00:05:00\n",
      "81722  2021-11-02  01:45:00 0 days 00:05:00\n",
      "81723  2021-11-02  01:55:00 0 days 00:10:00\n",
      "81724  2021-11-02  02:00:00 0 days 00:05:00\n",
      "81725  2021-11-02  02:05:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "90253  2021-12-01  16:45:00 0 days 00:05:00\n",
      "90254  2021-12-01  16:50:00 0 days 00:05:00\n",
      "90255  2021-12-01  17:00:00 0 days 00:10:00\n",
      "90256  2021-12-01  17:05:00 0 days 00:05:00\n",
      "90257  2021-12-01  17:10:00 0 days 00:05:00\n",
      "             date      time           delta\n",
      "92699  2021-12-10  04:40:00 0 days 00:05:00\n",
      "92700  2021-12-10  04:45:00 0 days 00:05:00\n",
      "92701  2021-12-10  04:55:00 0 days 00:10:00\n",
      "92702  2021-12-10  05:00:00 0 days 00:05:00\n",
      "92703  2021-12-10  05:05:00 0 days 00:05:00\n",
      "Outliers detected in train_public/cleaned/L09.B01_5min_cleaned.csv:\n",
      "             date      time  energy_consumption(kW)   z_score\n",
      "1130   2021-01-04  22:10:00                   344.0  3.122408\n",
      "1131   2021-01-04  22:15:00                   339.0  3.049274\n",
      "1134   2021-01-04  22:30:00                   341.0  3.078528\n",
      "1136   2021-01-04  22:40:00                   361.0  3.371063\n",
      "1141   2021-01-04  23:05:00                   369.0  3.488077\n",
      "...           ...       ...                     ...       ...\n",
      "97894  2021-12-28  05:40:00                   -92.0  3.254859\n",
      "98400  2021-12-29  23:50:00                  -146.0  4.044704\n",
      "98712  2021-12-31  01:50:00                  -148.0  4.073957\n",
      "98728  2021-12-31  03:10:00                  -118.0  3.635154\n",
      "98729  2021-12-31  03:15:00                  -127.0  3.766795\n",
      "\n",
      "[2096 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "# Function to check timestamp continuity\n",
    "def check_timestamp_continuity(file_path, time_resolution):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "    \n",
    "    # Convert date and time columns to a single datetime column\n",
    "    data['datetime'] = pd.to_datetime(data['date'].astype(str) + ' ' + data['time'])\n",
    "    \n",
    "    # Define the expected time delta based on the resolution\n",
    "    time_deltas = {\n",
    "        '1H': pd.Timedelta(hours=1),\n",
    "        '30min': pd.Timedelta(minutes=30),\n",
    "        '15min': pd.Timedelta(minutes=15),\n",
    "        '5min': pd.Timedelta(minutes=5)\n",
    "    }\n",
    "    \n",
    "    expected_delta = time_deltas[time_resolution]\n",
    "    \n",
    "    # Calculate the actual difference between consecutive timestamps\n",
    "    data['delta'] = data['datetime'].diff()\n",
    "    \n",
    "    # Find rows where the difference is not equal to the expected delta\n",
    "    irregularities = data[data['delta'] != expected_delta]\n",
    "    \n",
    "    # If any irregularities found, print them along with adjacent rows\n",
    "    if not irregularities.empty:\n",
    "        print(f\"Irregular timestamps found in {file_path}:\")\n",
    "        for idx in irregularities.index:\n",
    "            start_idx = max(0, idx - 2)\n",
    "            end_idx = min(len(data) - 2, idx + 2)\n",
    "            print(data.loc[start_idx:end_idx, ['date', 'time', 'delta']])\n",
    "    else:\n",
    "        print(f\"No irregular timestamps found in {file_path}\")\n",
    "\n",
    "# Function to detect outliers in energy consumption\n",
    "def detect_outliers(file_path):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "\n",
    "    # Calculate the Z-score for the energy consumption column\n",
    "    data['z_score'] = np.abs(stats.zscore(data['energy_consumption(kW)']))\n",
    "\n",
    "    # Define an outlier as a value with a Z-score > 3\n",
    "    outliers = data[data['z_score'] > 3]\n",
    "\n",
    "    # Print outliers if found\n",
    "    if not outliers.empty:\n",
    "        print(f\"Outliers detected in {file_path}:\")\n",
    "        print(outliers[['date', 'time', 'energy_consumption(kW)', 'z_score']])\n",
    "    else:\n",
    "        print(f\"No outliers detected in {file_path}\")\n",
    "\n",
    "# Directory paths for input files\n",
    "input_dir = 'train_public/cleaned/'  # Replace with the correct path\n",
    "\n",
    "# Files and their time resolutions\n",
    "files_with_resolutions = {\n",
    "    'L03.B02_1H_cleaned.csv': '1H',\n",
    "    'L06.B01_1H_cleaned.csv': '1H',\n",
    "    'L09.B01_1H_cleaned.csv': '1H',\n",
    "    'L10.B01_1H_cleaned.csv': '1H',\n",
    "    'L14.B01_1H_cleaned.csv': '1H',\n",
    "    'L14.B02_1H_cleaned.csv': '1H',\n",
    "    'L14.B03_1H_cleaned.csv': '1H',\n",
    "    'L14.B04_1H_cleaned.csv': '1H',\n",
    "    'L14.B05_1H_cleaned.csv': '1H',\n",
    "    'L09.B01_30min_cleaned.csv': '30min',\n",
    "    'L10.B01_30min_cleaned.csv': '30min',\n",
    "    'L09.B01_15min_cleaned.csv': '15min',\n",
    "    'L10.B01_15min_cleaned.csv': '15min',\n",
    "    'L09.B01_5min_cleaned.csv': '5min'\n",
    "}\n",
    "\n",
    "# Run checks for each file\n",
    "for file_name, resolution in files_with_resolutions.items():\n",
    "    file_path = os.path.join(input_dir, file_name)\n",
    "    \n",
    "    # Check timestamp continuity\n",
    "    check_timestamp_continuity(file_path, resolution)\n",
    "    \n",
    "    # Detect outliers\n",
    "    detect_outliers(file_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "5cfd3fba-ab99-4562-a431-ae5454b40534",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>energy_consumption(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>169.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>03:00:00</td>\n",
       "      <td>189.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>159.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>05:00:00</td>\n",
       "      <td>152.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>161.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>07:00:00</td>\n",
       "      <td>113.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>08:00:00</td>\n",
       "      <td>117.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>09:00:00</td>\n",
       "      <td>117.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>10:00:00</td>\n",
       "      <td>116.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>11:00:00</td>\n",
       "      <td>117.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>22:00:00</td>\n",
       "      <td>37.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>23:00:00</td>\n",
       "      <td>42.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>32.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>01:00:00</td>\n",
       "      <td>27.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>03:00:00</td>\n",
       "      <td>32.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>17.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>05:00:00</td>\n",
       "      <td>65.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>83.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>07:00:00</td>\n",
       "      <td>75.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date      time  energy_consumption(kW)\n",
       "170  2021-01-08  02:00:00              169.916667\n",
       "171  2021-01-08  03:00:00              189.833333\n",
       "172  2021-01-08  04:00:00              159.916667\n",
       "173  2021-01-08  05:00:00              152.250000\n",
       "174  2021-01-08  06:00:00              161.083333\n",
       "175  2021-01-08  07:00:00              113.083333\n",
       "176  2021-01-08  08:00:00              117.250000\n",
       "177  2021-01-08  09:00:00              117.750000\n",
       "178  2021-01-08  10:00:00              116.916667\n",
       "179  2021-01-08  11:00:00              117.000000\n",
       "180  2021-01-08  22:00:00               37.000000\n",
       "181  2021-01-08  23:00:00               42.833333\n",
       "182  2021-01-09  00:00:00               32.083333\n",
       "183  2021-01-09  01:00:00               27.583333\n",
       "184  2021-01-09  02:00:00               10.000000\n",
       "185  2021-01-09  03:00:00               32.833333\n",
       "186  2021-01-09  04:00:00               17.416667\n",
       "187  2021-01-09  05:00:00               65.250000\n",
       "188  2021-01-09  06:00:00               83.333333\n",
       "189  2021-01-09  07:00:00               75.666667"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l09b01_1H_cleaned = pd.read_csv('train_public/cleaned/L09.B01_1H_cleaned.csv')\n",
    "l09b01_1H_cleaned[170:190]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "3e7932f5-e04c-4431-97b1-ed07450ca2bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>main_meter(kW)</th>\n",
       "      <th>PV_battery_system(kW)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>2021-01-08 02:00:00+00:00</td>\n",
       "      <td>154.916667</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>2021-01-08 03:00:00+00:00</td>\n",
       "      <td>174.833333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>2021-01-08 04:00:00+00:00</td>\n",
       "      <td>144.916667</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>2021-01-08 05:00:00+00:00</td>\n",
       "      <td>137.250000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>2021-01-08 06:00:00+00:00</td>\n",
       "      <td>146.083333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>2021-01-08 07:00:00+00:00</td>\n",
       "      <td>98.083333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>2021-01-08 08:00:00+00:00</td>\n",
       "      <td>102.250000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>2021-01-08 09:00:00+00:00</td>\n",
       "      <td>102.750000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>2021-01-08 10:00:00+00:00</td>\n",
       "      <td>101.916667</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>2021-01-08 11:00:00+00:00</td>\n",
       "      <td>102.000000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>2021-01-08 22:00:00+00:00</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>2021-01-08 23:00:00+00:00</td>\n",
       "      <td>27.833333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>2021-01-09 00:00:00+00:00</td>\n",
       "      <td>17.083333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>2021-01-09 01:00:00+00:00</td>\n",
       "      <td>12.583333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>2021-01-09 02:00:00+00:00</td>\n",
       "      <td>-5.000000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>2021-01-09 03:00:00+00:00</td>\n",
       "      <td>17.833333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>2021-01-09 04:00:00+00:00</td>\n",
       "      <td>2.416667</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>2021-01-09 05:00:00+00:00</td>\n",
       "      <td>50.250000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>2021-01-09 06:00:00+00:00</td>\n",
       "      <td>68.333333</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>2021-01-09 07:00:00+00:00</td>\n",
       "      <td>60.666667</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     timestamp  main_meter(kW)  PV_battery_system(kW)\n",
       "170  2021-01-08 02:00:00+00:00      154.916667                   15.0\n",
       "171  2021-01-08 03:00:00+00:00      174.833333                   15.0\n",
       "172  2021-01-08 04:00:00+00:00      144.916667                   15.0\n",
       "173  2021-01-08 05:00:00+00:00      137.250000                   15.0\n",
       "174  2021-01-08 06:00:00+00:00      146.083333                   15.0\n",
       "175  2021-01-08 07:00:00+00:00       98.083333                   15.0\n",
       "176  2021-01-08 08:00:00+00:00      102.250000                   15.0\n",
       "177  2021-01-08 09:00:00+00:00      102.750000                   15.0\n",
       "178  2021-01-08 10:00:00+00:00      101.916667                   15.0\n",
       "179  2021-01-08 11:00:00+00:00      102.000000                   15.0\n",
       "180  2021-01-08 22:00:00+00:00       22.000000                   15.0\n",
       "181  2021-01-08 23:00:00+00:00       27.833333                   15.0\n",
       "182  2021-01-09 00:00:00+00:00       17.083333                   15.0\n",
       "183  2021-01-09 01:00:00+00:00       12.583333                   15.0\n",
       "184  2021-01-09 02:00:00+00:00       -5.000000                   15.0\n",
       "185  2021-01-09 03:00:00+00:00       17.833333                   15.0\n",
       "186  2021-01-09 04:00:00+00:00        2.416667                   15.0\n",
       "187  2021-01-09 05:00:00+00:00       50.250000                   15.0\n",
       "188  2021-01-09 06:00:00+00:00       68.333333                   15.0\n",
       "189  2021-01-09 07:00:00+00:00       60.666667                   15.0"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l09b01_1h[170:190]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "f525e967-e255-44d3-a0ef-303e26869cdb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8287, 3)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l09b01_1h.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e147707-8d9e-4207-9f56-384ff84eeeae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Directory paths for input files\n",
    "input_energy_dir = 'train_public/cleaned/'  # Replace with the correct path\n",
    "input_weather_dir = 'train_public/weather_cleaned'  # Replace with the correct path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "146742a9-2430-4c9e-8b5b-482ea0844aed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concatenated dataset saved to: L03.B02_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L06.B01_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L09.B01_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L10.B01_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L14.B01_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L14.B02_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L14.B03_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L14.B04_1H_with_weather.csv\n",
      "Concatenated dataset saved to: L14.B05_1H_with_weather.csv\n",
      "Shape of concatenated dataset for train_public/cleaned/L09.B01_30min_cleaned.csv: (16554, 12)\n",
      "Concatenated dataset saved to: L09.B01_30min_with_weather.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/2151781911.py:34: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.\n",
      "  weather_data_interpolated = weather_data.resample(freq).interpolate(method='linear').reset_index()\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/2151781911.py:34: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.\n",
      "  weather_data_interpolated = weather_data.resample(freq).interpolate(method='linear').reset_index()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of concatenated dataset for train_public/cleaned/L10.B01_30min_cleaned.csv: (27731, 12)\n",
      "Concatenated dataset saved to: L10.B01_30min_with_weather.csv\n",
      "Shape of concatenated dataset for train_public/cleaned/L09.B01_15min_cleaned.csv: (33044, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/2151781911.py:34: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.\n",
      "  weather_data_interpolated = weather_data.resample(freq).interpolate(method='linear').reset_index()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concatenated dataset saved to: L09.B01_15min_with_weather.csv\n",
      "Shape of concatenated dataset for train_public/cleaned/L10.B01_15min_cleaned.csv: (55460, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/2151781911.py:34: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.\n",
      "  weather_data_interpolated = weather_data.resample(freq).interpolate(method='linear').reset_index()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concatenated dataset saved to: L10.B01_15min_with_weather.csv\n",
      "Shape of concatenated dataset for train_public/cleaned/L09.B01_5min_cleaned.csv: (98978, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/2151781911.py:34: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.\n",
      "  weather_data_interpolated = weather_data.resample(freq).interpolate(method='linear').reset_index()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concatenated dataset saved to: L09.B01_5min_with_weather.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "# Function to match and concatenate energy data with weather data\n",
    "def match_and_concatenate(energy_file, weather_file, time_resolution):\n",
    "    # Load the building energy and weather datasets\n",
    "    energy_data = pd.read_csv(energy_file)\n",
    "    weather_data = pd.read_csv(weather_file)\n",
    "\n",
    "    # Convert date and time columns to datetime\n",
    "    energy_data['datetime'] = pd.to_datetime(energy_data['date'] + ' ' + energy_data['time'])\n",
    "    weather_data['datetime'] = pd.to_datetime(weather_data['date'] + ' ' + weather_data['time'])\n",
    "\n",
    "    if time_resolution == '1H':\n",
    "        # 1. Concatenate for 1H time resolution\n",
    "        merged_data = pd.merge(energy_data, weather_data, on=['datetime'], how='left')\n",
    "        dropped_rows = merged_data[merged_data.isnull().any(axis=1)]\n",
    "        if not dropped_rows.empty:\n",
    "            print(f\"Dropped rows in {energy_file}:\")\n",
    "            print(dropped_rows[['date', 'time']])\n",
    "        merged_data.dropna(inplace=True)\n",
    "    \n",
    "    else:\n",
    "        # Interpolation for 30min, 15min, and 5min resolutions\n",
    "        if time_resolution == '30min':\n",
    "            freq = '30min'\n",
    "        elif time_resolution == '15min':\n",
    "            freq = '15min'\n",
    "        elif time_resolution == '5min':\n",
    "            freq = '5min'\n",
    "        \n",
    "        # Interpolate weather data\n",
    "        weather_data.set_index('datetime', inplace=True)\n",
    "        weather_data_interpolated = weather_data.resample(freq).interpolate(method='linear').reset_index()\n",
    "        \n",
    "        # Merge with energy data\n",
    "        merged_data = pd.merge(energy_data, weather_data_interpolated, on=['datetime'], how='left')\n",
    "        print(f\"Shape of concatenated dataset for {energy_file}: {merged_data.shape}\")\n",
    "    \n",
    "    # Drop the datetime column as it's no longer needed after the merge\n",
    "    merged_data.drop(columns=['datetime'], inplace=True)\n",
    "\n",
    "    return merged_data\n",
    "\n",
    "# Directory paths for input files\n",
    "input_energy_dir = 'train_public/cleaned/'  # Replace with the correct path\n",
    "input_weather_dir = 'train_public/weather_cleaned/'  # Replace with the correct path\n",
    "\n",
    "# Location and time resolution based files\n",
    "files_with_resolutions = {\n",
    "    'L03.B02_1H_cleaned.csv': '1H',\n",
    "    'L06.B01_1H_cleaned.csv': '1H',\n",
    "    'L09.B01_1H_cleaned.csv': '1H',\n",
    "    'L10.B01_1H_cleaned.csv': '1H',\n",
    "    'L14.B01_1H_cleaned.csv': '1H',\n",
    "    'L14.B02_1H_cleaned.csv': '1H',\n",
    "    'L14.B03_1H_cleaned.csv': '1H',\n",
    "    'L14.B04_1H_cleaned.csv': '1H',\n",
    "    'L14.B05_1H_cleaned.csv': '1H',\n",
    "    'L09.B01_30min_cleaned.csv': '30min',\n",
    "    'L10.B01_30min_cleaned.csv': '30min',\n",
    "    'L09.B01_15min_cleaned.csv': '15min',\n",
    "    'L10.B01_15min_cleaned.csv': '15min',\n",
    "    'L09.B01_5min_cleaned.csv': '5min'\n",
    "}\n",
    "\n",
    "# Match location to weather file\n",
    "location_to_weather = {\n",
    "    'L03': 'L03_weather_train_cleaned.csv',\n",
    "    'L06': 'L06_weather_train_cleaned.csv',\n",
    "    'L09': 'L09_weather_train_cleaned.csv',\n",
    "    'L10': 'L10_weather_train_cleaned.csv',\n",
    "    'L14': 'L14_weather_train_cleaned.csv'\n",
    "}\n",
    "\n",
    "# Run the match and concatenate process for each file\n",
    "for energy_file, time_resolution in files_with_resolutions.items():\n",
    "    # Extract the location from the file name\n",
    "    location = energy_file.split('.')[0]\n",
    "    \n",
    "    # Get the corresponding weather file\n",
    "    weather_file = location_to_weather[location]\n",
    "    \n",
    "    # Full paths to the files\n",
    "    energy_file_path = os.path.join(input_energy_dir, energy_file)\n",
    "    weather_file_path = os.path.join(input_weather_dir, weather_file)\n",
    "    \n",
    "    # Perform the match and concatenate\n",
    "    concatenated_data = match_and_concatenate(energy_file_path, weather_file_path, time_resolution)\n",
    "    \n",
    "    # Save the concatenated data to a new file\n",
    "    output_file = energy_file.replace('_cleaned.csv', '_with_weather.csv')\n",
    "    concatenated_data.to_csv(os.path.join(input_energy_dir, output_file), index=False)\n",
    "    print(f\"Concatenated dataset saved to: {output_file}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "203f278c-6b72-493a-9ce3-a4df25498e24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed file saved to: train_public/pre_trained/L03.B02_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L10.B01_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L06.B01_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L10.B01_30min_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L09.B01_15min_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L14.B01_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L09.B01_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L14.B02_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L09.B01_30min_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L14.B03_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L09.B01_5min_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L14.B04_1H_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L10.B01_15min_with_weather.csv\n",
      "Processed file saved to: train_public/pre_trained/L14.B05_1H_with_weather.csv\n"
     ]
    }
   ],
   "source": [
    "# Function to process the concatenated files\n",
    "def process_concatenated_file(file_path, output_dir):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "    \n",
    "    # Drop the 'date_y' and 'time_y' columns\n",
    "    data = data.drop(columns=['date_y', 'time_y'])\n",
    "    \n",
    "    # Rename 'date_x' and 'time_x' to 'date' and 'time'\n",
    "    data = data.rename(columns={'date_x': 'date', 'time_x': 'time'})\n",
    "    \n",
    "    # Create the output directory if it doesn't exist\n",
    "    if not os.path.exists(output_dir):\n",
    "        os.makedirs(output_dir)\n",
    "    \n",
    "    # Save the processed data to the output directory\n",
    "    output_file = os.path.join(output_dir, os.path.basename(file_path))\n",
    "    data.to_csv(output_file, index=False)\n",
    "    print(f\"Processed file saved to: {output_file}\")\n",
    "\n",
    "# Directory paths\n",
    "input_dir = 'train_public/concatenated/'  # Replace with the correct path\n",
    "output_dir = 'train_public/pre_trained/'  # Replace with the correct path\n",
    "\n",
    "# List of concatenated files to process\n",
    "concatenated_files = [\n",
    "    'L03.B02_1H_with_weather.csv',\n",
    "    'L10.B01_1H_with_weather.csv',\n",
    "    'L06.B01_1H_with_weather.csv',\n",
    "    'L10.B01_30min_with_weather.csv',\n",
    "    'L09.B01_15min_with_weather.csv',\n",
    "    'L14.B01_1H_with_weather.csv',\n",
    "    'L09.B01_1H_with_weather.csv',\n",
    "    'L14.B02_1H_with_weather.csv',\n",
    "    'L09.B01_30min_with_weather.csv',\n",
    "    'L14.B03_1H_with_weather.csv',\n",
    "    'L09.B01_5min_with_weather.csv',\n",
    "    'L14.B04_1H_with_weather.csv',\n",
    "    'L10.B01_15min_with_weather.csv',\n",
    "    'L14.B05_1H_with_weather.csv'\n",
    "]\n",
    "\n",
    "# Process each file\n",
    "for file_name in concatenated_files:\n",
    "    file_path = os.path.join(input_dir, file_name)\n",
    "    process_concatenated_file(file_path, output_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "0f88230f-c62f-45f8-9a0d-ad8463e91b2b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L03.B02_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L10.B01_1H_disaggregated.csv\n",
      "Disaggregated file saved to: results/L06.B01_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/1332892890.py:46: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Input X contains NaN.\nNMF does not accept missing values encoded as NaN natively. For supervised learning, you might want to consider sklearn.ensemble.HistGradientBoostingClassifier and Regressor which accept missing values encoded as NaNs natively. Alternatively, it is possible to preprocess the data, for instance by using an imputer transformer in a pipeline or drop samples with missing values. See https://scikit-learn.org/stable/modules/impute.html You can find a list of all estimators that handle NaN values at the following page: https://scikit-learn.org/stable/modules/impute.html#estimators-that-handle-nan-values",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[181], line 92\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m file_name \u001b[38;5;129;01min\u001b[39;00m cleaned_files:\n\u001b[1;32m     91\u001b[0m     file_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(input_dir, file_name)\n\u001b[0;32m---> 92\u001b[0m     \u001b[43mdisaggregate_heating_cooling\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_dir\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[181], line 42\u001b[0m, in \u001b[0;36mdisaggregate_heating_cooling\u001b[0;34m(file_path, output_dir)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[38;5;66;03m# Apply NMF\u001b[39;00m\n\u001b[1;32m     41\u001b[0m nmf \u001b[38;5;241m=\u001b[39m NMF(n_components\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, init\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrandom\u001b[39m\u001b[38;5;124m'\u001b[39m, random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m42\u001b[39m, max_iter\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1000\u001b[39m)\n\u001b[0;32m---> 42\u001b[0m W \u001b[38;5;241m=\u001b[39m \u001b[43mnmf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     43\u001b[0m H \u001b[38;5;241m=\u001b[39m nmf\u001b[38;5;241m.\u001b[39mcomponents_\n\u001b[1;32m     45\u001b[0m \u001b[38;5;66;03m# Assume the first component (W[:, 0]) corresponds to the temperature-dependent load\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/utils/_set_output.py:313\u001b[0m, in \u001b[0;36m_wrap_method_output.<locals>.wrapped\u001b[0;34m(self, X, *args, **kwargs)\u001b[0m\n\u001b[1;32m    311\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(f)\n\u001b[1;32m    312\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapped\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 313\u001b[0m     data_to_wrap \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    314\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(data_to_wrap, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m    315\u001b[0m         \u001b[38;5;66;03m# only wrap the first output for cross decomposition\u001b[39;00m\n\u001b[1;32m    316\u001b[0m         return_tuple \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    317\u001b[0m             _wrap_data_with_container(method, data_to_wrap[\u001b[38;5;241m0\u001b[39m], X, \u001b[38;5;28mself\u001b[39m),\n\u001b[1;32m    318\u001b[0m             \u001b[38;5;241m*\u001b[39mdata_to_wrap[\u001b[38;5;241m1\u001b[39m:],\n\u001b[1;32m    319\u001b[0m         )\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/base.py:1473\u001b[0m, in \u001b[0;36m_fit_context.<locals>.decorator.<locals>.wrapper\u001b[0;34m(estimator, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1466\u001b[0m     estimator\u001b[38;5;241m.\u001b[39m_validate_params()\n\u001b[1;32m   1468\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[1;32m   1469\u001b[0m     skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[1;32m   1470\u001b[0m         prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[1;32m   1471\u001b[0m     )\n\u001b[1;32m   1472\u001b[0m ):\n\u001b[0;32m-> 1473\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfit_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mestimator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/decomposition/_nmf.py:1650\u001b[0m, in \u001b[0;36mNMF.fit_transform\u001b[0;34m(self, X, y, W, H)\u001b[0m\n\u001b[1;32m   1622\u001b[0m \u001b[38;5;129m@_fit_context\u001b[39m(prefer_skip_nested_validation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m   1623\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit_transform\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, y\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, W\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, H\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m   1624\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Learn a NMF model for the data X and returns the transformed data.\u001b[39;00m\n\u001b[1;32m   1625\u001b[0m \n\u001b[1;32m   1626\u001b[0m \u001b[38;5;124;03m    This is more efficient than calling fit followed by transform.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1648\u001b[0m \u001b[38;5;124;03m        Transformed data.\u001b[39;00m\n\u001b[1;32m   1649\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1650\u001b[0m     X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_data\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1651\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccept_sparse\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcsr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcsc\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat64\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat32\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m   1652\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1654\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m config_context(assume_finite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m   1655\u001b[0m         W, H, n_iter \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fit_transform(X, W\u001b[38;5;241m=\u001b[39mW, H\u001b[38;5;241m=\u001b[39mH)\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/base.py:633\u001b[0m, in \u001b[0;36mBaseEstimator._validate_data\u001b[0;34m(self, X, y, reset, validate_separately, cast_to_ndarray, **check_params)\u001b[0m\n\u001b[1;32m    631\u001b[0m         out \u001b[38;5;241m=\u001b[39m X, y\n\u001b[1;32m    632\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m no_val_X \u001b[38;5;129;01mand\u001b[39;00m no_val_y:\n\u001b[0;32m--> 633\u001b[0m     out \u001b[38;5;241m=\u001b[39m \u001b[43mcheck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minput_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mX\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mcheck_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    634\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m no_val_X \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m no_val_y:\n\u001b[1;32m    635\u001b[0m     out \u001b[38;5;241m=\u001b[39m _check_y(y, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mcheck_params)\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/utils/validation.py:1064\u001b[0m, in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_writeable, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator, input_name)\u001b[0m\n\u001b[1;32m   1058\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1059\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFound array with dim \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m expected <= 2.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1060\u001b[0m         \u001b[38;5;241m%\u001b[39m (array\u001b[38;5;241m.\u001b[39mndim, estimator_name)\n\u001b[1;32m   1061\u001b[0m     )\n\u001b[1;32m   1063\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m force_all_finite:\n\u001b[0;32m-> 1064\u001b[0m     \u001b[43m_assert_all_finite\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1065\u001b[0m \u001b[43m        \u001b[49m\u001b[43marray\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1066\u001b[0m \u001b[43m        \u001b[49m\u001b[43minput_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1067\u001b[0m \u001b[43m        \u001b[49m\u001b[43mestimator_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mestimator_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1068\u001b[0m \u001b[43m        \u001b[49m\u001b[43mallow_nan\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_all_finite\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mallow-nan\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1069\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1071\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copy:\n\u001b[1;32m   1072\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m _is_numpy_namespace(xp):\n\u001b[1;32m   1073\u001b[0m         \u001b[38;5;66;03m# only make a copy if `array` and `array_orig` may share memory`\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/utils/validation.py:123\u001b[0m, in \u001b[0;36m_assert_all_finite\u001b[0;34m(X, allow_nan, msg_dtype, estimator_name, input_name)\u001b[0m\n\u001b[1;32m    120\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m first_pass_isfinite:\n\u001b[1;32m    121\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[0;32m--> 123\u001b[0m \u001b[43m_assert_all_finite_element_wise\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    124\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    125\u001b[0m \u001b[43m    \u001b[49m\u001b[43mxp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mxp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    126\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_nan\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mallow_nan\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    127\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmsg_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmsg_dtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    128\u001b[0m \u001b[43m    \u001b[49m\u001b[43mestimator_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mestimator_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    129\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    130\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/competition/lib/python3.12/site-packages/sklearn/utils/validation.py:172\u001b[0m, in \u001b[0;36m_assert_all_finite_element_wise\u001b[0;34m(X, xp, allow_nan, msg_dtype, estimator_name, input_name)\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m estimator_name \u001b[38;5;129;01mand\u001b[39;00m input_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mX\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m has_nan_error:\n\u001b[1;32m    156\u001b[0m     \u001b[38;5;66;03m# Improve the error message on how to handle missing values in\u001b[39;00m\n\u001b[1;32m    157\u001b[0m     \u001b[38;5;66;03m# scikit-learn.\u001b[39;00m\n\u001b[1;32m    158\u001b[0m     msg_err \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    159\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mestimator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not accept missing values\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    160\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m encoded as NaN natively. For supervised learning, you might want\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    170\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m#estimators-that-handle-nan-values\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    171\u001b[0m     )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg_err)\n",
      "\u001b[0;31mValueError\u001b[0m: Input X contains NaN.\nNMF does not accept missing values encoded as NaN natively. For supervised learning, you might want to consider sklearn.ensemble.HistGradientBoostingClassifier and Regressor which accept missing values encoded as NaNs natively. Alternatively, it is possible to preprocess the data, for instance by using an imputer transformer in a pipeline or drop samples with missing values. See https://scikit-learn.org/stable/modules/impute.html You can find a list of all estimators that handle NaN values at the following page: https://scikit-learn.org/stable/modules/impute.html#estimators-that-handle-nan-values"
     ]
    }
   ],
   "source": [
    "# Function to assign a season based on the date\n",
    "def get_season(month):\n",
    "    if month in [12, 1, 2]:\n",
    "        return 'winter'\n",
    "    elif month in [3, 4, 5]:\n",
    "        return 'spring'\n",
    "    elif month in [6, 7, 8]:\n",
    "        return 'summer'\n",
    "    else:\n",
    "        return 'fall'\n",
    "\n",
    "# Function to perform NMF and save the disaggregated heating/cooling load\n",
    "def disaggregate_heating_cooling(file_path, output_dir):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "    \n",
    "    # Add a season column based on the month\n",
    "    data['datetime'] = pd.to_datetime(data['date'] + ' ' + data['time'])\n",
    "    data['season'] = data['datetime'].dt.month.apply(get_season)\n",
    "    \n",
    "    # Enhance the feature matrix for NMF by including additional relevant variables\n",
    "    features = ['energy_consumption(kW)', 'air_temperature_at_2m(deg_C)', \n",
    "                'relative_humidity_at_2m(%)', 'direct_solar_radiation(W/m^2)', \n",
    "                'diffuse_solar_radiation(W/m^2)', 'wind_speed_at_10m(km/h)', \n",
    "                'wind_direction_at_10m(deg)']\n",
    "    \n",
    "    # Clip negative values to 0 to meet NMF requirements\n",
    "    data[features] = data[features].clip(lower=0)\n",
    "    \n",
    "    # Apply NMF separately for each season\n",
    "    results = []\n",
    "    for season in ['winter', 'spring', 'summer', 'fall']:\n",
    "        season_data = data[data['season'] == season]\n",
    "        if season_data.empty:\n",
    "            continue\n",
    "        \n",
    "        # Prepare the feature matrix for NMF\n",
    "        X = season_data[features]\n",
    "        \n",
    "        # Apply NMF\n",
    "        nmf = NMF(n_components=2, init='random', random_state=42, max_iter=1000)\n",
    "        W = nmf.fit_transform(X)\n",
    "        H = nmf.components_\n",
    "        \n",
    "        # Assume the first component (W[:, 0]) corresponds to the temperature-dependent load\n",
    "        season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
    "        results.append(season_data[['datetime', 'Temperature_dependent(kW)']])\n",
    "    \n",
    "    # Concatenate the results for all seasons\n",
    "    final_result = pd.concat(results)\n",
    "    \n",
    "    # Create the timestamp column\n",
    "    final_result['timestamp'] = final_result['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S+00:00')\n",
    "    \n",
    "    # Select the relevant columns for output\n",
    "    final_output = final_result[['timestamp', 'Temperature_dependent(kW)']]\n",
    "    \n",
    "    # Create the output directory if it doesn't exist\n",
    "    if not os.path.exists(output_dir):\n",
    "        os.makedirs(output_dir)\n",
    "    \n",
    "    # Save the result to a new CSV file\n",
    "    output_file = os.path.join(output_dir, os.path.basename(file_path).replace('_with_weather.csv', '_disaggregated.csv'))\n",
    "    final_output.to_csv(output_file, index=False)\n",
    "    print(f\"Disaggregated file saved to: {output_file}\")\n",
    "\n",
    "# Directory paths\n",
    "input_dir = 'train_public/pre_trained/'  # Replace with the correct path\n",
    "output_dir = 'results/'  # Replace with the correct path\n",
    "\n",
    "# List of cleaned files to process\n",
    "cleaned_files = [\n",
    "    'L03.B02_1H_with_weather.csv',\n",
    "    'L10.B01_1H_with_weather.csv',\n",
    "    'L06.B01_1H_with_weather.csv',\n",
    "    'L10.B01_30min_with_weather.csv',\n",
    "    'L09.B01_15min_with_weather.csv',\n",
    "    'L14.B01_1H_with_weather.csv',\n",
    "    'L09.B01_1H_with_weather.csv',\n",
    "    'L14.B02_1H_with_weather.csv',\n",
    "    'L09.B01_30min_with_weather.csv',\n",
    "    'L14.B03_1H_with_weather.csv',\n",
    "    'L09.B01_5min_with_weather.csv',\n",
    "    'L14.B04_1H_with_weather.csv',\n",
    "    'L10.B01_15min_with_weather.csv',\n",
    "    'L14.B05_1H_with_weather.csv'\n",
    "]\n",
    "\n",
    "# Perform NMF for each file\n",
    "for file_name in cleaned_files:\n",
    "    file_path = os.path.join(input_dir, file_name)\n",
    "    disaggregate_heating_cooling(file_path, output_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "2d1fb310-cfda-4e08-a005-09c6d079a256",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L03.B02_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L10.B01_1H_disaggregated.csv\n",
      "Disaggregated file saved to: results/L06.B01_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L10.B01_30min_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L09.B01_15min_disaggregated.csv\n",
      "Disaggregated file saved to: results/L14.B01_1H_disaggregated.csv\n",
      "Disaggregated file saved to: results/L09.B01_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L14.B02_1H_disaggregated.csv\n",
      "Disaggregated file saved to: results/L09.B01_30min_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L14.B03_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L09.B01_5min_disaggregated.csv\n",
      "Disaggregated file saved to: results/L14.B04_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregated file saved to: results/L10.B01_15min_disaggregated.csv\n",
      "Disaggregated file saved to: results/L14.B05_1H_disaggregated.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
      "/var/folders/wj/rnnxbxj55kn0jfp29lqz0_k00000gp/T/ipykernel_67220/3478108615.py:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  season_data['Temperature_dependent(kW)'] = W[:, 0]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.decomposition import NMF\n",
    "import os\n",
    "\n",
    "# Function to assign a season based on the date\n",
    "def get_season(month):\n",
    "    if month in [12, 1, 2]:\n",
    "        return 'winter'\n",
    "    elif month in [3, 4, 5]:\n",
    "        return 'spring'\n",
    "    elif month in [6, 7, 8]:\n",
    "        return 'summer'\n",
    "    else:\n",
    "        return 'fall'\n",
    "\n",
    "# Function to perform NMF and save the disaggregated heating/cooling load\n",
    "def disaggregate_heating_cooling(file_path, output_dir):\n",
    "    # Load the dataset\n",
    "    data = pd.read_csv(file_path)\n",
    "    \n",
    "    # Add a season column based on the month\n",
    "    data['datetime'] = pd.to_datetime(data['date'] + ' ' + data['time'])\n",
    "    data['season'] = data['datetime'].dt.month.apply(get_season)\n",
    "    \n",
    "    # Enhance the feature matrix for NMF by including additional relevant variables\n",
    "    features = ['energy_consumption(kW)', 'air_temperature_at_2m(deg_C)', \n",
    "                'relative_humidity_at_2m(%)', 'direct_solar_radiation(W/m^2)', \n",
    "                'diffuse_solar_radiation(W/m^2)', 'wind_speed_at_10m(km/h)', \n",
    "                'wind_direction_at_10m(deg)']\n",
    "    \n",
    "    # Clip negative values to 0 to meet NMF requirements\n",
    "    data[features] = data[features].clip(lower=0)\n",
    "    \n",
    "    # Handle missing values by using linear interpolation\n",
    "    data[features] = data[features].interpolate(method='linear', limit_direction='both')\n",
    "    \n",
    "    # Apply NMF separately for each season\n",
    "    results = []\n",
    "    for season in ['winter', 'spring', 'summer', 'fall']:\n",
    "        season_data = data[data['season'] == season]\n",
    "        if season_data.empty:\n",
    "            continue\n",
    "        \n",
    "        # Prepare the feature matrix for NMF\n",
    "        X = season_data[features]\n",
    "        \n",
    "        # Apply NMF\n",
    "        nmf = NMF(n_components=2, init='random', random_state=42, max_iter=1000)\n",
    "        W = nmf.fit_transform(X)\n",
    "        H = nmf.components_\n",
    "        \n",
    "        # Assume the first component (W[:, 0]) corresponds to the temperature-dependent load\n",
    "        season_data['Temperature_dependent(kW)'] = W[:, 0]\n",
    "        results.append(season_data[['datetime', 'Temperature_dependent(kW)']])\n",
    "    \n",
    "    # Concatenate the results for all seasons\n",
    "    final_result = pd.concat(results)\n",
    "    \n",
    "    # Create the timestamp column\n",
    "    final_result['timestamp'] = final_result['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S+00:00')\n",
    "    \n",
    "    # Select the relevant columns for output\n",
    "    final_output = final_result[['timestamp', 'Temperature_dependent(kW)']]\n",
    "    \n",
    "    # Create the output directory if it doesn't exist\n",
    "    if not os.path.exists(output_dir):\n",
    "        os.makedirs(output_dir)\n",
    "    \n",
    "    # Save the result to a new CSV file\n",
    "    output_file = os.path.join(output_dir, os.path.basename(file_path).replace('_with_weather.csv', '_disaggregated.csv'))\n",
    "    final_output.to_csv(output_file, index=False)\n",
    "    print(f\"Disaggregated file saved to: {output_file}\")\n",
    "\n",
    "# Directory paths\n",
    "input_dir = 'train_public/pre_trained/'  # Replace with the correct path\n",
    "output_dir = 'results/'  # Replace with the correct path\n",
    "\n",
    "# List of cleaned files to process\n",
    "cleaned_files = [\n",
    "    'L03.B02_1H_with_weather.csv',\n",
    "    'L10.B01_1H_with_weather.csv',\n",
    "    'L06.B01_1H_with_weather.csv',\n",
    "    'L10.B01_30min_with_weather.csv',\n",
    "    'L09.B01_15min_with_weather.csv',\n",
    "    'L14.B01_1H_with_weather.csv',\n",
    "    'L09.B01_1H_with_weather.csv',\n",
    "    'L14.B02_1H_with_weather.csv',\n",
    "    'L09.B01_30min_with_weather.csv',\n",
    "    'L14.B03_1H_with_weather.csv',\n",
    "    'L09.B01_5min_with_weather.csv',\n",
    "    'L14.B04_1H_with_weather.csv',\n",
    "    'L10.B01_15min_with_weather.csv',\n",
    "    'L14.B05_1H_with_weather.csv'\n",
    "]\n",
    "\n",
    "# Perform NMF for each file\n",
    "for file_name in cleaned_files:\n",
    "    file_path = os.path.join(input_dir, file_name)\n",
    "    disaggregate_heating_cooling(file_path, output_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac381ab4-de3a-4a22-bb71-b30933a47b0c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}