Merge remote-tracking branch 'origin/main' into jacob/lstm

Author: jacobbieker

data/gcb/process_data.ipynb

@@ -0,0 +1,99 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import regionmask\n",
+    "import xarray as xr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process_ds(path):\n",
+    "    ds = xr.open_zarr(path, consolidated=True)\n",
+    "\n",
+    "    ds['ELUC'] = ds['ELUC'].shift(time=1)\n",
+    "    ds['ELUC_diff'] = ds['ELUC_diff'].shift(time=1)\n",
+    "    ds['time'] = ds.time - 1\n",
+    "\n",
+    "    ds = ds.stack(latlon=('lat', 'lon'))\n",
+    "\n",
+    "    mask = ds['ELUC_diff'].isnull().compute()\n",
+    "    ds = ds.where(~mask, drop=True)\n",
+    "\n",
+    "    country_mask = regionmask.defined_regions.natural_earth_v5_0_0.countries_110.mask(ds)\n",
+    "    ds = ds.assign_coords(country=country_mask)\n",
+    "    \n",
+    "    return ds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def convert_to_dataframe(da):\n",
+    "    df = da.to_dataframe()\n",
+    "    df.index = df.index.set_names(['time', 'i_lat', 'i_lon'])\n",
+    "    df = df.reset_index()\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "PATH_TO_DATASET = \"../merged_aggregated_dataset_1850_2022.zarr.zip\"\n",
+    "ds = process_ds(PATH_TO_DATASET)\n",
+    "\n",
+    "countries = regionmask.defined_regions.natural_earth_v5_0_0.countries_110\n",
+    "countries_df = countries.to_dataframe()\n",
+    "\n",
+    "# Change this\n",
+    "country_list = [\"GB\", \"BR\", \"CH\"]\n",
+    "country_id_list = countries_df.index[countries_df['abbrevs'].isin(country_list)].tolist()\n",
+    "\n",
+    "da = ds.where(ds.country.isin(country_id_list), drop=True)\n",
+    "df = convert_to_dataframe(da)\n",
+    "path = \"\"\n",
+    "for c in country_list:\n",
+    "    path += c + \"_\"\n",
+    "path += \"eluc.csv\"\n",
+    "df.to_csv(f\"processed/path.lower()\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "leaf",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

demo/Predictor.py

@@ -0,0 +1,148 @@
+import warnings
+import json
+
+import pandas as pd
+import torch
+
+from xgboost import XGBRegressor
+from unileaf_util.framework.transformers.data_encoder import DataEncoder
+
+from constants import fields
+from constants import cao_mapping
+from constants import LAND_USE_COLS
+from constants import COLS_MAP
+from constants import DIFF_LAND_USE_COLS
+from constants import XGBOOST_FILE_PATH
+from constants import LSTM_CONFIG_PATH
+from constants import LSTM_FILE_PATH
+from constants import CONTEXT_COLUMNS
+from constants import ALL_DIFF_LAND_USE_COLS
+from constants import XGBOOST_FEATURES
+
+# Silence xgboost warnings
+warnings.filterwarnings("ignore")
+
+class Predictor:
+    """
+    Wraps XGBoost model and DataEncoder.
+    To be updated later to handle LSTM/other models.
+    """
+
+    def __init__(self):
+        pass
+
+    def run_predictor(self, context: pd.DataFrame, prescribed: pd.DataFrame) -> float:
+        """
+        Runs predictor using context and prescription.
+        """
+        pass
+
+
+class XGBoostPredictor(Predictor):
+    def __init__(self, model_path=XGBOOST_FILE_PATH):
+        """
+        :param model_path: Path to XGBoost model file
+        """
+        super().__init__()
+        self.predictor_model = XGBRegressor()
+        self.predictor_model.load_model(model_path)
+
+        self.encoder = DataEncoder(fields, cao_mapping)
+
+
+    def run_predictor(self, context: pd.DataFrame, prescribed: pd.DataFrame) -> float:
+        """
+        Runs predictor model.
+        :param context: DataFrame of context.
+        :param prescribed: DataFrame of prescribed land usage to be diffed.
+        :return: Tuple of predicted ELUC and percentage of land use changed.
+        """
+
+        # TODO: Encoder deletes columns not in it. Manual add c4per
+        encoded_sample_context_df = self.encoder.encode_as_df(context)
+
+        prescribed_actions_df = prescribed[LAND_USE_COLS].reset_index(drop=True) \
+            - context[LAND_USE_COLS].reset_index(drop=True)
+        prescribed_actions_df.rename(COLS_MAP, axis=1, inplace=True)
+
+        encoded_prescribed_actions_df = self.encoder.encode_as_df(prescribed_actions_df)
+
+        # TODO: Hacky
+        encoded_prescribed_actions_df["primn_diff"] = 0
+        encoded_prescribed_actions_df["primf_diff"] = 0
+        encoded_prescribed_actions_df["c4per_diff"] = 0
+        encoded_sample_context_df["c4per"] = 0
+
+        context_actions = pd.concat([encoded_sample_context_df, encoded_prescribed_actions_df], axis=1)
+
+        pred = self.predictor_model.predict(context_actions[XGBOOST_FEATURES])
+        pred_df = pd.DataFrame(pred, columns=["ELUC"])
+        # Decode output
+        out_df = self.encoder.decode_as_df(pred_df)
+        return out_df.iloc[0, 0]
+
+
+class LSTMPredictor(Predictor):
+
+    class LSTMModel(torch.nn.Module):
+        def __init__(self, in_features, layers=1, hidden_dim=64, dropout=0):
+            super().__init__()
+            self.norm = torch.nn.LayerNorm(in_features)
+            self.lstm = torch.nn.LSTM(in_features, hidden_dim, num_layers=layers, batch_first=True, dropout=dropout)
+            self.ff = torch.nn.Linear(hidden_dim, 1)
+
+        def forward(self, x):
+            x = self.norm(x)
+            x, _ = self.lstm(x)
+            x = self.ff(x[:,-1,:])
+            x = torch.squeeze(x)
+            return x
+    
+    def __init__(self, config_path=LSTM_CONFIG_PATH, model_path=LSTM_FILE_PATH):
+        """
+        :param model_path: Path to XGBoost model file
+        """
+        super().__init__()
+        config = json.load(open(LSTM_CONFIG_PATH))
+        self.predictor_model = self.LSTMModel(in_features=len(CONTEXT_COLUMNS + ALL_DIFF_LAND_USE_COLS), **config)
+        self.predictor_model.load_state_dict(torch.load(model_path))
+        self.predictor_model.eval()
+
+        self.encoder = DataEncoder(fields, cao_mapping)
+
+
+    def run_predictor(self, context: pd.DataFrame, prescribed: pd.DataFrame) -> float:
+        """
+        Runs predictor model.
+        :param context: DataFrame of context.
+        :param prescribed: DataFrame of prescribed land usage to be diffed.
+        :return: Tuple of predicted ELUC and percentage of land use changed.
+        """
+
+        context_df = context.reset_index(drop=True)
+        encoded_context_df = self.encoder.encode_as_df(context_df)
+        # TODO: This is yucky because we have to add primn and primf 0 diffs since our
+        # prescriptor doesn't handle them.
+        prescribed_actions_df = prescribed[LAND_USE_COLS].reset_index(drop=True) \
+            - context_df[LAND_USE_COLS].iloc[[-1]].reset_index(drop=True)
+        prescribed_actions_df.rename(COLS_MAP, inplace=True, axis=1)
+        encoded_prescribed_actions_df = self.encoder.encode_as_df(prescribed_actions_df)
+
+        # TODO: @IMPORTANT WE DONT PRESCRIBE C4PER. Does this destroy our df?
+        encoded_prescribed_actions_df["primn_diff"] = 0
+        encoded_prescribed_actions_df["primf_diff"] = 0
+        encoded_prescribed_actions_df["c4per_diff"] = 0
+        encoded_context_df["c4per"] = 0
+        encoded_context_df["primn_diff"] = 0
+        encoded_context_df["primf_diff"] = 0
+        encoded_context_df["c4per_diff"] = 0
+
+        encoded_context_df.loc[encoded_context_df.index[-1], DIFF_LAND_USE_COLS] = encoded_prescribed_actions_df.loc[encoded_prescribed_actions_df.index[0], DIFF_LAND_USE_COLS]
+        df_np = encoded_context_df[CONTEXT_COLUMNS + ALL_DIFF_LAND_USE_COLS].to_numpy()
+        inp = torch.from_numpy(df_np)
+        inp = inp.type(torch.FloatTensor)
+        inp = inp.unsqueeze(0)
+
+        out = self.predictor_model(inp)
+
+        return out.item()

demo/Prescriptor.py

@@ -0,0 +1,119 @@
+import os
+from typing import List
+import pandas as pd
+import numpy as np
+from keras.models import load_model
+
+from unileaf_util.framework.transformers.data_encoder import DataEncoder
+from constants import fields
+from constants import cao_mapping
+from constants import PRESCRIPTOR_OUTPUT_COLS
+
+class Prescriptor:
+    """
+    Wrapper for Keras prescriptor and encoder.
+    """
+
+    def __init__(self, prescriptor_id: str):
+        """
+        :param prescriptor_id: ID of Keras prescriptor to load.
+        """
+        prescriptor_model_filename = os.path.join("prescriptors",
+                                                prescriptor_id + '.h5')
+
+        print(f'Loading prescriptor model: {prescriptor_model_filename}')
+        self.prescriptor_model = load_model(prescriptor_model_filename, compile=False)
+
+        self.encoder = DataEncoder(fields, cao_mapping)
+
+
+    def _is_single_action_prescriptor(self, actions):
+        """
+        Checks how many Actions have been defined in the Context, Actions, Outcomes mapping.
+        :return: True if only 1 action is defined, False otherwise
+        """
+        return len(actions) == 1
+
+    def _is_scalar(self, prescribed_action):
+        """
+        Checks if the prescribed action contains a single value, i.e. a scalar, or an array.
+        A prescribed action contains a single value if it has been prescribed for a single context sample
+        :param prescribed_action: a scalar or an array
+        :return: True if the prescribed action contains a scalar, False otherwise.
+        """
+        return prescribed_action.shape[0] == 1 and prescribed_action.shape[1] == 1
+
+    def _convert_to_nn_input(self, context_df: pd.DataFrame) -> List[np.ndarray]:
+        """
+        Converts a context DataFrame to a list of numpy arrays a neural network can ingest
+        :param context_df: a DataFrame containing inputs for a neural network. Number of inputs and size must match
+        :return: a list of numpy ndarray, on ndarray per neural network input
+        """
+        # The NN expects a list of i inputs by s samples (e.g. 9 x 299).
+        # So convert the data frame to a numpy array (gives shape 299 x 9), transpose it (gives 9 x 299)
+        # and convert to list(list of 9 arrays of 299)
+        context_as_nn_input = list(context_df.to_numpy().transpose())
+        # Convert each column's list of 1D array to a 2D array
+        context_as_nn_input = [np.stack(context_as_nn_input[i], axis=0) for i in
+                            range(len(context_as_nn_input))]
+        return context_as_nn_input
+
+    def __prescribe_from_model(self, context_df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Generates prescriptions using the passed neural network candidate and context
+        ::param context_df: a DataFrame containing the context to prescribe for,
+        :return: a pandas DataFrame of action name to action value or list of action values
+        """
+        action_list = ['recommended_land_use']
+
+        # Convert the input df
+        context_as_nn_input = self._convert_to_nn_input(context_df)
+        row_index = context_df.index
+
+        # Get the prescrib?ed actions
+        prescribed_actions = self.prescriptor_model.predict(context_as_nn_input)
+        actions = {}
+
+        if self._is_single_action_prescriptor(action_list):
+            # Put the single action in an array to process it like multiple actions
+            prescribed_actions = [prescribed_actions]
+
+        for idx, action_col in enumerate(action_list):
+            if self._is_scalar(prescribed_actions[idx]):
+                # We have a single row and this action is numerical. Convert it to a scalar.
+                actions[action_col] = prescribed_actions[idx].item()
+            else:
+                actions[action_col] = prescribed_actions[idx].tolist()
+
+        # Convert the prescribed actions to a DataFrame
+        prescribed_actions_df = pd.DataFrame(actions,
+                                            columns=action_list,
+                                            index=row_index)
+        return prescribed_actions_df
+
+
+    def run_prescriptor(self, sample_context_df):
+        """
+        Runs prescriptor on context. Then re-scales prescribed land
+        use to match how much was used in the sample.
+
+        :param sample_context_df: a DataFrame containing the context
+        :return: DataFrame of prescribed land use
+        """
+        encoded_sample_context_df = self.encoder.encode_as_df(sample_context_df)
+        prescribed_actions_df = self.__prescribe_from_model(encoded_sample_context_df)
+        reco_land_use_df = pd.DataFrame(prescribed_actions_df.recommended_land_use.tolist(),
+                                    columns=PRESCRIPTOR_OUTPUT_COLS)
+
+        # Re-scales our prescribed land to match the amount of land used in the sample
+        used = sum(sample_context_df[PRESCRIPTOR_OUTPUT_COLS].iloc[0].tolist())
+        for col in PRESCRIPTOR_OUTPUT_COLS:
+            reco_land_use_df[col] *= used
+
+        # Assuming there's no primary land left in this cell
+        # TODO: not correct. Need to account for primf and primn, that can't increase
+        # (no way to return to primary forest)
+        prescribed_land_use_pct = reco_land_use_df.iloc[0][PRESCRIPTOR_OUTPUT_COLS].sum() * 100
+        print(f"Presribed land usage: {prescribed_land_use_pct:.2f}% of land")
+
+        return reco_land_use_df[PRESCRIPTOR_OUTPUT_COLS]

demo/README.md

@@ -0,0 +1,20 @@
+# MVP Climate Change Demo
+
+This is a demo of the MVP Climate Change app. It allows users to select a location and year from a map of the UK, Switzerland, or Brazil, see its land use composition, and prescribe or manually make changes to it and see the predicted ELUC (Emissions from Land Use Change). It is a simple Dash app.
+
+## Dependencies:
+
+The demo relies on the `unileaf_util` package. This has to be manually installed from the `.whl` file.
+The requirements.txt is set up for M1 macs. Your installations of tensorflow and pytorch may differ.
+
+## Running the app:
+
+To run the app use: ``python app.py``
+
+## Predictors:
+
+Saved predictors can be found in `predictors/`. The XGBoost predictor's weights are stored in a `.json` file whereas the LSTM predictor's weights are stored in a `.pt` file and its configuration is saved in the corresponding `.json` file.
+
+## Prescriptors:
+
+Prescriptors are stored in `prescriptors/` as well as the pareto front image and a CSV of pareto info from training the prescriptors.