Adds modules for ingesting new LSM data and modifies CreateInflowFile…

RAPIDpy/gis/weight.py

@@ -104,6 +104,12 @@ def rtree_create_weight_table(lsm_grid_lat, lsm_grid_lon,
         ogr_catchment_shapefile = ogr.Open(in_catchment_shapefile)
         ogr_catchment_shapefile_lyr = ogr_catchment_shapefile.GetLayer()
 
+    # MPG DEBUG:
+    # ldefn = ogr_catchment_shapefile_lyr.GetLayerDefn()
+    # for n in range(ldefn.GetFieldCount()):
+    #     fdefn = ldefn.GetFieldDefn(n)
+    #     print(fdefn.name)
+
     ogr_catchment_shapefile_lyr_proj = \
         ogr_catchment_shapefile_lyr.GetSpatialRef()
     original_catchment_proj = \

RAPIDpy/inflow/CreateInflowFileFromGALWEMRunoff.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+   CreateInflowFileFromGALWEMRunoff.py
+   RAPIDpy
+
+   Created by Matthew P. Geheran, 2018
+   Adapted from CreateInflowFileFromECMWFRunoff.py.
+"""
+from netCDF4 import Dataset
+
+from .CreateInflowFileFromGriddedRunoff import \
+    CreateInflowFileFromGriddedRunoff
+
+
+class CreateInflowFileFromGALWEMRunoff(CreateInflowFileFromGriddedRunoff):
+    """Create Inflow File From LDAS Runoff
+
+    Base class for creating RAPID NetCDF input
+    of water inflow based on LDAS land surface model
+    runoff and previously created weight table.
+    """
+    land_surface_model_name = "LDAS"
+
+    def __init__(self,
+                 lat_dim,  # "lat",
+                 lon_dim,  # "lon",
+                 lat_var,  # "lat",
+                 lon_var,  # "lon",
+                 runoff_vars):  # ["ssrun", "bgrun"],
+        """Define the attributes to look for"""
+        self.dims_oi = [lon_dim, lat_dim]
+        self.vars_oi = [lon_var, lat_var] + runoff_vars
+        self.runoff_vars = runoff_vars
+        self.length_time = {"3-Hourly": 1}
+
+        super(CreateInflowFileFromGALWEMRunoff, self).__init__()
+
+    def data_validation(self, in_nc):
+        """Check the necessary dimensions and variables in the
+        input netcdf data"""
+        data_nc = Dataset(in_nc)
+        for dim in self.dims_oi:
+            if dim not in data_nc.dimensions.keys():
+                data_nc.close()
+                raise Exception(self.error_messages[1])
+
+        for var in self.vars_oi:
+            if var not in data_nc.variables.keys():
+                data_nc.close()
+                raise Exception(self.error_messages[2])
+
+        data_nc.close()
+        return

RAPIDpy/inflow/CreateInflowFileFromGriddedRunoff.py

@@ -11,7 +11,7 @@
 from datetime import datetime
 import os
 
-from netCDF4 import Dataset
+from netCDF4 import Dataset, num2date, date2num
 import numpy as np
 from pytz import utc
 from past.builtins import xrange  # pylint: disable=redefined-builtin
@@ -309,7 +309,7 @@ def data_validation(self, in_nc):
         pass
 
     def execute(self, nc_file_list, index_list, in_weight_table,
-                out_nc, grid_type, mp_lock):
+                out_nc, grid_type, mp_lock): #, all_simulation_time):
 
         """The source code of the tool."""
         if not os.path.exists(out_nc):
@@ -403,6 +403,26 @@ def execute(self, nc_file_list, index_list, in_weight_table,
                             len_time_subset,
                             (len_lat_subset * len_lon_subset))
 
+                    # MPG ADDED:
+                    all_simulation_time_units = (
+                        'seconds since 1970-01-01 00:00:00+00:00')
+                    # data_in_nc_start_time = num2date(data_in_nc['time'][0],
+                    #                                  data_in_nc['time'].units)
+
+                    # MPG FIX FOR TIME OFFSET BETWEEN LSM NC AND M3 NC:
+                    # data_in_nc_start_time = datetime.strptime(
+                    #     file_re_match.search(nc_file).group(0), 
+                    #     file_datetime_pattern)
+
+                    # DEBUG
+                    # print 'DATA_IN_START_TIME', data_in_nc_start_time
+                    # data_in_nc_start_time = date2num(data_in_nc_start_time,
+                    #                                  all_simulation_time_units)
+                    # data_out_nc_start_idx = np.abs(data_in_nc_start_time -
+                    #                                all_simulation_time).argmin()
+                    # DEBUG
+                    # print 'ALL_SIMULATION_TIME', num2date(all_simulation_time[data_out_nc_start_idx], all_simulation_time_units)
+
                 data_in_nc.close()
 
                 if not index_new:
@@ -500,10 +520,32 @@ def execute(self, nc_file_list, index_list, in_weight_table,
             mp_lock.acquire()
             data_out_nc = Dataset(out_nc, "a", format="NETCDF3_CLASSIC")
             if runoff_dimension_size == 3 and len_time_subset > 1:
+                # MPG ADDED:
+                # data_out_nc_end_idx = data_out_nc_start_idx + len_time_subset
+                # data_out_nc_end_idx = data_out_nc_start_idx + len_time_subset
+                # try:
+                #     data_out_nc.variables['m3_riv'][
+                #         data_out_nc_start_idx:data_out_nc_end_idx, :] = \
+                #         inflow_data
+                # except IndexError:
+                #     print ("WARNING: Inflow data dimensions inconsistent" + 
+                #            " with 'm3_riv' variable slice")
+                #     print "NC_FILE", nc_file
+                #     print "INFLOW_DATA.SHAPE", inflow_data.shape
+                #     print "DATA_OUT_NC_START_IDX", data_out_nc_start_idx
+                #     print "DATA_OUT_NC_END_IDX", data_out_nc_end_idx
+
+                # MPG DEBUG
+                # import pdb
+                # pdb.set_trace()
+
                 data_out_nc.variables['m3_riv'][
                     index*len_time_subset:(index+1)*len_time_subset, :] = \
                     inflow_data
             else:
+                # MPG DEBUG:
+                # print "INDEX", index
+                # print "M3_RIV DIMENSIONS", data_out_nc.variables['m3_riv'].shape
                 data_out_nc.variables['m3_riv'][index] = inflow_data
             data_out_nc.close()
             mp_lock.release()

RAPIDpy/inflow/CreateInflowFileFromJULESRunoff.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+   CreateInflowFileFromLDASRunoff.py
+   RAPIDpy
+   Created by Matthew P. Geheran, 2018.
+   Adapted from CreateInflowFileFromLDASRunoff.py.
+"""
+from netCDF4 import Dataset
+
+from .CreateInflowFileFromGriddedRunoff import \
+    CreateInflowFileFromGriddedRunoff
+
+
+class CreateInflowFileFromJULESRunoff(CreateInflowFileFromGriddedRunoff):
+    """Create Inflow File From JULES Runoff
+
+    Base class for creating RAPID NetCDF input
+    of water inflow based on JULES land surface model
+    runoff and previously created weight table.
+    """
+    land_surface_model_name = "JULES"
+
+    def __init__(self,
+                 lat_dim,  # "lat",
+                 lon_dim,  # "lon",
+                 lat_var,  # "latitude",
+                 lon_var,  # "longitude",
+                 runoff_vars):  # ["surface_runoff", "sub_surface_runoff"],
+        """Define the attributes to look for"""
+        self.dims_oi = [lon_dim, lat_dim]
+        self.vars_oi = [lon_var, lat_var] + runoff_vars
+        self.runoff_vars = runoff_vars
+        self.length_time = {"Hourly": 3}
+
+        super(CreateInflowFileFromJULESRunoff, self).__init__()
+
+    def data_validation(self, in_nc):
+        """Check the necessary dimensions and variables in the
+        input netcdf data"""
+        data_nc = Dataset(in_nc)
+        for dim in self.dims_oi:
+            # MPG DEBUG:
+            print dim
+            print data_nc.dimensions.keys()
+            if dim not in data_nc.dimensions.keys():
+                data_nc.close()
+                raise Exception(self.error_messages[1])
+
+        for var in self.vars_oi:
+            # MPG DEBUG:
+            print var
+            print data_nc.dimensions.keys()
+            if var not in data_nc.variables.keys():
+                data_nc.close()
+                raise Exception(self.error_messages[2])
+
+        data_nc.close()
+        return

RAPIDpy/inflow/CreateInflowFileFromLDASRunoff.py

@@ -40,6 +40,8 @@ def data_validation(self, in_nc):
         """Check the necessary dimensions and variables in the
         input netcdf data"""
         data_nc = Dataset(in_nc)
+        print "dims", self.dims_oi
+        print "vars", self.vars_oi
         for dim in self.dims_oi:
             if dim not in data_nc.dimensions.keys():
                 data_nc.close()

RAPIDpy/inflow/CreateInflowFileFromNewERAInterimRunoff.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+   CreateInflowFileFromERAInterimRunoff.py
+   RAPIDpy
+
+   Created by Alan D. Snow, 2015
+   Adapted from CreateInflowFileFromECMWFRunoff.py.
+   License: BSD-3-Clause
+"""
+from netCDF4 import Dataset
+
+from .CreateInflowFileFromGriddedRunoff import \
+    CreateInflowFileFromGriddedRunoff
+
+
+class CreateInflowFileFromNewERAInterimRunoff(CreateInflowFileFromGriddedRunoff):
+    """Create Inflow File From ERA Interim Runoff
+
+    Creates RAPID NetCDF input of water inflow based on
+    ERA Interim runoff and previously created weight table.
+    """
+    land_surface_model_name = "ERA Interim"
+    header_wt = ['rivid', 'area_sqm', 'lon_index', 'lat_index', 'npoints']
+    dims_oi = [['time', 'lon', 'lat'], ['time', 'longitude', 'latitude']]
+    vars_oi = [['time', 'lon', 'lat', 'RO'],
+               ['time', 'longitude', 'latitude', 'ro']]
+    length_time = {"Daily": 1, "6-Hourly": 4}
+
+    def __init__(self):
+        """Define the attributes to look for"""
+        self.runoff_vars = ['ro']
+        super(CreateInflowFileFromNewERAInterimRunoff, self).__init__()
+
+    def data_validation(self, in_nc):
+        """Check the necessary dimensions and variables in the input
+        netcdf data"""
+        data_nc = Dataset(in_nc)
+
+        dims = list(data_nc.dimensions)
+
+        if dims not in self.dims_oi:
+            data_nc.close()
+            raise Exception("{0} {1}".format(self.error_messages[1], dims))
+
+        nc_vars = list(data_nc.variables)
+
+        if nc_vars == self.vars_oi[0]:
+            self.runoff_vars = [self.vars_oi[0][-1]]
+        elif nc_vars == self.vars_oi[1]:
+            self.runoff_vars = [self.vars_oi[1][-1]]
+        else:
+            data_nc.close()
+            raise Exception("{0} {1}".format(self.error_messages[2], nc_vars))
+        data_nc.close()

RAPIDpy/inflow/CreateInflowFileFromWSIMRunoff.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+   CreateInflowFileFromLDASRunoff.py
+   RAPIDpy
+   Created by Matthew P. Geheran, 2018.
+   Adapted from CreateInflowFileFromLDASRunoff.py.
+"""
+from netCDF4 import Dataset
+
+from .CreateInflowFileFromGriddedRunoff import \
+    CreateInflowFileFromGriddedRunoff
+
+
+class CreateInflowFileFromWSIMRunoff(CreateInflowFileFromGriddedRunoff):
+    """Create Inflow File From WSIM Runoff
+
+    Base class for creating RAPID NetCDF input
+    of water inflow based on the WSIM land surface model
+    runoff and previously created weight table.
+    """
+    land_surface_model_name = "WSIM"
+
+    def __init__(self,
+                 lat_dim,  # "lat",
+                 lon_dim,  # "lon",
+                 lat_var,  # "latitude",
+                 lon_var,  # "longitude",
+                 runoff_vars):  # ["surface_runoff", "sub_surface_runoff"],
+        """Define the attributes to look for"""
+        self.dims_oi = [lon_dim, lat_dim]
+        self.vars_oi = [lon_var, lat_var] + runoff_vars
+        self.runoff_vars = runoff_vars
+        self.length_time = {"Hourly": 6}
+
+        super(CreateInflowFileFromWSIMRunoff, self).__init__()
+
+    def data_validation(self, in_nc):
+        """Check the necessary dimensions and variables in the
+        input netcdf data"""
+        data_nc = Dataset(in_nc)
+        for dim in self.dims_oi:
+            # MPG DEBUG:
+            print dim
+            print data_nc.dimensions.keys()
+            if dim not in data_nc.dimensions.keys():
+                data_nc.close()
+                raise Exception(self.error_messages[1])
+
+        for var in self.vars_oi:
+            # MPG DEBUG:
+            print var
+            print data_nc.variables.keys()
+            if var not in data_nc.variables.keys():
+                data_nc.close()
+                raise Exception(self.error_messages[2])
+
+        data_nc.close()
+        return