Update README and improve code formatting in data model

- Clarified README to specify the contents of the low-volume data folder.
- Enhanced formatting in data_model.py for better readability.
- Added detailed docstrings for RelativeHumidity and DewPoint classes in merra.py, including references.

Author: bnb32

nsrdb/data_model/data/README.md

@@ -1,4 +1,4 @@
 # NSRDB Supporting Data
-This folder holds some low-volume data to support the NSRDB data model.
-All of the real full-volume source data is stored on HPC in /projects/pxs/.
-The data saved here is just metadata and other minor supplemental information.
+This folder holds some low-volume data to support the NSRDB data model. The
+txt files contain climatological correction factors for alpha and aod which
+are used to correct for systemic biases in MERRA2 data.

nsrdb/data_model/data_model.py

@@ -70,6 +70,7 @@ class DataModel:
     CACHE_DIR = '/projects/pxs/reference_grids/_nn_query_cache'
 
     # source files for weight factors
+
     WEIGHTS: ClassVar = {
         'aod': os.path.join(
             DATADIR, 'Monthly_pixel_correction_MERRA2_AOD.txt'
@@ -230,8 +231,9 @@ def _parse_nsrdb_grid(
             self._nsrdb_grid = clean_meta(pd.read_csv(inp, index_col=0))
         else:
             raise TypeError(
-                'Expected csv grid file or DataFrame but '
-                'received: {}'.format(inp)
+                'Expected csv grid file or DataFrame but received: {}'.format(
+                    inp
+                )
             )
 
         # check requirements
@@ -473,7 +475,7 @@ def get_cloud_nn(fp_cloud, cloud_kwargs, nsrdb_grid, dist_lim=1.0):
         try:
             grid = cloud_obj_single.grid
         except Exception as e:
-            msg = 'Exception building cloud NN ' 'tree for {}: {}'.format(
+            msg = 'Exception building cloud NN tree for {}: {}'.format(
                 cloud_obj_single, e
             )
             logger.error(msg)
@@ -1244,12 +1246,11 @@ def _process_clouds(self, cloud_vars, dist_lim=1.0, max_workers=None):
                 cloud_data[var] = self.scale_data(var, arr)
 
         logger.info(
-            'Finished extracting cloud data ' 'and writing to NSRDB arrays.'
+            'Finished extracting cloud data and writing to NSRDB arrays.'
         )
         mem = psutil.virtual_memory()
         logger.info(
-            'Current memory usage is '
-            '{:.3f} GB out of {:.3f} GB total.'.format(
+            'Current memory usage is {:.3f} GB out of {:.3f} GB total.'.format(
                 mem.used / 1e9, mem.total / 1e9
             )
         )
@@ -1412,8 +1413,9 @@ def _interpolate(self, var):
 
         # run spatial interpolation
         logger.debug(
-            'Performing spatial interpolation on "{}" ' 'with shape {}'.format(
-                var, data.shape
+            'Performing spatial interpolation on "{}" with method "{}", '
+            'metric "{}", and with shape {}'.format(
+                var, var_obj.spatial_method, var_obj.NN_METHOD, data.shape
             )
         )
         data = spatial_interp(
@@ -1893,7 +1895,7 @@ def run_single(
             data = method(var, **kwargs)
         except Exception as e:
             logger.exception(
-                'Processing method "DataModel.{}()" failed for ' '"{}"'.format(
+                'Processing method "DataModel.{}()" failed for "{}"'.format(
                     method.__name__, var
                 )
             )
@@ -2141,8 +2143,9 @@ def run_multiple(
             )
         else:
             raise TypeError(
-                'Expected csv grid file or DataFrame but '
-                'received: {}'.format(nsrdb_grid)
+                'Expected csv grid file or DataFrame but received: {}'.format(
+                    nsrdb_grid
+                )
             )
 
         logger.debug(

nsrdb/data_model/merra.py

@@ -277,9 +277,10 @@ def grid(self):
             with NSRDBfs(self.file) as f:
                 lon2d, lat2d = np.meshgrid(f['lon'][:], f['lat'][:])
 
-            self._grid = pd.DataFrame(
-                {'longitude': lon2d.ravel(), 'latitude': lat2d.ravel()}
-            )
+            self._grid = pd.DataFrame({
+                'longitude': lon2d.ravel(),
+                'latitude': lat2d.ravel(),
+            })
 
             # merra grid has some bad values around 0 lat/lon
             # quick fix is to set to zero
@@ -310,7 +311,21 @@ def grid(self):
 
 
 class RelativeHumidity(BaseDerivedVar):
-    """Class to derive the relative humidity from other MERRA2 vars."""
+    """Class to derive the relative humidity from other MERRA2 vars. This is
+    derived as the ratio of mixing ratio to saturation mixing ratio, times 100.
+    Intermediate calculations are done to get saturation vapor pressure,
+    mixing ratio, and saturation mixing ratio. Saturation vapor pressure is
+    calculated using the Tetens formula. Mixing ratio is calculated from
+    specific humidity. Saturation mixing ratio is calculated from saturation
+    vapor pressure.
+
+    References
+    ----------
+    .. https://www.conservationphysics.org/atmcalc/atmoclc2.pdf
+    .. http://snowball.millersville.edu/~adecaria/ESCI241/esci241_lesson06_humidity.pdf
+    .. https://www.weather.gov/media/epz/wxcalc/mixingRatio.pdf
+
+    """
 
     DEPENDENCIES = ('air_temperature', 'specific_humidity', 'surface_pressure')
 
@@ -383,7 +398,16 @@ def derive(air_temperature, specific_humidity, surface_pressure):
 
 
 class DewPoint(BaseDerivedVar):
-    """Class to derive the dew point from other MERRA2 vars."""
+    """Class to derive the dew point from other MERRA2 vars. The dew point is
+    derived from air temperature, specific humidity, and surface pressure.
+    Relative humidity is first derived from these variables, then the dew
+    point is calculated from temperature and relative humidity using the
+    Magnus formula.
+
+    References
+    ----------
+    .. https://journals.ametsoc.org/view/journals/bams/86/2/bams-86-2-225.xml?tab_body=pdf
+    """
 
     DEPENDENCIES = ('air_temperature', 'specific_humidity', 'surface_pressure')
 
@@ -406,7 +430,7 @@ def derive(air_temperature, specific_humidity, surface_pressure):
             Dew point in Celsius.
         """
         logger.info(
-            'Deriving Dew Point from temperature, ' 'humidity, and pressure'
+            'Deriving Dew Point from temperature, humidity, and pressure'
         )
 
         # ensure that Temperature is in C (scale from Kelvin if not)