Unit/functional test improvements to enable CTSM SystemTests

.gitattributes

@@ -5,16 +5,13 @@
 # This is primarily being implemented to allow users to develop code with any operating system (OS)
 # preferred and mitigates potential problems with end of line (eol) character differences.
 # -----------------------------------------------------------------------------------------------------
-
 ## Set the default end of line behavior (i.e. normalization to `lf` upon commit) for all files git recognizes as text
 * text=auto
-
 # Note that the above *only* applies to newly commited files.  If the file previously existed with a `crlf` end of file
 # and was checked out to local, then git will not change the eol character during check-in (i.e. commit).  For 
 # windows users they will see a warning like this:
 # warning: CRLF will be replaced by LF in functional_unit_testing/allometry/drive_allomtests.py.
 # The file will have its original line endings in your working directory
-
 ## Explicitly declare to git which files should be normalized (i.e. treated as text files)
 *.cdl text
 *.F90 text
@@ -23,6 +20,6 @@
 *.sh text
 *.txt text
 *.xml text
-
 ## Declare to git which file types are binary files and should not have end of line modified
-*.mod binary
+*.mod binary
+testing/test_data/*.nc filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -24,6 +24,9 @@
 *.zip
 *.nc
 
+# Allow netcdf files in the test data directory
+!testing/test_data/*.nc
+
 # Logs and databases #
 ######################
 *.log
@@ -54,4 +57,4 @@ _run/
 # Old Files
 *~
 # Editor specific setting files
-*.vscode
+*.vscode

testing/functional_class_with_drivers.py

@@ -0,0 +1,15 @@
+import os
+from functional_class import FunctionalTest
+
+
+class FunctionalTestWithDrivers(FunctionalTest):
+    """Class for running FATES functional tests with driver files"""
+
+    def __init__(self, datm_file: str, *args):
+
+        # Check that datm exists and save its absolute path
+        self.datm_file = os.path.abspath(datm_file)
+        if not os.path.exists(self.datm_file):
+            raise FileNotFoundError(f"datm_file not found: '{self.datm_file}'")
+
+        super().__init__(*args)

testing/functional_testing/allometry/allometry_test.py

@@ -6,7 +6,8 @@
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
-from utils import round_up, get_color_palette, blank_plot
+from utils import round_up
+from utils_plotting import blank_plot, get_color_palette
 from functional_class import FunctionalTest
 
 

testing/functional_testing/fire/fuel/fuel_test.py

@@ -5,16 +5,17 @@
 import numpy as np
 import xarray as xr
 import matplotlib.pyplot as plt
-from functional_class import FunctionalTest
+from functional_class_with_drivers import FunctionalTestWithDrivers
 
 
-class FuelTest(FunctionalTest):
+class FuelTest(FunctionalTestWithDrivers):
     """Fuel test class"""
 
     name = "fuel"
 
     def __init__(self, test_dict):
         super().__init__(
+            test_dict["datm_file"],
             FuelTest.name,
             test_dict["test_dir"],
             test_dict["test_exe"],

testing/functional_testing/fire/ros/ros_test.py

@@ -7,7 +7,7 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 from functional_class import FunctionalTest
-from utils import blank_plot
+from utils_plotting import blank_plot
 
 COLORS = ["#793922", "#6B8939", "#99291F", "#CC9728", "#2C778A"]
 CM_TO_FT = 30.48

testing/functional_testing/fire/shr/SyntheticFuelModels.F90

@@ -178,7 +178,7 @@ integer function FuelModelPosition(this, fuel_model_index)
       end if
     end do
     write(*, '(a, i2, a)') "Cannot find the fuel model index ", fuel_model_index, "."
-    stop
+    call abort()
 
   end function FuelModelPosition
 

testing/functional_testing/math_utils/math_utils_test.py

@@ -5,7 +5,7 @@
 import xarray as xr
 import numpy as np
 import matplotlib.pyplot as plt
-from utils import get_color_palette
+from utils_plotting import get_color_palette
 from functional_class import FunctionalTest
 
 

testing/functional_testing/patch/patch_test.py

@@ -6,7 +6,8 @@
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
-from utils import round_up, get_color_palette, blank_plot
+from utils import round_up
+from utils_plotting import blank_plot, get_color_palette
 from functional_class import FunctionalTest
 
 

testing/functional_tests.cfg

@@ -17,7 +17,8 @@ test_dir = fates_fuel_ftest
 test_exe = FATES_fuel_exe
 out_file = fuel_out.nc
 use_param_file = True
-other_args = ['../testing/test_data/BONA_datm.nc']
+datm_file = ../testing/test_data/BONA_datm.nc
+other_args = []
 
 [ros]
 test_dir = fates_ros_ftest

testing/load_functional_tests.py

@@ -1,6 +1,7 @@
 # add testing subclasses here
 
 from functional_class import FunctionalTest
+from functional_class_with_drivers import FunctionalTestWithDrivers
 from functional_testing.allometry.allometry_test import AllometryTest
 from functional_testing.math_utils.math_utils_test import QuadraticTest
 from functional_testing.fire.fuel.fuel_test import FuelTest 

testing/run_functional_tests.py

@@ -28,9 +28,11 @@
 """
 import os
 import argparse
+import subprocess
 import matplotlib.pyplot as plt
 
 from build_fortran_tests import build_tests, build_exists
+from functional_class_with_drivers import FunctionalTestWithDrivers
 from path_utils import add_cime_lib_to_path
 from utils import copy_file, create_nc_from_cdl, config_to_dict, parse_test_list
 
@@ -39,12 +41,20 @@
 
 add_cime_lib_to_path()
 
-from CIME.utils import run_cmd_no_fail
+from CIME.utils import run_cmd
 
 # constants for this script
-_DEFAULT_CONFIG_FILE = "functional_tests.cfg"
-_DEFAULT_CDL_PATH = os.path.abspath("../parameter_files/fates_params_default.cdl")
-_CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../")
+_FILE_DIR = os.path.dirname(__file__)
+_DEFAULT_CONFIG_FILE = os.path.join(_FILE_DIR, "functional_tests.cfg")
+_DEFAULT_CDL_PATH = os.path.abspath(
+    os.path.join(
+        _FILE_DIR,
+        os.pardir,
+        "parameter_files",
+        "fates_params_default.cdl",
+    )
+)
+_CMAKE_BASE_DIR = os.path.join(_FILE_DIR, os.pardir)
 _TEST_SUB_DIR = "testing"
 
 
@@ -74,6 +84,13 @@ def commandline_args():
         "parameter_files directory.\n",
     )
 
+    parser.add_argument(
+        "--config-file",
+        type=str,
+        default=_DEFAULT_CONFIG_FILE,
+        help=f"Configuration file where test list is defined. Default: '{_DEFAULT_CONFIG_FILE}'",
+    )
+
     parser.add_argument(
         "-b",
         "--build-dir",
@@ -179,6 +196,12 @@ def check_arg_validity(args):
             )
         check_build_dir(args.build_dir, args.test_dict)
 
+    # Check that config file exists and is a file
+    if not os.path.exists(args.config_file):
+        raise FileNotFoundError(args.config_file)
+    if not os.path.isfile(args.config_file):
+        raise RuntimeError(f"config 'file' is a directory: '{args.config_file}'")
+
 
 def check_param_file(param_file):
     """Checks to see if param_file exists and is of the correct form (.nc or .cdl)
@@ -196,7 +219,7 @@ def check_param_file(param_file):
             None, "Must supply parameter file with .cdl or .nc ending."
         )
     if not os.path.isfile(param_file):
-        raise argparse.ArgumentError(None, f"Cannot find file {param_file}.")
+        raise FileNotFoundError(param_file)
 
 
 def check_build_dir(build_dir, test_dict):
@@ -295,8 +318,11 @@ def run_functional_tests(
     if run_executables:
         print("Running executables")
         for _, test in test_dict.items():
-            # prepend parameter file (if required) to argument list
             args = test.other_args
+            # prepend datm file (if required) to argument list
+            if isinstance(test, FunctionalTestWithDrivers) and test.datm_file:
+                args.insert(0, test.datm_file)
+            # prepend parameter file (if required) to argument list
             if test.use_param_file:
                 args.insert(0, param_file)
             # run
@@ -386,22 +412,41 @@ def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args):
     run_command.extend(args)
 
     os.chdir(run_dir)
-    out = run_cmd_no_fail(" ".join(run_command), combine_output=True)
+    cmd = " ".join(run_command)
+    stat, out, _ = run_cmd(cmd, combine_output=True)
+    if stat:
+        print(out)
+        raise subprocess.CalledProcessError(stat, cmd, out)
     print(out)
 
 
+def get_test_subclasses(*argv):
+    """
+    Given a FunctionalTest* class, find all its test subclasses. Do not include child
+    FunctionalTest* classes.
+    """
+    test_subclasses = []
+    for ftest_class in argv:
+        test_subclasses += [x for x in ftest_class.__subclasses__() if hasattr(x, "name")]
+    return test_subclasses
+
+
 def main():
     """Main script
     Reads in command-line arguments and then runs the tests.
     """
 
-    full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE)
-    subclasses = FunctionalTest.__subclasses__()
-
     args = commandline_args()
+
+    full_test_dict = config_to_dict(args.config_file)
     config_dict = parse_test_list(full_test_dict, args.test_list)
 
     test_dict = {}
+
+    # Get all the possible test subclasses.
+    subclasses = get_test_subclasses(FunctionalTest, FunctionalTestWithDrivers)
+
+    # Associate each test in the config file with the appropriate test subclass
     for name in config_dict.keys():
         test_class = list(filter(lambda subclass: subclass.name == name, subclasses))[
             0

testing/run_unit_tests.py

@@ -29,8 +29,9 @@
 from CIME.utils import run_cmd_no_fail  # pylint: disable=wrong-import-position,import-error,wrong-import-order
 
 # constants for this script
-_CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../")
-_DEFAULT_CONFIG_FILE = "unit_tests.cfg"
+_FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+_CMAKE_BASE_DIR = os.path.join(_FILE_DIR, os.pardir)
+_DEFAULT_CONFIG_FILE = os.path.join(_FILE_DIR, "unit_tests.cfg")
 _TEST_SUB_DIR = "testing"
 
 
@@ -58,6 +59,13 @@ def commandline_args():
         "Will be created if it does not exist.\n",
     )
 
+    parser.add_argument(
+        "--config-file",
+        type=str,
+        default=_DEFAULT_CONFIG_FILE,
+        help=f"Configuration file where test list is defined. Default: '{_DEFAULT_CONFIG_FILE}'",
+    )
+
     parser.add_argument(
         "--make-j",
         type=int,
@@ -129,9 +137,8 @@ def main():
     Reads in command-line arguments and then runs the tests.
     """
 
-    full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE)
-
     args = commandline_args()
+    full_test_dict = config_to_dict(args.config_file)
     test_dict = parse_test_list(full_test_dict, args.test_list)
 
     run_unit_tests(

testing/testing_shr/FatesArgumentUtils.F90

@@ -24,7 +24,7 @@ function command_line_arg(arg_position)
 
       if (n_args < arg_position) then
         write(*, '(a, i2, a, i2)') "Incorrect number of arguments: ", n_args, ". Should be at least", arg_position, "."
-        stop
+        call abort()
       end if 
 
       call get_command_argument(arg_position, length=arglen)

testing/testing_shr/FatesFactoryMod.F90

@@ -520,7 +520,7 @@ subroutine CreateTestPatchList(patch, heights, dbhs)
     if (present(dbhs)) then 
       if (size(heights) /= size(dbhs)) then 
         write(*, '(a)') "Size of heights array must match size of dbh array."
-        stop
+        call abort()
       end if 
     end if 
 

testing/testing_shr/FatesUnitTestIOMod.F90

@@ -106,7 +106,7 @@ subroutine Check(status)
 
     if (status /= nf90_noerr) then
       write(*,*) trim(nf90_strerror(status))
-      stop
+      call abort()
     end if
 
   end subroutine Check
@@ -134,11 +134,11 @@ subroutine OpenNCFile(nc_file, ncid, fmode)
         call Check(nf90_create(trim(nc_file), NF90_CLOBBER, ncid))
       case DEFAULT
         write(*,*) 'Need to specify read, write, or readwrite'
-        stop
+        call abort()
       end select
     else
       write(*,*) 'Problem reading file'
-      stop
+      call abort()
     end if
 
   end subroutine OpenNCFile
@@ -480,7 +480,7 @@ subroutine RegisterVar(ncid, var_name, dimID, type, att_names, atts, num_atts, v
       nc_type = NF90_CHAR
     else
       write(*, *) "Must pick correct type"
-      stop
+      call abort()
     end if
 
     call Check(nf90_def_var(ncid, var_name, nc_type, dimID, varID))

testing/testing_shr/FatesUnitTestParamReaderMod.F90

@@ -89,7 +89,7 @@ subroutine ReadParameters(this, fates_params)
           case default
             write(*, '(a,a)') 'dimension shape:', dimension_shape
             write(*, '(a)') 'unsupported number of dimensions reading parameters.'
-            stop
+            call abort()
         end select
     end do
 

testing/testing_shr/SyntheticPatchTypes.F90

@@ -163,7 +163,7 @@ integer function PatchDataPosition(this, patch_id, patch_name)
     ! can't supply both
     if (present(patch_id) .and. present(patch_name)) then 
       write(*, '(a)') "Can only supply either a patch_id or a patch_name - not both"
-      stop
+      call abort()
     end if 
 
     do i = 1, this%num_patches
@@ -179,11 +179,11 @@ integer function PatchDataPosition(this, patch_id, patch_name)
         end if
       else 
         write(*, '(a)') "Must supply either a patch_id or a patch_name."
-        stop
+        call abort()
       end if
     end do
     write(*, '(a)') "Cannot find the synthetic patch type supplied"
-    stop
+    call abort()
 
   end function PatchDataPosition