Add SystemTests for FATES unit/functional testing

.gitmodules

@@ -27,8 +27,8 @@
 #
 [submodule "fates"]
 path = src/fates
-url = https://github.com/NGEET/fates
-fxtag = sci.1.84.0_api.40.0.0
+url = https://github.com/samsrabin/fates
+fxtag = e0037facd11bdc8866af1cf4578cdbc49269c8bd  # fix_testing_ssr: Remove datm_file_url.
 fxrequired = AlwaysRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NGEET/fates

cime_config/SystemTests/functionalfates.py

@@ -0,0 +1,39 @@
+"""
+Implementation of the FATES functional tests.
+
+This "system" test runs FATES's Fortran functional tests. We're abusing the system test
+infrastructure to run these, so that a run of the test suite can result in the unit tests
+being run as well.
+
+Grid and compset are irrelevant for this test type.
+"""
+
+import os
+import sys
+from CIME.SystemTests.funit import FUNIT
+from CIME.XML.standard_module_setup import *
+import systemtest_utils as stu
+
+_FATES_TESTING_PYTHON = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "src", "fates", "testing"
+)
+
+logger = logging.getLogger(__name__)
+
+
+class FUNCTIONALFATES(FUNIT):
+    def __init__(self, case):
+        FUNIT.__init__(self, case)
+
+    def run_phase(self):
+        tool_path = os.path.join(_FATES_TESTING_PYTHON, "run_functional_tests.py")
+        build_dir = os.path.join("bld", "fates_unit_tests")
+        run_dir = "run"
+        cmd = f"{tool_path} -b {build_dir} -r {run_dir} --save-figs"
+
+        stu.run_python_script(
+            self._get_caseroot(),
+            "ctsm_pylib",
+            cmd,
+            tool_path,
+        )

cime_config/SystemTests/funitfates.py

@@ -0,0 +1,38 @@
+"""
+Implementation of the FATES unit tests.
+
+This "system" test runs FATES's Fortran unit tests. We're abusing the system test
+infrastructure to run these, so that a run of the test suite can result in the unit tests
+being run as well.
+
+Grid and compset are irrelevant for this test type.
+"""
+
+import os
+import sys
+from CIME.SystemTests.funit import FUNIT
+from CIME.XML.standard_module_setup import *
+import systemtest_utils as stu
+
+_FATES_TESTING_PYTHON = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "src", "fates", "testing"
+)
+
+logger = logging.getLogger(__name__)
+
+
+class FUNITFATES(FUNIT):
+    def __init__(self, case):
+        FUNIT.__init__(self, case)
+
+    def run_phase(self):
+        tool_path = os.path.join(_FATES_TESTING_PYTHON, "run_unit_tests.py")
+        build_dir = os.path.join("bld", "fates_unit_tests")
+        cmd = f"{tool_path} -b {build_dir}"
+
+        stu.run_python_script(
+            self._get_caseroot(),
+            "ctsm_pylib",
+            cmd,
+            tool_path,
+        )

cime_config/SystemTests/systemtest_utils.py

@@ -43,6 +43,9 @@ def cmds_to_run_via_conda(caseroot, conda_run_call, command):
 
 def run_python_script(caseroot, this_conda_env, command_in, tool_path):
 
+    if not isinstance(command_in, str):
+        raise TypeError(f"command_in must be a string, not {type(command_in)}")
+
     # First, try with "conda run -n"
     command = cmds_to_run_via_conda(caseroot, f"conda run -n {this_conda_env}", command_in)
 

cime_config/config_tests.xml

@@ -15,6 +15,24 @@ This defines various CTSM-specific system tests
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
+  <test NAME="FUNITFATES">
+    <DESC>Run FATES's Fortran unit tests. Grid and compset (and most case settings) are ignored.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_N>11</STOP_N>
+    <CHECK_TIMING>FALSE</CHECK_TIMING>
+    <DOUT_S>FALSE</DOUT_S>
+  </test>
+
+  <test NAME="FUNCTIONALFATES">
+    <DESC>Run FATES's Fortran functional tests. Grid and compset (and most case settings) are ignored.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_N>11</STOP_N>
+    <CHECK_TIMING>FALSE</CHECK_TIMING>
+    <DOUT_S>FALSE</DOUT_S>
+  </test>
+
   <test NAME="MKSURFDATAESMF">
     <DESC>Build and run the mksurfdata_esmf tool to generate a new fsurdat; then run the CTSM with this fsurdat</DESC>
     <INFO_DBUG>1</INFO_DBUG>

cime_config/testdefs/testlist_clm.xml

@@ -13,6 +13,7 @@
         rxcropmaturity: Short tests to be run during development related to prescribed crop calendars
         matrixcn: Tests exercising the matrix-CN capability
         aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy.
+        unit_tests: tests that just exercise FORTRAN unit tests
 -->
 <testlist version="2.0">
   <test name="ERI_D_Ld9" grid="f10_f10_mg37" compset="I1850Clm60Bgc" testmods="clm/default">
@@ -4130,13 +4131,45 @@
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>
       <machine name="izumi" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="unit_tests"/>
+      <machine name="izumi" compiler="intel" category="unit_tests"/>
     </machines>
     <options>
       <option name="wallclock">00:30:00</option>
       <option name="comment">This test runs CTSM's Fortran unit tests. We're abusing the system test infrastructure to run these, so that a run of the test suite results in the unit tests being run as well. Grid and compset are irrelevant here, except that compset must be one that includes CTSM in order for CIME to find the test definition.</option>
     </options>
   </test>
 
+  <test name="FUNITFATES_P1x1" grid="f10_f10_mg37" compset="I2000Clm50Sp">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="izumi" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="fates"/>
+      <machine name="izumi" compiler="intel" category="fates"/>
+      <machine name="derecho" compiler="intel" category="unit_tests"/>
+      <machine name="izumi" compiler="intel" category="unit_tests"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:30:00</option>
+      <option name="comment">This test runs FATES's Fortran unit tests. We're abusing the system test infrastructure to run these, so that a run of the test suite results in the unit tests being run as well. Grid and compset are irrelevant here, except that compset must be one that includes CTSM in order for CIME to find the test definition.</option>
+    </options>
+  </test>
+
+  <test name="FUNCTIONALFATES_P1x1" grid="f10_f10_mg37" compset="I2000Clm50Sp">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="izumi" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="fates"/>
+      <machine name="izumi" compiler="intel" category="fates"/>
+      <machine name="derecho" compiler="intel" category="unit_tests"/>
+      <machine name="izumi" compiler="intel" category="unit_tests"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:30:00</option>
+      <option name="comment">This test runs FATES's Fortran functional tests. We're abusing the system test infrastructure to run these, so that a run of the test suite results in the unit tests being run as well. Grid and compset are irrelevant here, except that compset must be one that includes CTSM in order for CIME to find the test definition.</option>
+    </options>
+  </test>
+
   <test name="RXCROPMATURITY_Lm61" grid="f09_g17" compset="IHistClm60BgcCrop" testmods="clm/cropMonthOutput">
     <machines>
       <machine name="derecho" compiler="intel" category="ctsm_sci"/>

python/ctsm/path_utils.py

@@ -105,6 +105,24 @@ def add_ctsm_systests_to_path(standalone_only=False):
     sys.path.insert(1, ctsm_systest_dir)
 
 
+def add_fates_testing_to_path():
+    """
+    Adds the FATES testing dir to the python path, to allow importing Python modules from there
+    """
+    fates_testing_path = os.path.join(
+        os.path.dirname(__file__),
+        os.pardir,
+        os.pardir,
+        "src",
+        "fates",
+        "testing",
+    )
+    if not os.path.exists(fates_testing_path):
+        raise RuntimeError(f"src_path not found: '{fates_testing_path}'")
+    prepend_to_python_path(fates_testing_path)
+    sys.path.insert(1, fates_testing_path)
+
+
 # ========================================================================
 # Private functions
 # ========================================================================

python/ctsm/run_sys_tests.py

@@ -24,7 +24,7 @@
 )
 from ctsm.machine_defaults import MACHINE_DEFAULTS
 from ctsm.os_utils import make_link
-from ctsm.path_utils import path_to_ctsm_root
+from ctsm.path_utils import path_to_ctsm_root, add_fates_testing_to_path
 from ctsm.joblauncher.job_launcher_factory import JOB_LAUNCHER_NOBATCH
 
 logger = logging.getLogger(__name__)
@@ -749,6 +749,14 @@ def _check_py_env(test_attributes):
         except ModuleNotFoundError as err:
             raise ModuleNotFoundError("modify_fsurdat" + err_msg) from err
 
+    # Check requirements for FUNCTIONALFATES, if needed
+    if any("FUNCTIONALFATES" in t for t in test_attributes):
+        add_fates_testing_to_path()
+        try:
+            import run_functional_tests
+        except ModuleNotFoundError as err:
+            raise ModuleNotFoundError("FATES run_functional_tests" + err_msg) from err
+
     # Check requirements for RXCROPMATURITY, if needed
     if any("RXCROPMATURITY" in t for t in test_attributes):
         try: