From 1145d45e028dcd2ba9f1c012ac29adce73136f07 Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Fri, 30 May 2025 01:26:14 +0000
Subject: [PATCH 1/6] Cleanup tabs

---
 cime_config/machines/config_machines.xml | 63 ++++++++++++------------
 1 file changed, 31 insertions(+), 32 deletions(-)

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 82f7f37227a0..743c1e208266 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -3603,32 +3603,32 @@
     <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
-       <executable>mpiexec</executable>
-       <!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
-       <arguments>
-          <arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
-          <arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
-          <arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND}</arg>
-          <arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
-          <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
-       </arguments>
+      <executable>mpiexec</executable>
+      <!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
+      <arguments>
+        <arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
+        <arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
+        <arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND}</arg>
+        <arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
+        <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
+      </arguments>
     </mpirun>
     <module_system type="module" allow_error="true">
-       <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
-       <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
-       <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
-       <cmd_path lang="sh">module</cmd_path>
-       <cmd_path lang="csh">module</cmd_path>
-       <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
-       <modules>
-          <command name="load">cmake/3.30.5</command>
-          <command name="load">oneapi/release/2025.0.5</command>
-        </modules>
-     </module_system>
-     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
-     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
-     <MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
-     <environment_variables>
+      <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
+      <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
+      <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
+      <modules>
+        <command name="load">cmake/3.30.5</command>
+        <command name="load">oneapi/release/2025.0.5</command>
+      </modules>
+    </module_system>
+    <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
+    <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
+    <MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
+    <environment_variables>
         <env name="NETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002</env>
         <env name="PNETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002</env>
         <env name="LD_LIBRARY_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:$ENV{LD_LIBRARY_PATH}</env>
@@ -3638,19 +3638,18 @@
      </environment_variables>
      <environment_variables compiler="oneapi-ifxgpu">
         <env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
-	<env name="MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE"></env>
+        <env name="MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE"></env>
         <env name="MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE"></env>
         <env name="MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE"></env>
         <env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
         <env name="UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS">1</env>
         <env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>
-	<!--<env name="FI_PROVIDER">cxi</env>-->
-	<env name="FI_MR_CACHE_MONITOR">disabled</env>
+        <env name="FI_MR_CACHE_MONITOR">disabled</env>
         <env name="FI_CXI_OVFLOW_BUF_SIZE">8388608</env>
         <env name="PALS_PING_PERIOD">240</env>
         <env name="PALS_RPC_TIMEOUT">240</env>
 
-	<env name="MPIR_CVAR_ENABLE_GPU">1</env>
+        <env name="MPIR_CVAR_ENABLE_GPU">1</env>
         <env name="romio_cb_read">disable</env>
         <env name="romio_cb_write">disable</env>
         <env name="SYCL_CACHE_PERSISTENT">1</env>
@@ -3659,10 +3658,10 @@
         <env name="GPU_TILE_COMPACT">/lus/flare/projects/E3SM_Dec/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
         <env name="RANKS_BIND">list:1-8:9-16:17-24:25-32:33-40:41-48:53-60:61-68:69-76:77-84:85-92:93-100 --gpu-bind list:0.0:0.1:1.0:1.1:2.0:2.1:3.0:3.1:4.0:4.1:5.0:5.1 --mem-bind list:0:0:0:0:0:0:1:1:1:1:1:1</env>
         <env name="ZES_ENABLE_SYSMAN">1</env>
-	<!-- default is ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE: enable this to run 4 MPI/tile or 48 MPI/node
-	<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4</env>-->
-	<!-- <env name="ZE_FLAT_DEVICE_HIERARCHY">FLAT</env>
-	<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>-->
+        <!-- default is ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE: enable this to run 4 MPI/tile or 48 MPI/node
+        <env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4</env>-->
+        <!-- <env name="ZE_FLAT_DEVICE_HIERARCHY">FLAT</env>
+        <env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>-->
     </environment_variables>
     <environment_variables compiler="oneapi-ifx">
         <env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->

From 5b276d8c53c3e9a9330c4744272d230b12dea194 Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Fri, 30 May 2025 01:33:06 +0000
Subject: [PATCH 2/6] Set and export unlimited core file size in debug-mode
 runs

---
 cime_config/machines/config_machines.xml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 743c1e208266..0e9926d6b21d 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -3609,7 +3609,7 @@
         <arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
         <arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
         <arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND}</arg>
-        <arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
+        <arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS} $ENV{RLIMITS}</arg>
         <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
       </arguments>
     </mpirun>
@@ -3635,6 +3635,7 @@
         <env name="PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:$ENV{PATH}</env>
         <env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
         <env name="FI_CXI_CQ_FILL_PERCENT">20</env>
+        <env name="RLIMITS"> </env>
      </environment_variables>
      <environment_variables compiler="oneapi-ifxgpu">
         <env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
@@ -3674,6 +3675,12 @@
         <env name="KMP_AFFINITY">granularity=core,balanced</env>
         <env name="OMP_STACKSIZE">128M</env>
     </environment_variables>
+    <environment_variables DEBUG="TRUE">
+        <env name="RLIMITS">--rlimits CORE</env>
+    </environment_variables>
+    <resource_limits DEBUG="TRUE">
+        <resource name="RLIMIT_CORE">-1</resource>
+    </resource_limits>
     <resource_limits>
         <resource name="RLIMIT_STACK">-1</resource>
     </resource_limits>

From e75d3fea1c0eb5cc22f334f72938a237dac0fb17 Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Fri, 30 May 2025 01:37:25 +0000
Subject: [PATCH 3/6] Load recommended mpich-config module

---
 cime_config/machines/config_machines.xml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 0e9926d6b21d..2e4e4bd26a2d 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -3623,6 +3623,7 @@
       <modules>
         <command name="load">cmake/3.30.5</command>
         <command name="load">oneapi/release/2025.0.5</command>
+        <command name="load">mpich-config/collective-tuning/1024</command>
       </modules>
     </module_system>
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
@@ -3639,9 +3640,6 @@
      </environment_variables>
      <environment_variables compiler="oneapi-ifxgpu">
         <env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
-        <env name="MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE"></env>
-        <env name="MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE"></env>
-        <env name="MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE"></env>
         <env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
         <env name="UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS">1</env>
         <env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>

From b0a2a02452d8dca5242408e5f2984dc62d557c1f Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Fri, 30 May 2025 01:41:55 +0000
Subject: [PATCH 4/6] Set recommended env-vars for threaded runs

---
 cime_config/machines/config_machines.xml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 2e4e4bd26a2d..335d08fc64cf 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -3647,11 +3647,15 @@
         <env name="FI_CXI_OVFLOW_BUF_SIZE">8388608</env>
         <env name="PALS_PING_PERIOD">240</env>
         <env name="PALS_RPC_TIMEOUT">240</env>
+        <env name="SYCL_PI_LEVEL_ZERO_SINGLE_THREAD_MODE">1</env>
+        <env name="SYCL_PI_LEVEL_ZERO_DISABLE_USM_ALLOCATOR">1</env>
+        <env name="SYCL_PI_LEVEL_ZERO_USM_RESIDENT">0x001</env>
+        <env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
+        <env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>
 
         <env name="MPIR_CVAR_ENABLE_GPU">1</env>
         <env name="romio_cb_read">disable</env>
         <env name="romio_cb_write">disable</env>
-        <env name="SYCL_CACHE_PERSISTENT">1</env>
         <env name="GATOR_INITIAL_MB">4000MB</env>
         <env name="GATOR_DISABLE">0</env>
         <env name="GPU_TILE_COMPACT">/lus/flare/projects/E3SM_Dec/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>

From 429b10ff2fb0207edf0f4bc6d229bfcc1230fec9 Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Thu, 5 Jun 2025 03:31:54 +0000
Subject: [PATCH 5/6] Refine jobid_pattern regex on Aurora

---
 cime_config/machines/config_batch.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml
index 09da68864fad..da01cdb66b67 100644
--- a/cime_config/machines/config_batch.xml
+++ b/cime_config/machines/config_batch.xml
@@ -553,6 +553,7 @@
 
     <batch_system MACH="aurora" type="pbspro">
       <batch_submit>/lus/flare/projects/E3SM_Dec/tools/qsub/throttle</batch_submit>
+      <jobid_pattern>(\d+)\.aurora-pbs</jobid_pattern>
       <directives>
         <directive> -l filesystems=home:flare </directive>
       </directives>

From e072fff24703b21f32cca305c2de37ea1ecd7928 Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Wed, 11 Jun 2025 20:47:19 +0000
Subject: [PATCH 6/6] Load collectives module with `--mpilib mpich1024`

---
 cime_config/machines/config_machines.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 335d08fc64cf..0f354cdbccc6 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -3582,7 +3582,7 @@
     <NODENAME_REGEX>aurora-uan-.*</NODENAME_REGEX>
     <OS>LINUX</OS>
     <COMPILERS>oneapi-ifxgpu,oneapi-ifx</COMPILERS>
-    <MPILIBS>mpich</MPILIBS>
+    <MPILIBS>mpich,mpich1024</MPILIBS>
     <PROJECT>E3SM_Dec</PROJECT>
     <SAVE_TIMING_DIR>/lus/flare/projects/E3SM_Dec/performance_archive</SAVE_TIMING_DIR>
     <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
@@ -3623,6 +3623,8 @@
       <modules>
         <command name="load">cmake/3.30.5</command>
         <command name="load">oneapi/release/2025.0.5</command>
+      </modules>
+      <modules mpilib="mpich1024">
         <command name="load">mpich-config/collective-tuning/1024</command>
       </modules>
     </module_system>