Skip to content

Commit 17a0cbd

Browse files
nusbaumenusbaume
authored
Merge pull request #1260 from sjsprecious/fix_rrtmgp_gpu
cam6_4_072: Fix broken RRTMGP GPU tests
2 parents d7080a1 + ce826fd commit 17a0cbd

File tree

4 files changed

+99
-4
lines changed

4 files changed

+99
-4
lines changed

‎cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_default/shell_commands‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
./xmlchange ROOTPE='0'
44
./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
55
./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
6-
./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -rad rrtmg' --append
6+
./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -rad rrtmgp_gpu ' --append
77
./xmlchange TIMER_DETAIL='6'
88
./xmlchange TIMER_LEVEL='999'
99
./xmlchange GPU_TYPE=a100

‎cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_pcols760/shell_commands‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
./xmlchange ROOTPE='0'
44
./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
55
./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
6-
./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -rad rrtmg -pcols 760 ' --append
6+
./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -rad rrtmgp_gpu -pcols 760 ' --append
77
./xmlchange TIMER_DETAIL='6'
88
./xmlchange TIMER_LEVEL='999'
99
./xmlchange GPU_TYPE=a100

‎doc/ChangeLog‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,70 @@
11
===============================================================
22

3+
Tag name: cam6_4_072
4+
Originator(s): sjsprecious
5+
Date: 28 February 2025
6+
One-line Summary: Fix broken RRTMGP GPU tests
7+
Github PR URL: https://github.com/ESCOMP/CAM/pull/1260
8+
9+
Purpose of changes (include the issue number and title text for each relevant GitHub issue):
10+
11+
Fixes #997 - RRTMGP not working with GPUs on derecho
12+
13+
Describe any changes made to build system: N/A
14+
15+
Describe any changes made to the namelist: N/A
16+
17+
List any changes to the defaults for the boundary datasets: N/A
18+
19+
Describe any substantial timing or memory changes: N/A
20+
21+
Code reviewed by: nusbaume
22+
23+
List all files eliminated: N/A
24+
25+
List all files added and what they do: N/A
26+
27+
List all existing files that have been modified, and describe the changes:
28+
29+
M cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_default/shell_commands
30+
M cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_pcols760/shell_commands
31+
- Update Derecho GPU regression test to use RRTMGP.
32+
33+
M src/physics/rrtmgp/radiation.F90
34+
- Update OpenACC calls to allow RRTMGP to run on Derecho's GPUs.
35+
36+
If there were any failures reported from running test_driver.sh on any test
37+
platform, and checkin with these failures has been OK'd by the gatekeeper,
38+
then copy the lines from the td.*.status files for the failed tests to the
39+
appropriate machine below. All failed tests must be justified.
40+
41+
derecho/intel/aux_cam:
42+
43+
ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL)
44+
SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: DIFF)
45+
- pre-existing failures due to HEMCO not having reproducible results (issues #1018 and #856)
46+
47+
SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: FAIL)
48+
SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: FAIL)
49+
- pre-existing failures due to build-namelist error requiring CLM/CTSM external update
50+
51+
derecho/nvhpc/aux_cam:
52+
53+
ERS_Ln9.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_gpu_default (Overall: DIFF)
54+
- Expected namelist and baseline answer changes due to the addition of RRTMGP.
55+
56+
izumi/nag/aux_cam: ALL PASS
57+
58+
izumi/gnu/aux_cam: ALL PASS
59+
60+
CAM tag used for the baseline comparison tests if different than previous
61+
tag:
62+
63+
Summarize any changes to answers: b4b
64+
65+
===============================================================
66+
===============================================================
67+
368
Tag name: cam6_4_071
469
Originator(s): mwaxmonsky
570
Date: 26 February 2025

‎src/physics/rrtmgp/radiation.F90‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,9 +1170,13 @@ subroutine radiation_tend( &
11701170

11711171
! Compute the gas optics (stored in atm_optics_sw).
11721172
! toa_flux is the reference solar source from RRTMGP data.
1173+
!$acc data copyin(kdist_sw,pmid_day,pint_day,t_day,gas_concs_sw) &
1174+
!$acc copy(atm_optics_sw) &
1175+
!$acc copyout(toa_flux)
11731176
errmsg = kdist_sw%gas_optics( &
11741177
pmid_day, pint_day, t_day, gas_concs_sw, atm_optics_sw, &
11751178
toa_flux)
1179+
!$acc end data
11761180
call stop_on_err(errmsg, sub, 'kdist_sw%gas_optics')
11771181

11781182
! Scale the solar source
@@ -1190,6 +1194,15 @@ subroutine radiation_tend( &
11901194
if (nday > 0) then
11911195

11921196
! Increment the gas optics (in atm_optics_sw) by the aerosol optics in aer_sw.
1197+
!$acc data copyin(coszrs_day, toa_flux, alb_dir, alb_dif, &
1198+
!$acc atm_optics_sw, atm_optics_sw%tau, &
1199+
!$acc atm_optics_sw%ssa, atm_optics_sw%g, &
1200+
!$acc aer_sw, aer_sw%tau, &
1201+
!$acc aer_sw%ssa, aer_sw%g, &
1202+
!$acc cloud_sw, cloud_sw%tau, &
1203+
!$acc cloud_sw%ssa, cloud_sw%g) &
1204+
!$acc copy(fswc, fswc%flux_net,fswc%flux_up,fswc%flux_dn, &
1205+
!$acc fsw, fsw%flux_net, fsw%flux_up, fsw%flux_dn)
11931206
errmsg = aer_sw%increment(atm_optics_sw)
11941207
call stop_on_err(errmsg, sub, 'aer_sw%increment')
11951208

@@ -1208,7 +1221,7 @@ subroutine radiation_tend( &
12081221
atm_optics_sw, top_at_1, coszrs_day, toa_flux, &
12091222
alb_dir, alb_dif, fsw)
12101223
call stop_on_err(errmsg, sub, 'all-sky rte_sw')
1211-
1224+
!$acc end data
12121225
end if
12131226

12141227
! Transform RRTMGP outputs to CAM outputs and compute heating rates.
@@ -1264,15 +1277,31 @@ subroutine radiation_tend( &
12641277
call rrtmgp_set_gases_lw(icall, state, pbuf, nlay, gas_concs_lw)
12651278

12661279
! Compute the gas optics and Planck sources.
1280+
!$acc data copyin(kdist_lw, pmid_rad, pint_rad, &
1281+
!$acc t_rad, t_sfc, gas_concs_lw) &
1282+
!$acc copy(atm_optics_lw, atm_optics_lw%tau, &
1283+
!$acc sources_lw, sources_lw%lay_source, &
1284+
!$acc sources_lw%sfc_source, sources_lw%lev_source_inc, &
1285+
!$acc sources_lw%lev_source_dec, sources_lw%sfc_source_jac)
12671286
errmsg = kdist_lw%gas_optics( &
12681287
pmid_rad, pint_rad, t_rad, t_sfc, gas_concs_lw, &
12691288
atm_optics_lw, sources_lw)
1289+
!$acc end data
12701290
call stop_on_err(errmsg, sub, 'kdist_lw%gas_optics')
12711291

12721292
! Set LW aerosol optical properties in the aer_lw object.
12731293
call rrtmgp_set_aer_lw(icall, state, pbuf, aer_lw)
12741294

12751295
! Increment the gas optics by the aerosol optics.
1296+
!$acc data copyin(atm_optics_lw, atm_optics_lw%tau, &
1297+
!$acc aer_lw, aer_lw%tau, &
1298+
!$acc cloud_lw, cloud_lw%tau, &
1299+
!$acc sources_lw, sources_lw%lay_source, &
1300+
!$acc sources_lw%sfc_source, sources_lw%lev_source_inc, &
1301+
!$acc sources_lw%lev_source_dec, sources_lw%sfc_source_Jac, &
1302+
!$acc emis_sfc) &
1303+
!$acc copy(flwc, flwc%flux_net, flwc%flux_up, flwc%flux_dn, &
1304+
!$acc flw, flw%flux_net, flw%flux_up, flw%flux_dn)
12761305
errmsg = aer_lw%increment(atm_optics_lw)
12771306
call stop_on_err(errmsg, sub, 'aer_lw%increment')
12781307

@@ -1287,7 +1316,8 @@ subroutine radiation_tend( &
12871316
! Compute all-sky LW fluxes
12881317
errmsg = rte_lw(atm_optics_lw, top_at_1, sources_lw, emis_sfc, flw)
12891318
call stop_on_err(errmsg, sub, 'all-sky rte_lw')
1290-
1319+
!$acc end data
1320+
12911321
! Transform RRTMGP outputs to CAM outputs and compute heating rates.
12921322
call set_lw_diags()
12931323

0 commit comments

Comments
 (0)