From e988f4f3d14eef1b5aa31ee0d0230c1f1f685c5a Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Wed, 17 Jul 2024 12:01:36 -0600 Subject: [PATCH 1/7] Wrap MPAS longitude values to [0,2pi) range This commit addresses issues that were seen when using ncdata with longitudes in the [-pi, pi] range in cases with full physics. This would cause the atmosphere to become unphysical and runs to fail. This change is made just after the longitude values are read to ensure they are correct for the remainder of execution. --- src/dynamics/mpas/driver/cam_mpas_subdriver.F90 | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 index 676bacd4af..7b9276acf3 100644 --- a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 +++ b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 @@ -884,15 +884,16 @@ subroutine cam_mpas_read_static(fh_ini, endrun) use pio, only : file_desc_t - use mpas_kind_types, only : StrKIND + use mpas_kind_types, only : StrKIND, RKIND use mpas_io_streams, only : MPAS_createStream, MPAS_closeStream, MPAS_streamAddField, MPAS_readStream use mpas_derived_types, only : MPAS_IO_READ, MPAS_IO_NETCDF, MPAS_Stream_type, MPAS_pool_type, & field0DReal, field1DReal, field2DReal, field3DReal, field1DInteger, field2DInteger, & MPAS_STREAM_NOERR use mpas_pool_routines, only : MPAS_pool_get_subpool, MPAS_pool_get_field, MPAS_pool_create_pool, MPAS_pool_destroy_pool, & - MPAS_pool_add_config + MPAS_pool_add_config, MPAS_pool_get_dimension, MPAS_pool_get_array use mpas_dmpar, only : MPAS_dmpar_exch_halo_field use mpas_stream_manager, only : postread_reindex + use mpas_constants, only : pii ! Arguments type (file_desc_t), pointer :: fh_ini @@ -929,8 +930,13 @@ subroutine cam_mpas_read_static(fh_ini, endrun) type (MPAS_Stream_type) :: mesh_stream + integer, pointer :: nCells + real(kind=RKIND), dimension(:), pointer :: lonCell_arr + nullify(cell_gradient_coef_x) nullify(cell_gradient_coef_y) + nullify(nCells) + nullify(lonCell_arr) call MPAS_createStream(mesh_stream, domain_ptr % ioContext, 'not_used', MPAS_IO_NETCDF, MPAS_IO_READ, & pio_file_desc=fh_ini, ierr=ierr) @@ -1171,6 +1177,12 @@ subroutine cam_mpas_read_static(fh_ini, endrun) call endrun(subname//': FATAL: Failed to close static input stream.') end if + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_array(meshPool, 'lonCell', lonCell_arr) + + ! Ensure longitudes w/i [0, 2*pi) range + lonCell_arr(:) = lonCell_arr(:) - (2.*pii) * floor(lonCell_arr(:) / (2.*pii)) + ! ! Perform halo updates for all decomposed fields (i.e., fields with ! an outermost dimension of nCells, nVertices, or nEdges) From 7edc1db74a07545d8cfeca0b020b51399a6c9e52 Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Thu, 18 Jul 2024 12:22:10 -0600 Subject: [PATCH 2/7] Add initial ChangeLog after making PR NOTE: this is before the 'aux_cam' tests have been run --- doc/ChangeLog | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 5ea328d2c8..ab83bacb34 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,69 @@ =============================================================== +Tag name: +Originator(s): gdicker +Date: Jul 18, 2024 +One-line Summary: Address run failures due to MPAS-A longitudes +Github PR URL: https://github.com/ESCOMP/CAM/pull/1095 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + - Wrap MPAS-A longitudes to [0,2pi) range: https://github.com/ESCOMP/CAM/issues/1094 + +Describe any changes made to build system: N/A + +Describe any changes made to the namelist: N/A + +List any changes to the defaults for the boundary datasets: N/A + +Describe any substantial timing or memory changes: N/A + +Code reviewed by: + +List all files eliminated: N/A + +List all files added and what they do: N/A + +List all existing files that have been modified, and describe the changes: +M src/dynamics/mpas/driver/cam_mpas_subdriver.F90 + - Modifies cam_mpas_read_static to ensure lonCell values are in [0,2pi) range + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +derecho/intel/aux_cam: + +izumi/nag/aux_cam: + +izumi/gnu/aux_cam: + +CAM tag used for the baseline comparison tests if different than previous +tag: + +Summarize any changes to answers, i.e., +- what code configurations: +- what platforms/compilers: +- nature of change (roundoff; larger than roundoff but same climate; new + climate): + +If bitwise differences were observed, how did you show they were no worse +than roundoff? + +If this tag changes climate describe the run(s) done to evaluate the new +climate in enough detail that it(they) could be reproduced, i.e., +- source tag (all code used must be in the repository): +- platform/compilers: +- configure commandline: +- build-namelist command (or complete namelist): +- MSS location of output: + +MSS location of control simulations used to validate new climate: + +URL for AMWG diagnostics output used to validate new climate: + +=============================================================== + Tag name: cam6_3_148 Originator(s): brianpm, courtneyp, eaton Date: Wed 21 Feb 2024 From f3dfa5173aea7d1c4a90dafd53f3e3fe2e0660a9 Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Fri, 19 Jul 2024 16:27:35 -0600 Subject: [PATCH 3/7] Remove unused nCells from cam_mpas_read_static Previous versions of the lonCell_arr adjustment used an explicit loop. Without this loop, the nCells variable and the routine to fetch it are no longer needed. --- src/dynamics/mpas/driver/cam_mpas_subdriver.F90 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 index 7b9276acf3..d9a1b63144 100644 --- a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 +++ b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 @@ -890,7 +890,7 @@ subroutine cam_mpas_read_static(fh_ini, endrun) field0DReal, field1DReal, field2DReal, field3DReal, field1DInteger, field2DInteger, & MPAS_STREAM_NOERR use mpas_pool_routines, only : MPAS_pool_get_subpool, MPAS_pool_get_field, MPAS_pool_create_pool, MPAS_pool_destroy_pool, & - MPAS_pool_add_config, MPAS_pool_get_dimension, MPAS_pool_get_array + MPAS_pool_add_config, MPAS_pool_get_array use mpas_dmpar, only : MPAS_dmpar_exch_halo_field use mpas_stream_manager, only : postread_reindex use mpas_constants, only : pii @@ -930,12 +930,10 @@ subroutine cam_mpas_read_static(fh_ini, endrun) type (MPAS_Stream_type) :: mesh_stream - integer, pointer :: nCells real(kind=RKIND), dimension(:), pointer :: lonCell_arr nullify(cell_gradient_coef_x) nullify(cell_gradient_coef_y) - nullify(nCells) nullify(lonCell_arr) call MPAS_createStream(mesh_stream, domain_ptr % ioContext, 'not_used', MPAS_IO_NETCDF, MPAS_IO_READ, & @@ -1177,7 +1175,6 @@ subroutine cam_mpas_read_static(fh_ini, endrun) call endrun(subname//': FATAL: Failed to close static input stream.') end if - call mpas_pool_get_dimension(meshPool, 'nCells', nCells) call mpas_pool_get_array(meshPool, 'lonCell', lonCell_arr) ! Ensure longitudes w/i [0, 2*pi) range From 288f3ed05d3e3e8d475bc39f967ab420f87bb937 Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Tue, 23 Jul 2024 18:07:43 -0600 Subject: [PATCH 4/7] Add kind to number literal in lonCell_arr adjustment Makes code consistent with CAM coding standards: 'All variables of type real must have a specified kind, including literals.' --- src/dynamics/mpas/driver/cam_mpas_subdriver.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 index d9a1b63144..376389efef 100644 --- a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 +++ b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 @@ -1178,7 +1178,7 @@ subroutine cam_mpas_read_static(fh_ini, endrun) call mpas_pool_get_array(meshPool, 'lonCell', lonCell_arr) ! Ensure longitudes w/i [0, 2*pi) range - lonCell_arr(:) = lonCell_arr(:) - (2.*pii) * floor(lonCell_arr(:) / (2.*pii)) + lonCell_arr(:) = lonCell_arr(:) - (2._RKIND*pii) * floor(lonCell_arr(:) / (2._RKIND*pii)) ! ! Perform halo updates for all decomposed fields (i.e., fields with From fabe37a9910838690e43b4a6695079971156e5aa Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Fri, 26 Jul 2024 10:39:34 -0600 Subject: [PATCH 5/7] Move lonCell wrap code to dyn_grid.F90 from cam_mpas_subdriver.F90 This commit also extends the comment that was with this code block, includes an if condition so valid values aren't modified, uses kinds and constants (pi) that are already used in the new file, and updates the ChangeLog so the correct file is referenced. --- doc/ChangeLog | 4 ++-- src/dynamics/mpas/driver/cam_mpas_subdriver.F90 | 13 ++----------- src/dynamics/mpas/dyn_grid.F90 | 13 +++++++++++++ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index ab83bacb34..48d6f320f7 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -24,8 +24,8 @@ List all files eliminated: N/A List all files added and what they do: N/A List all existing files that have been modified, and describe the changes: -M src/dynamics/mpas/driver/cam_mpas_subdriver.F90 - - Modifies cam_mpas_read_static to ensure lonCell values are in [0,2pi) range +M src/dynamics/mpas/dyn_grid.F90 + - Modifies setup_time_invariant to ensure lonCell values are in [0,2pi) range If there were any failures reported from running test_driver.sh on any test platform, and checkin with these failures has been OK'd by the gatekeeper, diff --git a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 index 376389efef..676bacd4af 100644 --- a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 +++ b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 @@ -884,16 +884,15 @@ subroutine cam_mpas_read_static(fh_ini, endrun) use pio, only : file_desc_t - use mpas_kind_types, only : StrKIND, RKIND + use mpas_kind_types, only : StrKIND use mpas_io_streams, only : MPAS_createStream, MPAS_closeStream, MPAS_streamAddField, MPAS_readStream use mpas_derived_types, only : MPAS_IO_READ, MPAS_IO_NETCDF, MPAS_Stream_type, MPAS_pool_type, & field0DReal, field1DReal, field2DReal, field3DReal, field1DInteger, field2DInteger, & MPAS_STREAM_NOERR use mpas_pool_routines, only : MPAS_pool_get_subpool, MPAS_pool_get_field, MPAS_pool_create_pool, MPAS_pool_destroy_pool, & - MPAS_pool_add_config, MPAS_pool_get_array + MPAS_pool_add_config use mpas_dmpar, only : MPAS_dmpar_exch_halo_field use mpas_stream_manager, only : postread_reindex - use mpas_constants, only : pii ! Arguments type (file_desc_t), pointer :: fh_ini @@ -930,11 +929,8 @@ subroutine cam_mpas_read_static(fh_ini, endrun) type (MPAS_Stream_type) :: mesh_stream - real(kind=RKIND), dimension(:), pointer :: lonCell_arr - nullify(cell_gradient_coef_x) nullify(cell_gradient_coef_y) - nullify(lonCell_arr) call MPAS_createStream(mesh_stream, domain_ptr % ioContext, 'not_used', MPAS_IO_NETCDF, MPAS_IO_READ, & pio_file_desc=fh_ini, ierr=ierr) @@ -1175,11 +1171,6 @@ subroutine cam_mpas_read_static(fh_ini, endrun) call endrun(subname//': FATAL: Failed to close static input stream.') end if - call mpas_pool_get_array(meshPool, 'lonCell', lonCell_arr) - - ! Ensure longitudes w/i [0, 2*pi) range - lonCell_arr(:) = lonCell_arr(:) - (2._RKIND*pii) * floor(lonCell_arr(:) / (2._RKIND*pii)) - ! ! Perform halo updates for all decomposed fields (i.e., fields with ! an outermost dimension of nCells, nVertices, or nEdges) diff --git a/src/dynamics/mpas/dyn_grid.F90 b/src/dynamics/mpas/dyn_grid.F90 index d0b53c5fa0..d289d84499 100644 --- a/src/dynamics/mpas/dyn_grid.F90 +++ b/src/dynamics/mpas/dyn_grid.F90 @@ -453,6 +453,8 @@ subroutine setup_time_invariant(fh_ini) type(mpas_pool_type), pointer :: meshPool real(r8), pointer :: rdzw(:) real(r8), allocatable :: dzw(:) + integer, pointer :: nCells + real(r8), dimension(:), pointer :: lonCell integer :: k, kk integer :: ierr @@ -473,6 +475,7 @@ subroutine setup_time_invariant(fh_ini) call mpas_pool_get_dimension(meshPool, 'nEdgesSolve', nEdgesSolve) call mpas_pool_get_dimension(meshPool, 'nVerticesSolve', nVerticesSolve) call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevelsSolve) ! MPAS always solves over the full column + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) ! check that number of vertical layers matches MPAS grid data if (plev /= nVertLevelsSolve) then @@ -482,6 +485,16 @@ subroutine setup_time_invariant(fh_ini) ') does not match plev ('//int2str(nVertLevelsSolve)//').') end if + ! Ensure longitudes are within the [0,2*pi) range, and only remap values that + ! are outside the range. Some non-simple physics in CAM require this + ! longitude range, MPAS may have any (radian) value in lonCell + call mpas_pool_get_array(meshPool, 'lonCell', lonCell) + do k=1,nCells + if (lonCell(k) < 0._r8 .or. lonCell(k) .ge. (2._r8 * pi)) then + lonCell(k) = lonCell(k) - (2._r8 * pi) * floor(lonCell(k) / (2._r8 * pi)) + end if + end do + ! Initialize fields needed for reconstruction of cell-centered winds from edge-normal winds ! Note: This same pair of calls happens a second time later in the initialization of ! the MPAS-A dycore (in atm_mpas_init_block), but the redundant calls do no harm From 005e7f45cdb8ec604dd707fc2af56f9351f7d25f Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Fri, 26 Jul 2024 16:11:55 -0600 Subject: [PATCH 6/7] Replace .ge. with >= in lonCell loop condition Conform to CAM Coding Standards: 'Use symbolic numerical comparison operators (e.g., ==, /=, <, >=) not old character versions (e.g., .eq.).' --- src/dynamics/mpas/dyn_grid.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dynamics/mpas/dyn_grid.F90 b/src/dynamics/mpas/dyn_grid.F90 index d289d84499..0b26e10b2f 100644 --- a/src/dynamics/mpas/dyn_grid.F90 +++ b/src/dynamics/mpas/dyn_grid.F90 @@ -490,7 +490,7 @@ subroutine setup_time_invariant(fh_ini) ! longitude range, MPAS may have any (radian) value in lonCell call mpas_pool_get_array(meshPool, 'lonCell', lonCell) do k=1,nCells - if (lonCell(k) < 0._r8 .or. lonCell(k) .ge. (2._r8 * pi)) then + if (lonCell(k) < 0._r8 .or. lonCell(k) >= (2._r8 * pi)) then lonCell(k) = lonCell(k) - (2._r8 * pi) * floor(lonCell(k) / (2._r8 * pi)) end if end do From 13259119275a64900e771931df912c5a4ea01982 Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Fri, 26 Jul 2024 16:09:14 -0600 Subject: [PATCH 7/7] Update comment on lonCell wrap code --- src/dynamics/mpas/dyn_grid.F90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dynamics/mpas/dyn_grid.F90 b/src/dynamics/mpas/dyn_grid.F90 index 0b26e10b2f..7efcc866dd 100644 --- a/src/dynamics/mpas/dyn_grid.F90 +++ b/src/dynamics/mpas/dyn_grid.F90 @@ -486,8 +486,9 @@ subroutine setup_time_invariant(fh_ini) end if ! Ensure longitudes are within the [0,2*pi) range, and only remap values that - ! are outside the range. Some non-simple physics in CAM require this - ! longitude range, MPAS may have any (radian) value in lonCell + ! are outside the range. Some non-simple physics in CAM require this + ! longitude range, the MPAS-A dycore does not require any specific range for + ! lonCell call mpas_pool_get_array(meshPool, 'lonCell', lonCell) do k=1,nCells if (lonCell(k) < 0._r8 .or. lonCell(k) >= (2._r8 * pi)) then