From f0540b3bc1792856ed3e808c2d54fc84fed1b65f Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Tue, 24 Sep 2024 23:56:39 -0600 Subject: [PATCH 01/16] Fix the DA->NUOPC interface to not violate unison rule. Previous version worked only by conincidence. --- ExternalDriverAPIWeakCplDAProto/atmDA.F90 | 8 +- .../externalApp.F90 | 37 ++++++- ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 | 103 ++++++++++++++++-- ExternalDriverAPIWeakCplDAProto/ocnDA.F90 | 8 +- 4 files changed, 133 insertions(+), 23 deletions(-) diff --git a/ExternalDriverAPIWeakCplDAProto/atmDA.F90 b/ExternalDriverAPIWeakCplDAProto/atmDA.F90 index c0cb3d7..6b7f029 100644 --- a/ExternalDriverAPIWeakCplDAProto/atmDA.F90 +++ b/ExternalDriverAPIWeakCplDAProto/atmDA.F90 @@ -14,7 +14,6 @@ module atmDA ! ATM DA Code !----------------------------------------------------------------------------- - use MPI use ESMF use NUOPC @@ -32,8 +31,8 @@ module atmDA contains !----------------------------------------------------------------------------- - subroutine exec(comm) - integer :: comm + subroutine exec(vm) + type(ESMF_VM) :: vm ! context of this interaction integer :: rc @@ -44,7 +43,8 @@ subroutine exec(comm) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Connect DA code with NUOPC system top component - call nuopc_da_connect(toNuopcTopStandardNames=(/"precipitation_flux"/), & + call nuopc_da_connect(vm=vm, & + toNuopcTopStandardNames=(/"precipitation_flux"/), & fmNuopcTopStandardNames=(/"surface_net_downward_shortwave_flux"/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & diff --git a/ExternalDriverAPIWeakCplDAProto/externalApp.F90 b/ExternalDriverAPIWeakCplDAProto/externalApp.F90 index 46a11c4..94c2625 100644 --- a/ExternalDriverAPIWeakCplDAProto/externalApp.F90 +++ b/ExternalDriverAPIWeakCplDAProto/externalApp.F90 @@ -21,8 +21,9 @@ program externalApp use ESM, only: esmSS => SetServices use nuopc_da, only: & - nuopc_da_init => init, & - nuopc_da_final => final + nuopc_da_init => init, & + nuopc_da_commToVM => commToVM, & + nuopc_da_final => final use atmDA, only: atmDAexec => exec use ocnDA, only: ocnDAexec => exec @@ -31,7 +32,8 @@ program externalApp integer :: rc integer :: size, rank - integer :: splitComm + integer :: commAtmDA, commOcnDA, splitComm + type(ESMF_VM) :: vmAtmDA, vmOcnDA ! Initialize the NUOPC-DA interface call nuopc_da_init(nuopcTopSetServices=esmSS, rc=rc) @@ -41,17 +43,40 @@ program externalApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Split up the MPI_COMM_WORLD into atmDA (first half) and ocnDA (second half) - ! of MPI ranks. Call into the respective DA routine + ! of MPI ranks. call MPI_Comm_size(MPI_COMM_WORLD, size, rc) call MPI_Comm_rank(MPI_COMM_WORLD, rank, rc) if (rank < size/2) then ! atm DA processes call MPI_Comm_split(MPI_COMM_WORLD, 1, rank, splitComm, rc) - call atmDAexec(comm=splitComm) + commAtmDA = splitComm + commOcnDA = MPI_COMM_NULL else ! ocn DA processes call MPI_Comm_split(MPI_COMM_WORLD, 2, rank, splitComm, rc) - call ocnDAexec(comm=splitComm) + commOcnDA = splitComm + commAtmDA = MPI_COMM_NULL + endif + + ! Create corresponding ESMF_VM objects for atmDA and ocnDA + vmAtmDA = nuopc_da_commToVM(commAtmDA, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + vmOcnDA = nuopc_da_commToVM(commOcnDA, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call into the respective DA routine for ATM and OCN + if (rank < size/2) then + ! atm DA processes + call atmDAexec(vmAtmDA) + else + ! ocn DA processes + call ocnDAexec(vmOcnDA) endif ! Finalize the NUOPC-DA interface diff --git a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 index 910ef35..0d333cc 100644 --- a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 +++ b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 @@ -11,7 +11,7 @@ module nuopc_da !----------------------------------------------------------------------------- - ! NUOPC - DA interface code + ! NUOPC - DA interface code - this is generic code that could move into NUOPC !----------------------------------------------------------------------------- use MPI @@ -21,12 +21,12 @@ module nuopc_da implicit none private - + type(ESMF_GridComp) :: nuopcTop type(ESMF_State) :: toNuopcTop, fmNuopcTop type(ESMF_Clock) :: clock - public init, connect, step, final + public init, commToVM, connect, step, final !----------------------------------------------------------------------------- contains @@ -116,10 +116,95 @@ recursive subroutine nuopcTopSetServices(gridcomp, rc) !----------------------------------------------------------------------------- - subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) - character(*), intent(in), optional :: toNuopcTopStandardNames(:) - character(*), intent(in), optional :: fmNuopcTopStandardNames(:) - integer, intent(out) :: rc + function commToVM(comm, rc) + type(ESMF_VM) :: commToVM + integer, intent(in) :: comm ! MPI communicator + integer, intent(out) :: rc + + type(ESMF_VM) :: vm + integer :: petCount, taskCount, temp_int(1) + integer :: i, j, urc + integer, allocatable :: temp_list(:), petList(:) + type(ESMF_GridComp) :: comp + + call ESMF_VMGetCurrent(vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + call ESMF_VMGet(vm, petCount=petCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + temp_int(1) = 0 ! default indicate task is not in comm + if (comm /= MPI_COMM_NULL) temp_int(1) = 1 ! indicate task is in comm + + allocate(temp_list(petCount)) + + call ESMF_VMAllGather(vm, temp_int, temp_list, 1, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! determine number of tasks in comm + taskCount = 0 + do i=1, petCount + if (temp_list(i)==1) taskCount = taskCount + 1 + enddo + + ! construct petList + allocate(petList(taskCount)) + j = 1 + do i=1, petCount + if (temp_list(i)==1) then + petList(j) = i-1 ! PETs are basis 0 + j = j+1 + endif + enddo + + deallocate(temp_list) + + ! create ESMF component on petList + comp = ESMF_GridCompCreate(petList=petList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + deallocate(petList) + + ! call SetServices with a dummy routine to trigger internal VM creation + call ESMF_GridCompSetServices(comp, dummySS, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! access the VM to be returned + call ESMF_GridCompGet(comp, vm=commToVM, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + contains + + recursive subroutine dummySS(gridcomp, rc) + type(ESMF_GridComp) :: gridcomp ! must not be optional + integer, intent(out) :: rc ! must not be optional + rc = ESMF_SUCCESS + end subroutine + + end function commToVM + + !----------------------------------------------------------------------------- + + subroutine connect(vm, toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) + type(ESMF_VM), intent(in) :: vm + character(*), intent(in), optional :: toNuopcTopStandardNames(:) + character(*), intent(in), optional :: fmNuopcTopStandardNames(:) + integer, intent(out) :: rc integer :: urc, phase, i type(ESMF_Field) :: field @@ -128,7 +213,7 @@ subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) if (present(toNuopcTopStandardNames)) then call NUOPC_Advertise(toNuopcTop, StandardNames=toNuopcTopStandardNames, & TransferOfferGeomObject="cannot provide", SharePolicyField="share", & - rc=rc) + vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__, rcToReturn=rc)) return @@ -137,7 +222,7 @@ subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) if (present(fmNuopcTopStandardNames)) then call NUOPC_Advertise(fmNuopcTop, StandardNames=fmNuopcTopStandardNames, & TransferOfferGeomObject="cannot provide", SharePolicyField="share", & - rc=rc) + vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__, rcToReturn=rc)) return diff --git a/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 b/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 index 23877e3..2d1ecbf 100644 --- a/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 +++ b/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 @@ -14,7 +14,6 @@ module ocnDA ! OCN DA Code !----------------------------------------------------------------------------- - use MPI use ESMF use NUOPC @@ -32,8 +31,8 @@ module ocnDA contains !----------------------------------------------------------------------------- - subroutine exec(comm) - integer :: comm + subroutine exec(vm) + type(ESMF_VM) :: vm ! context of this interaction integer :: rc @@ -44,7 +43,8 @@ subroutine exec(comm) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Connect DA code with NUOPC system top component - call nuopc_da_connect(fmNuopcTopStandardNames=(/"sea_surface_temperature"/), & + call nuopc_da_connect(vm=vm, & + fmNuopcTopStandardNames=(/"sea_surface_temperature"/), & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & From a6e7c0776ed1964e2d5f64eb73425a8879d01c25 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Fri, 25 Oct 2024 13:25:30 -0400 Subject: [PATCH 02/16] Protect access to fptr() pointer even more strongly (since NAG does not like it). --- AtmOcnScalarProto/atm.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AtmOcnScalarProto/atm.F90 b/AtmOcnScalarProto/atm.F90 index f607a68..5a80f79 100644 --- a/AtmOcnScalarProto/atm.F90 +++ b/AtmOcnScalarProto/atm.F90 @@ -307,9 +307,9 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out ! access the scalars through a 1D array - scalars => fptr(:,1) if (size(fptr)>0) then ! only use fptr on that PET which holds allocation + scalars => fptr(:,1) do i=lbound(scalars,1), ubound(scalars,1) scalars(i)=(i-1)*10+step enddo From 83bc51d57f1faa72c0418596d60cae8faee3c7f3 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Thu, 14 Nov 2024 14:22:38 -0800 Subject: [PATCH 03/16] Better comment on the failing setup due to ESMF_StateReconcile() issue. More complete test for geom/field sharing. --- AtmOcnMirrorFieldsProto/atm.F90 | 10 ++++++++-- AtmOcnMirrorFieldsProto/ocn.F90 | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/AtmOcnMirrorFieldsProto/atm.F90 b/AtmOcnMirrorFieldsProto/atm.F90 index 462652e..73ff534 100644 --- a/AtmOcnMirrorFieldsProto/atm.F90 +++ b/AtmOcnMirrorFieldsProto/atm.F90 @@ -245,7 +245,13 @@ subroutine ModifyAdvertised(model, rc) !TODO: settings, and go between components on different threading levels. !TODO: This is an edge case, but long term it should be supported. -#if 0 +!TODO: The problem in StateReconcile() comes from the UNIQUE_GEOM_INFO_TREAT_on +!TODO: handling. Once that part is reworked inside StateReconcile() the mixed +!TODO: sharing case should work again! + +!TODO: For now share Grid and Field for both fields. + +#if 1 ! set SharePolicyGeomObject = "share" call NUOPC_SetAttribute(field, name="SharePolicyGeomObject", & value="share", rc=rc) @@ -254,7 +260,7 @@ subroutine ModifyAdvertised(model, rc) file=__FILE__)) & return ! bail out #endif -#if 0 +#if 1 ! set SharePolicyField = "share" call NUOPC_SetAttribute(field, name="SharePolicyField", & value="share", rc=rc) diff --git a/AtmOcnMirrorFieldsProto/ocn.F90 b/AtmOcnMirrorFieldsProto/ocn.F90 index 96c3fed..f5a8a69 100644 --- a/AtmOcnMirrorFieldsProto/ocn.F90 +++ b/AtmOcnMirrorFieldsProto/ocn.F90 @@ -96,6 +96,7 @@ subroutine Advertise(model, rc) StandardName="air_pressure_at_sea_level", name="pmsl", & #ifdef TEST_SHARING SharePolicyGeomObject="share", & + SharePolicyField="share", & #endif rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & From 890fd3ba98cf7149a6005427a7949cc9990852dc Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Mon, 25 Nov 2024 15:52:32 -0600 Subject: [PATCH 04/16] add new test for mirroring with nested states --- AtmOcnMirrorFieldsNestedProto/Makefile | 76 ++++++ AtmOcnMirrorFieldsNestedProto/README | 46 ++++ AtmOcnMirrorFieldsNestedProto/atm.F90 | 313 +++++++++++++++++++++++ AtmOcnMirrorFieldsNestedProto/esm.F90 | 234 +++++++++++++++++ AtmOcnMirrorFieldsNestedProto/esmApp.F90 | 106 ++++++++ AtmOcnMirrorFieldsNestedProto/ocn.F90 | 278 ++++++++++++++++++++ testProtos.sh | 1 + 7 files changed, 1054 insertions(+) create mode 100644 AtmOcnMirrorFieldsNestedProto/Makefile create mode 100644 AtmOcnMirrorFieldsNestedProto/README create mode 100644 AtmOcnMirrorFieldsNestedProto/atm.F90 create mode 100644 AtmOcnMirrorFieldsNestedProto/esm.F90 create mode 100644 AtmOcnMirrorFieldsNestedProto/esmApp.F90 create mode 100644 AtmOcnMirrorFieldsNestedProto/ocn.F90 diff --git a/AtmOcnMirrorFieldsNestedProto/Makefile b/AtmOcnMirrorFieldsNestedProto/Makefile new file mode 100644 index 0000000..991c603 --- /dev/null +++ b/AtmOcnMirrorFieldsNestedProto/Makefile @@ -0,0 +1,76 @@ +# GNU Makefile template for user ESMF application + +################################################################################ +################################################################################ +## This Makefile must be able to find the "esmf.mk" Makefile fragment in the ## +## 'include' line below. Following the ESMF User's Guide, a complete ESMF ## +## installation should ensure that a single environment variable "ESMFMKFILE" ## +## is made available on the system. This variable should point to the ## +## "esmf.mk" file. ## +## ## +## This example Makefile uses the "ESMFMKFILE" environment variable. ## +## ## +## If you notice that this Makefile cannot find variable ESMFMKFILE then ## +## please contact the person responsible for the ESMF installation on your ## +## system. ## +## As a work-around you can simply hardcode the path to "esmf.mk" in the ## +## include line below. However, doing so will render this Makefile a lot less ## +## flexible and non-portable. ## +################################################################################ + +ifneq ($(origin ESMFMKFILE), environment) +$(error Environment variable ESMFMKFILE was not set.) +endif + +include $(ESMFMKFILE) + +# strip quotes around the ESMF_INTERNAL_MPIRUN value +ESMF_INTERNAL_MPIRUN := $(shell echo $(ESMF_INTERNAL_MPIRUN)) + +################################################################################ +################################################################################ + +.SUFFIXES: .f90 .F90 .c .C + +%.o : %.f90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREENOCPP) $< + +%.o : %.F90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $< + +%.o : %.c + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + +%.o : %.C + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + + +# ----------------------------------------------------------------------------- +esmApp: esmApp.o esm.o atm.o ocn.o + $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS) + +# module dependencies: +esmApp.o: esm.o +esm.o: atm.o ocn.o + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +.PHONY: dust clean distclean info edit +dust: + rm -f PET*.ESMF_LogFile *.nc *.stdout +clean: + rm -f esmApp *.o *.mod +distclean: dust clean + +info: + @echo ================================================================== + @echo ESMFMKFILE=$(ESMFMKFILE) + @echo ================================================================== + @cat $(ESMFMKFILE) + @echo ================================================================== + +edit: + nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & + +run: + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMirrorFieldsNestedProto/README b/AtmOcnMirrorFieldsNestedProto/README new file mode 100644 index 0000000..126229a --- /dev/null +++ b/AtmOcnMirrorFieldsNestedProto/README @@ -0,0 +1,46 @@ +README for ATM-OCN with field mirroring NUOPC prototype +------------------------------------------------------- + +A simple two model system, where one Model component requests field mirroring +on one of its States. + +Description: + + A two-way coupled system with a single driver (ESM) and two model components + (ATM, OCN). + + The ESM driver uses explicitly constructed petLists when adding the two + model components. + + The ESM driver component uses the default run sequence to implement coupling + between ATM and OCN components. + + The connector components are explicitly added by the driver. + + The OCN component use a simple two-phase initialization, consisting of + advertise and realize. + + The ATM component uses a three-phase initialization, consisting of + advertise/request-mirror, modify mirror-advertised fields, realize. + + +Build: + - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF + installation. + - make + +Execution: + - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. + - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) + +Output: + - PET*.Log files containing compliance checker output if turned on. + - The prototype outputs time stepping information to stdout. + +Code structure: + - Makefile - Makefile that is based on the standard esmf.mk mechanism. + - atm.F90 - The ATM component, specializing generic NUOPC_Model. + - ocn.F90 - The OCN component, specializing generic NUOPC_Model. + - esm.F90 - The Earth System Model (ESM) component, specializing generic + NUOPC_Driver. Define partial petLists for ATM and OCN. + - esmApp.F90 - ESM application. diff --git a/AtmOcnMirrorFieldsNestedProto/atm.F90 b/AtmOcnMirrorFieldsNestedProto/atm.F90 new file mode 100644 index 0000000..3398a8e --- /dev/null +++ b/AtmOcnMirrorFieldsNestedProto/atm.F90 @@ -0,0 +1,313 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module ATM + + !----------------------------------------------------------------------------- + ! ATM Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_RealizeAccepted, & + specRoutine=RealizeAccepted, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_SetAttribute(importState, "FieldTransferPolicy", "transferAllAsNests", & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine RealizeAccepted(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + integer :: i, j + type(ESMF_State) :: importState, exportState + integer :: importItemCount, importNestedItemCount + type(ESMF_State) :: importNestedState + character(ESMF_MAXSTR) :: nestedStateName + character(ESMF_MAXSTR), allocatable :: importItemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: importItemTypeList(:) + character(ESMF_MAXSTR), allocatable :: importNestedItemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: importNestedItemTypeList(:) + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query nested States + call ESMF_StateGet(importState, itemCount=importItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! allocate temporary data structures + allocate(importItemNameList(importItemCount)) + allocate(importItemTypeList(importItemCount)) + + ! query importState + call ESMF_StateGet(importState, nestedFlag=.false., itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! loop over items in importState + do i = 1, importItemCount + ! query item + if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then + ! pull out nested state + call ESMF_StateGet(importState, itemName=importItemNameList(i), nestedState=importNestedState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query nested state + call ESMF_StateGet(importNestedState, name=nestedStateName, itemCount=importNestedItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! allocate temporary data structures + allocate(importNestedItemNameList(importNestedItemCount)) + allocate(importNestedItemTypeList(importNestedItemCount)) + + ! query item name and types in the nested state + call ESMF_StateGet(importNestedState, itemNameList=importNestedItemNameList, & + itemTypeList=importNestedItemTypeList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! loop over items in importNestedState + do j = 1, importNestedItemCount + call ESMF_LogWrite('realize '//trim(importNestedItemNameList(j))//' import field received from '//trim(nestedStateName), ESMF_LOGMSG_INFO) + + call NUOPC_Realize(importNestedState, fieldName=trim(importNestedItemNameList(j)), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end do + + ! clean memory + deallocate(importNestedItemNameList) + deallocate(importNestedItemTypeList) + end if + end do + + ! clean memory + deallocate(importItemNameList) + deallocate(importItemTypeList) + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_State) :: importNestedState + character(ESMF_MAXSTR) :: nestedStateName + character(ESMF_MAXSTR), allocatable :: importItemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: importItemTypeList(:) + integer :: i, importItemCount + integer, save :: slice=1 + character(len=160) :: msgString + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set by default, + ! its timeStep is equal to the parent timeStep. As a consequence the + ! currTime + timeStep is equal to the stopTime of the internal Clock + ! for this call of the Advance() routine. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="------>Advancing ATM from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockPrint(clock, options="stopTime", & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query nested States + call ESMF_StateGet(importState, nestedFlag=.false., & + itemCount=importItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query number of item in importState + call ESMF_StateGet(importState, itemCount=importItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! allocate temporary data structures + allocate(importItemNameList(importItemCount)) + allocate(importItemTypeList(importItemCount)) + + ! query importState + call ESMF_StateGet(importState, nestedFlag=.false., itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! loop over items in importState + do i = 1, importItemCount + ! query item + if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then + ! pull out nested state + call ESMF_StateGet(importState, & + itemName=importItemNameList(i), nestedState=importNestedState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_StateGet(importNestedState, name=nestedStateName, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! write out the Fields in the importState and exportState + call NUOPC_Write(importNestedState, & + fileNamePrefix='field_atm_import_from_'//trim(nestedStateName)//'_', & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end if + end do + slice = slice+1 + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/AtmOcnMirrorFieldsNestedProto/esm.F90 b/AtmOcnMirrorFieldsNestedProto/esm.F90 new file mode 100644 index 0000000..a1270d3 --- /dev/null +++ b/AtmOcnMirrorFieldsNestedProto/esm.F90 @@ -0,0 +1,234 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +! For testing with threading keep one or both SETVM macros active +#define WITH_ATM_SETVM +#define WITH_OCN_SETVM + +! For testing with partially overlapping components keep one active, other _off +#define WITH_ATM_PETLIST_off +#define WITH_OCN_PETLIST_off + +module ESM + + !----------------------------------------------------------------------------- + ! Code that specializes generic ESM Component code. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Driver, & + driverSS => SetServices + + use ATM, only: atmSVM => SetVM, atmSS => SetServices + use OCN, only: ocnSVM => SetVM, ocnSS => SetServices + + use NUOPC_Connector, only: cplSS => SetServices + + implicit none + + private + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Driver + call NUOPC_CompDerive(driver, driverSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize driver + call NUOPC_CompSpecialize(driver, specLabel=label_SetModelServices, & + specRoutine=SetModelServices, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set driver verbosity + call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetModelServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + ! local variables + type(ESMF_Grid) :: grid + type(ESMF_Field) :: field + type(ESMF_Time) :: startTime + type(ESMF_Time) :: stopTime + type(ESMF_TimeInterval) :: timeStep + type(ESMF_Clock) :: internalClock + type(ESMF_GridComp) :: child + type(ESMF_CplComp) :: connector + integer :: petCount, i + integer, allocatable :: petList(:) + type(ESMF_Info) :: info + + rc = ESMF_SUCCESS + + ! Create and set the info object that is used to pass hints into methods + info = ESMF_InfoCreate(rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! get the petCount + call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ATM with petList on first half of PETs + allocate(petList(petCount/2)) + do i=1, petCount/2 + petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 + enddo + call NUOPC_DriverAddComp(driver, "ATM", atmSS, & +#ifdef WITH_ATM_SETVM + atmSVM, info=info, & +#endif +#ifdef WITH_ATM_PETLIST + petList=petList, & +#endif + comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(petList) + + ! SetServices for OCN with petList on second half of PETs + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=4, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + allocate(petList(petCount/2)) + do i=1, petCount/2 + petList(i) = petCount/2 + i-1 ! PET labeling goes from 0 to petCount-1 + enddo + call NUOPC_DriverAddComp(driver, "OCN", ocnSS, & +#ifdef WITH_OCN_SETVM + ocnSVM, info=info, & +#endif +#ifdef WITH_OCN_PETLIST + petList=petList, & +#endif + comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(petList) + + ! SetServices for atm2ocn + call NUOPC_DriverAddComp(driver, srcCompLabel="ATM", dstCompLabel="OCN", & + compSetServicesRoutine=cplSS, comp=connector, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(connector, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ocn2atm + call NUOPC_DriverAddComp(driver, srcCompLabel="OCN", dstCompLabel="ATM", & + compSetServicesRoutine=cplSS, comp=connector, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(connector, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set the driver clock + call ESMF_TimeIntervalSet(timeStep, m=15, rc=rc) ! 15 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimeSet(startTime, yy=2010, mm=6, dd=1, h=0, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimeSet(stopTime, yy=2010, mm=6, dd=1, h=1, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + internalClock = ESMF_ClockCreate(name="Application Clock", & + timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_GridCompSet(driver, clock=internalClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/AtmOcnMirrorFieldsNestedProto/esmApp.F90 b/AtmOcnMirrorFieldsNestedProto/esmApp.F90 new file mode 100644 index 0000000..8796566 --- /dev/null +++ b/AtmOcnMirrorFieldsNestedProto/esmApp.F90 @@ -0,0 +1,106 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +program esmApp + + !----------------------------------------------------------------------------- + ! Generic ESM application driver + !----------------------------------------------------------------------------- + + use ESMF + use ESM, only: esmSS => SetServices + + implicit none + + integer :: rc, urc + type(ESMF_GridComp) :: esmComp + + ! Initialize ESMF + call ESMF_Initialize(logkindflag=ESMF_LOGKIND_MULTI, & + defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_LogWrite("esmApp STARTING", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Create the earth system Component + esmComp = ESMF_GridCompCreate(name="esm", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! SetServices for the earth system Component + call ESMF_GridCompSetServices(esmComp, esmSS, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call Initialize for the earth system Component + call ESMF_GridCompInitialize(esmComp, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call Run for earth the system Component + call ESMF_GridCompRun(esmComp, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call Finalize for the earth system Component + call ESMF_GridCompFinalize(esmComp, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Destroy the earth system Component + call ESMF_GridCompDestroy(esmComp, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_LogWrite("esmApp FINISHED", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Finalize ESMF + call ESMF_Finalize() + +end program diff --git a/AtmOcnMirrorFieldsNestedProto/ocn.F90 b/AtmOcnMirrorFieldsNestedProto/ocn.F90 new file mode 100644 index 0000000..c410036 --- /dev/null +++ b/AtmOcnMirrorFieldsNestedProto/ocn.F90 @@ -0,0 +1,278 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module OCN + + !----------------------------------------------------------------------------- + ! OCN Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_SetClock, & + specRoutine=SetClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_temperature + call NUOPC_Advertise(exportState, & + StandardName="sea_surface_temperature", name="sst", & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + type(ESMF_TimeInterval) :: stabilityTimeStep + type(ESMF_Field) :: field + type(ESMF_Grid) :: grid + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create a Grid object for Fields + grid = ESMF_GridCreateNoPeriDimUfrm(maxIndex=(/100, 20/), & + minCornerCoord=(/10._ESMF_KIND_R8, 20._ESMF_KIND_R8/), & + maxCornerCoord=(/100._ESMF_KIND_R8, 200._ESMF_KIND_R8/), & + coordSys=ESMF_COORDSYS_CART, staggerLocList=(/ESMF_STAGGERLOC_CENTER/), & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_temperature + field = ESMF_FieldCreate(name="sst", grid=grid, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldFill(field, dataFillScheme="sincos", member=2, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetClock(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_TimeInterval) :: stabilityTimeStep + + rc = ESMF_SUCCESS + + ! query for clock + call NUOPC_ModelGet(model, modelClock=clock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! initialize internal clock + ! here: parent Clock and stability timeStep determine actual model timeStep + !TODO: stabilityTimeStep should be read in from configuation + !TODO: or computed from internal Grid information + call ESMF_TimeIntervalSet(stabilityTimeStep, m=5, rc=rc) ! 5 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSetClock(model, clock, stabilityTimeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_Time) :: currTime + type(ESMF_TimeInterval) :: timeStep + integer, save :: slice=1 + character(len=160) :: msgString + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set in SetClock(), + ! its timeStep is likely smaller than the parent timeStep. As a consequence + ! the time interval covered by a single parent timeStep will result in + ! multiple calls to the Advance() routine. Every time the currTime + ! will come in by one internal timeStep advanced. This goes until the + ! stopTime of the internal Clock has been reached. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="------>Advancing OCN from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockGet(clock, currTime=currTime, timeStep=timeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimePrint(currTime + timeStep, & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! update data in export fields + call ESMF_StateGet(exportState, itemName="sst", field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldFill(field, dataFillScheme="sincos", member=1, & + step=slice, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + slice = slice+1 + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/testProtos.sh b/testProtos.sh index 4ec9f4e..a648d8d 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -399,6 +399,7 @@ TestProto AtmOcnMedPetListTimescalesProto esmApp TestProto AtmOcnMedPetListTimescalesSplitFastProto esmApp TestProto AtmOcnMedProto esmApp TestProto AtmOcnMirrorFieldsProto esmApp +TestProto AtmOcnMirrorFieldsNestedProto esmApp TestProto AtmOcnPetListProto esmApp TestProto AtmOcnProto esmApp TestProto AtmOcnRtmTwoTimescalesProto esmApp From 82cda01e209b0c64bd2e77ac312c2ae635afb8d9 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Mon, 25 Nov 2024 16:16:32 -0600 Subject: [PATCH 05/16] use lower case for state name --- AtmOcnMirrorFieldsNestedProto/atm.F90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AtmOcnMirrorFieldsNestedProto/atm.F90 b/AtmOcnMirrorFieldsNestedProto/atm.F90 index 3398a8e..4b45be4 100644 --- a/AtmOcnMirrorFieldsNestedProto/atm.F90 +++ b/AtmOcnMirrorFieldsNestedProto/atm.F90 @@ -294,6 +294,9 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out + ! convert state name to lower case + nestedStateName = ESMF_UtilStringLowerCase(nestedStateName) + ! write out the Fields in the importState and exportState call NUOPC_Write(importNestedState, & fileNamePrefix='field_atm_import_from_'//trim(nestedStateName)//'_', & From 475d2e046c71453608f35f4afc644518705ca3ed Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 2 Dec 2024 16:31:48 -0700 Subject: [PATCH 06/16] Compiler workaround: Pull internal subroutine out to module-level This shouldn't be needed, but is needed with some Mac configurations to work around an issue with clang. Resolves esmf-org/nuopc-app-prototypes#8 --- ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 index 0d333cc..0c71781 100644 --- a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 +++ b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 @@ -116,6 +116,15 @@ recursive subroutine nuopcTopSetServices(gridcomp, rc) !----------------------------------------------------------------------------- + ! The following is used in commToVM. Some Mac configurations have issues with this being + ! an internal subroutine to commToVM, so we have pulled it out to a module-level + ! subroutine. (See https://github.com/esmf-org/nuopc-app-prototypes/issues/8.) + recursive subroutine dummySS(gridcomp, rc) + type(ESMF_GridComp) :: gridcomp ! must not be optional + integer, intent(out) :: rc ! must not be optional + rc = ESMF_SUCCESS + end subroutine + function commToVM(comm, rc) type(ESMF_VM) :: commToVM integer, intent(in) :: comm ! MPI communicator @@ -188,14 +197,6 @@ function commToVM(comm, rc) line=__LINE__, & file=__FILE__)) return - contains - - recursive subroutine dummySS(gridcomp, rc) - type(ESMF_GridComp) :: gridcomp ! must not be optional - integer, intent(out) :: rc ! must not be optional - rc = ESMF_SUCCESS - end subroutine - end function commToVM !----------------------------------------------------------------------------- From b4937ed7a60c9f20ed1eb670dc981c4bd9d44eac Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 3 Dec 2024 17:52:23 -0700 Subject: [PATCH 07/16] Get location of libomp from a spack installation if possible This is needed to support the new spack-based setup on the Darwin system hearhear --- testProtos.sh | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/testProtos.sh b/testProtos.sh index 4ec9f4e..58606cd 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -26,10 +26,28 @@ then # Darwin systems that use Apple Clang need extra environment variables # to find the libomp installation. These are important for the ESMX # tests where CMake is used under the hood to determine the correct - # compiler and linker flags. Set them here to be available: - export LDFLAGS=-L/opt/homebrew/opt/libomp/lib - export CXXFLAGS=-I/opt/homebrew/opt/libomp/include - export CFLAGS=-I/opt/homebrew/opt/libomp/include + # compiler and linker flags. Set them here to be available. + + # We'll try to get the location from a spack installation of llvm-openmp if possible, + # but if we can't find that, then we'll fall back on assuming that libomp is available + # via homebrew. + homebrew_libomp_dir=/opt/homebrew/opt/libomp + + if command -v spack &>/dev/null; then + # First try getting these from a spack-installed llvm-openmp: + omp_dir=$(spack location -i --first llvm-openmp 2>/dev/null) + if [ $? -ne 0 ]; then + # llvm-openmp isn't installed with spack; fall back on the homebrew location + omp_dir=${homebrew_libomp_dir} + fi + else + # spack isn't found; fall back on the homebrew location + omp_dir=${homebrew_libomp_dir} + fi + + export LDFLAGS="-L${omp_dir}/lib -Wl,-rpath,${omp_dir}/lib" + export CXXFLAGS=-I${omp_dir}/include + export CFLAGS=-I${omp_dir}/include fi function TestProto { From dc43e0f417661766c6ec4bc36662d39c892f0269 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 3 Dec 2024 17:55:35 -0700 Subject: [PATCH 08/16] Add a comment --- testProtos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/testProtos.sh b/testProtos.sh index 58606cd..620ea26 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -45,6 +45,7 @@ then omp_dir=${homebrew_libomp_dir} fi + # The spack-based setup requires -Wl,-rpath in addition to -L export LDFLAGS="-L${omp_dir}/lib -Wl,-rpath,${omp_dir}/lib" export CXXFLAGS=-I${omp_dir}/include export CFLAGS=-I${omp_dir}/include From 22e9215bc864aff116e499ecc978fe3b5ca43787 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Wed, 4 Dec 2024 12:21:15 -0600 Subject: [PATCH 09/16] clean and remove extra code --- AtmOcnMirrorFieldsNestedProto/esm.F90 | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/AtmOcnMirrorFieldsNestedProto/esm.F90 b/AtmOcnMirrorFieldsNestedProto/esm.F90 index a1270d3..2de813e 100644 --- a/AtmOcnMirrorFieldsNestedProto/esm.F90 +++ b/AtmOcnMirrorFieldsNestedProto/esm.F90 @@ -8,14 +8,6 @@ ! Licensed under the University of Illinois-NCSA License. !============================================================================== -! For testing with threading keep one or both SETVM macros active -#define WITH_ATM_SETVM -#define WITH_OCN_SETVM - -! For testing with partially overlapping components keep one active, other _off -#define WITH_ATM_PETLIST_off -#define WITH_OCN_PETLIST_off - module ESM !----------------------------------------------------------------------------- @@ -119,12 +111,6 @@ subroutine SetModelServices(driver, rc) petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 enddo call NUOPC_DriverAddComp(driver, "ATM", atmSS, & -#ifdef WITH_ATM_SETVM - atmSVM, info=info, & -#endif -#ifdef WITH_ATM_PETLIST - petList=petList, & -#endif comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -149,12 +135,6 @@ subroutine SetModelServices(driver, rc) petList(i) = petCount/2 + i-1 ! PET labeling goes from 0 to petCount-1 enddo call NUOPC_DriverAddComp(driver, "OCN", ocnSS, & -#ifdef WITH_OCN_SETVM - ocnSVM, info=info, & -#endif -#ifdef WITH_OCN_PETLIST - petList=petList, & -#endif comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & From 9820b5e40390826bedcd3371b7cca61da44dcdbc Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Mon, 9 Dec 2024 09:21:04 -0800 Subject: [PATCH 10/16] Implement the petList option again to test functionality with concurrent components. --- AtmOcnMirrorFieldsNestedProto/esm.F90 | 34 ++++++++++----------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/AtmOcnMirrorFieldsNestedProto/esm.F90 b/AtmOcnMirrorFieldsNestedProto/esm.F90 index 2de813e..3d88c93 100644 --- a/AtmOcnMirrorFieldsNestedProto/esm.F90 +++ b/AtmOcnMirrorFieldsNestedProto/esm.F90 @@ -8,6 +8,10 @@ ! Licensed under the University of Illinois-NCSA License. !============================================================================== +! For testing with partially overlapping components keep one active, other _off +#define WITH_ATM_PETLIST +#define WITH_OCN_PETLIST + module ESM !----------------------------------------------------------------------------- @@ -19,8 +23,8 @@ module ESM use NUOPC_Driver, & driverSS => SetServices - use ATM, only: atmSVM => SetVM, atmSS => SetServices - use OCN, only: ocnSVM => SetVM, ocnSS => SetServices + use ATM, only: atmSS => SetServices + use OCN, only: ocnSS => SetServices use NUOPC_Connector, only: cplSS => SetServices @@ -81,23 +85,9 @@ subroutine SetModelServices(driver, rc) type(ESMF_CplComp) :: connector integer :: petCount, i integer, allocatable :: petList(:) - type(ESMF_Info) :: info rc = ESMF_SUCCESS - ! Create and set the info object that is used to pass hints into methods - info = ESMF_InfoCreate(rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & - rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - ! get the petCount call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -111,6 +101,9 @@ subroutine SetModelServices(driver, rc) petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 enddo call NUOPC_DriverAddComp(driver, "ATM", atmSS, & +#ifdef WITH_ATM_PETLIST + petList=petList, & +#endif comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -124,17 +117,14 @@ subroutine SetModelServices(driver, rc) deallocate(petList) ! SetServices for OCN with petList on second half of PETs - call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=4, & - rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out allocate(petList(petCount/2)) do i=1, petCount/2 petList(i) = petCount/2 + i-1 ! PET labeling goes from 0 to petCount-1 enddo call NUOPC_DriverAddComp(driver, "OCN", ocnSS, & +#ifdef WITH_OCN_PETLIST + petList=petList, & +#endif comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & From 2a215a5a17af1dee9c348b1fb2060e3032be2f1f Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Mon, 9 Dec 2024 09:23:35 -0800 Subject: [PATCH 11/16] Add NUOPC_Write() during Advance() to provide output of producer data. --- AtmOcnMirrorFieldsNestedProto/ocn.F90 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/AtmOcnMirrorFieldsNestedProto/ocn.F90 b/AtmOcnMirrorFieldsNestedProto/ocn.F90 index c410036..016677b 100644 --- a/AtmOcnMirrorFieldsNestedProto/ocn.F90 +++ b/AtmOcnMirrorFieldsNestedProto/ocn.F90 @@ -269,6 +269,13 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out + ! write out the Fields in the exportState + call NUOPC_Write(exportState, fileNamePrefix="field_ocn_export_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out slice = slice+1 end subroutine From 29c6804f0aff9b138b528b599d67d5240406e028 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Mon, 9 Dec 2024 09:25:04 -0800 Subject: [PATCH 12/16] Formatting and switch to usage of standard NUOPC Attribute "Namespace" when identifying the fields in nested state during output. --- AtmOcnMirrorFieldsNestedProto/atm.F90 | 102 ++++++++++++++------------ 1 file changed, 56 insertions(+), 46 deletions(-) diff --git a/AtmOcnMirrorFieldsNestedProto/atm.F90 b/AtmOcnMirrorFieldsNestedProto/atm.F90 index 4b45be4..bf19cb3 100644 --- a/AtmOcnMirrorFieldsNestedProto/atm.F90 +++ b/AtmOcnMirrorFieldsNestedProto/atm.F90 @@ -85,8 +85,8 @@ subroutine Advertise(model, rc) file=__FILE__)) & return ! bail out - call NUOPC_SetAttribute(importState, "FieldTransferPolicy", "transferAllAsNests", & - rc=rc) + call NUOPC_SetAttribute(importState, "FieldTransferPolicy", & + "transferAllAsNests", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -133,7 +133,8 @@ subroutine RealizeAccepted(model, rc) allocate(importItemTypeList(importItemCount)) ! query importState - call ESMF_StateGet(importState, nestedFlag=.false., itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) + call ESMF_StateGet(importState, nestedFlag=.false., & + itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -144,14 +145,16 @@ subroutine RealizeAccepted(model, rc) ! query item if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then ! pull out nested state - call ESMF_StateGet(importState, itemName=importItemNameList(i), nestedState=importNestedState, rc=rc) + call ESMF_StateGet(importState, itemName=importItemNameList(i), & + nestedState=importNestedState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & - return ! bail out + return ! bail out ! query nested state - call ESMF_StateGet(importNestedState, name=nestedStateName, itemCount=importNestedItemCount, rc=rc) + call ESMF_StateGet(importNestedState, name=nestedStateName, & + itemCount=importNestedItemCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -162,7 +165,8 @@ subroutine RealizeAccepted(model, rc) allocate(importNestedItemTypeList(importNestedItemCount)) ! query item name and types in the nested state - call ESMF_StateGet(importNestedState, itemNameList=importNestedItemNameList, & + call ESMF_StateGet(importNestedState, & + itemNameList=importNestedItemNameList, & itemTypeList=importNestedItemTypeList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -171,9 +175,13 @@ subroutine RealizeAccepted(model, rc) ! loop over items in importNestedState do j = 1, importNestedItemCount - call ESMF_LogWrite('realize '//trim(importNestedItemNameList(j))//' import field received from '//trim(nestedStateName), ESMF_LOGMSG_INFO) + call ESMF_LogWrite('realize '//& + trim(importNestedItemNameList(j))//& + ' import field received from '//trim(nestedStateName), & + ESMF_LOGMSG_INFO) - call NUOPC_Realize(importNestedState, fieldName=trim(importNestedItemNameList(j)), rc=rc) + call NUOPC_Realize(importNestedState, & + fieldName=trim(importNestedItemNameList(j)), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -197,12 +205,12 @@ subroutine RealizeAccepted(model, rc) subroutine Advance(model, rc) type(ESMF_GridComp) :: model integer, intent(out) :: rc - + ! local variables type(ESMF_Clock) :: clock type(ESMF_State) :: importState, exportState type(ESMF_State) :: importNestedState - character(ESMF_MAXSTR) :: nestedStateName + character(ESMF_MAXSTR) :: namespace character(ESMF_MAXSTR), allocatable :: importItemNameList(:) type(ESMF_StateItem_Flag), allocatable :: importItemTypeList(:) integer :: i, importItemCount @@ -210,7 +218,7 @@ subroutine Advance(model, rc) character(len=160) :: msgString rc = ESMF_SUCCESS - + ! query for clock, importState and exportState call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & exportState=exportState, rc=rc) @@ -220,12 +228,12 @@ subroutine Advance(model, rc) return ! bail out ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep - + ! Because of the way that the internal Clock was set by default, ! its timeStep is equal to the parent timeStep. As a consequence the ! currTime + timeStep is equal to the stopTime of the internal Clock ! for this call of the Advance() routine. - + call ESMF_ClockPrint(clock, options="currTime", & preString="------>Advancing ATM from: ", unit=msgString, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -237,7 +245,7 @@ subroutine Advance(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out - + call ESMF_ClockPrint(clock, options="stopTime", & preString="---------------------> to: ", unit=msgString, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -258,7 +266,7 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out - ! query number of item in importState + ! query number of item in importState call ESMF_StateGet(importState, itemCount=importItemCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -270,7 +278,8 @@ subroutine Advance(model, rc) allocate(importItemTypeList(importItemCount)) ! query importState - call ESMF_StateGet(importState, nestedFlag=.false., itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) + call ESMF_StateGet(importState, nestedFlag=.false., & + itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -278,35 +287,36 @@ subroutine Advance(model, rc) ! loop over items in importState do i = 1, importItemCount - ! query item - if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then - ! pull out nested state - call ESMF_StateGet(importState, & - itemName=importItemNameList(i), nestedState=importNestedState, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - - call ESMF_StateGet(importNestedState, name=nestedStateName, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - - ! convert state name to lower case - nestedStateName = ESMF_UtilStringLowerCase(nestedStateName) - - ! write out the Fields in the importState and exportState - call NUOPC_Write(importNestedState, & - fileNamePrefix='field_atm_import_from_'//trim(nestedStateName)//'_', & - timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - end if - end do + ! query item + if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then + ! pull out nested state + call ESMF_StateGet(importState, & + itemName=importItemNameList(i), nestedState=importNestedState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_GetAttribute(importNestedState, name="Namespace", & + value=namespace, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! convert state name to lower case + namespace = ESMF_UtilStringLowerCase(namespace) + + ! write out the Fields in the importState and exportState + call NUOPC_Write(importNestedState, & + fileNamePrefix='field_atm_import_namespace:'//trim(namespace)//'_', & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end if + end do slice = slice+1 end subroutine From 7c2a4f37a8a675ef5c71c63357a4cf0a14436695 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Mon, 9 Dec 2024 11:49:40 -0800 Subject: [PATCH 13/16] Use the updated "transferAllWithNamespace" setting. --- AtmOcnMirrorFieldsNestedProto/atm.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AtmOcnMirrorFieldsNestedProto/atm.F90 b/AtmOcnMirrorFieldsNestedProto/atm.F90 index bf19cb3..fe63085 100644 --- a/AtmOcnMirrorFieldsNestedProto/atm.F90 +++ b/AtmOcnMirrorFieldsNestedProto/atm.F90 @@ -86,7 +86,7 @@ subroutine Advertise(model, rc) return ! bail out call NUOPC_SetAttribute(importState, "FieldTransferPolicy", & - "transferAllAsNests", rc=rc) + "transferAllWithNamespace", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & From b20b62b6828fd17cbd65a6f9b259c079dba95a59 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Mon, 9 Dec 2024 11:54:59 -0800 Subject: [PATCH 14/16] Update the proto name to better align with mirror transfer option naming. --- .../Makefile | 0 .../README | 0 .../atm.F90 | 0 .../esm.F90 | 0 .../esmApp.F90 | 0 .../ocn.F90 | 0 testProtos.sh | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename {AtmOcnMirrorFieldsNestedProto => AtmOcnMirrorFieldsWithNamespaceProto}/Makefile (100%) rename {AtmOcnMirrorFieldsNestedProto => AtmOcnMirrorFieldsWithNamespaceProto}/README (100%) rename {AtmOcnMirrorFieldsNestedProto => AtmOcnMirrorFieldsWithNamespaceProto}/atm.F90 (100%) rename {AtmOcnMirrorFieldsNestedProto => AtmOcnMirrorFieldsWithNamespaceProto}/esm.F90 (100%) rename {AtmOcnMirrorFieldsNestedProto => AtmOcnMirrorFieldsWithNamespaceProto}/esmApp.F90 (100%) rename {AtmOcnMirrorFieldsNestedProto => AtmOcnMirrorFieldsWithNamespaceProto}/ocn.F90 (100%) diff --git a/AtmOcnMirrorFieldsNestedProto/Makefile b/AtmOcnMirrorFieldsWithNamespaceProto/Makefile similarity index 100% rename from AtmOcnMirrorFieldsNestedProto/Makefile rename to AtmOcnMirrorFieldsWithNamespaceProto/Makefile diff --git a/AtmOcnMirrorFieldsNestedProto/README b/AtmOcnMirrorFieldsWithNamespaceProto/README similarity index 100% rename from AtmOcnMirrorFieldsNestedProto/README rename to AtmOcnMirrorFieldsWithNamespaceProto/README diff --git a/AtmOcnMirrorFieldsNestedProto/atm.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/atm.F90 similarity index 100% rename from AtmOcnMirrorFieldsNestedProto/atm.F90 rename to AtmOcnMirrorFieldsWithNamespaceProto/atm.F90 diff --git a/AtmOcnMirrorFieldsNestedProto/esm.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/esm.F90 similarity index 100% rename from AtmOcnMirrorFieldsNestedProto/esm.F90 rename to AtmOcnMirrorFieldsWithNamespaceProto/esm.F90 diff --git a/AtmOcnMirrorFieldsNestedProto/esmApp.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/esmApp.F90 similarity index 100% rename from AtmOcnMirrorFieldsNestedProto/esmApp.F90 rename to AtmOcnMirrorFieldsWithNamespaceProto/esmApp.F90 diff --git a/AtmOcnMirrorFieldsNestedProto/ocn.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/ocn.F90 similarity index 100% rename from AtmOcnMirrorFieldsNestedProto/ocn.F90 rename to AtmOcnMirrorFieldsWithNamespaceProto/ocn.F90 diff --git a/testProtos.sh b/testProtos.sh index 958a06e..8f98afe 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -418,7 +418,7 @@ TestProto AtmOcnMedPetListTimescalesProto esmApp TestProto AtmOcnMedPetListTimescalesSplitFastProto esmApp TestProto AtmOcnMedProto esmApp TestProto AtmOcnMirrorFieldsProto esmApp -TestProto AtmOcnMirrorFieldsNestedProto esmApp +TestProto AtmOcnMirrorFieldsWithNamespaceProto esmApp TestProto AtmOcnPetListProto esmApp TestProto AtmOcnProto esmApp TestProto AtmOcnRtmTwoTimescalesProto esmApp From eb2125ef39b4aaa197b50a7aec5a24ae3a8cb95c Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Mon, 9 Dec 2024 12:07:55 -0800 Subject: [PATCH 15/16] Improve READMEs. --- AtmOcnMirrorFieldsProto/README | 21 +++++++++--------- AtmOcnMirrorFieldsWithNamespaceProto/README | 24 +++++++++++---------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/AtmOcnMirrorFieldsProto/README b/AtmOcnMirrorFieldsProto/README index fd77dbe..4e70031 100644 --- a/AtmOcnMirrorFieldsProto/README +++ b/AtmOcnMirrorFieldsProto/README @@ -12,28 +12,29 @@ Description: The ESM driver uses explicitly constructed petLists when adding the two model components. - + The ESM driver component uses the default run sequence to implement coupling - between ATM and OCN components. - + between ATM and OCN components. + The connector components are explicitly added by the driver. - + The OCN component use a simple two-phase initialization, consisting of advertise and realize. - - The ATM component uses a three-phase initialization, consisting of - advertise/request-mirror, modify mirror-advertised fields, realize. - + + The ATM component uses a three-phase initialization, consisting of + advertise/request-mirror with FieldTransferPolicy="transferAll", + modify mirror-advertised fields, realize. + Build: - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF installation. - make - + Execution: - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) - + Output: - PET*.Log files containing compliance checker output if turned on. - The prototype outputs time stepping information to stdout. diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/README b/AtmOcnMirrorFieldsWithNamespaceProto/README index 126229a..b5a459c 100644 --- a/AtmOcnMirrorFieldsWithNamespaceProto/README +++ b/AtmOcnMirrorFieldsWithNamespaceProto/README @@ -2,7 +2,7 @@ README for ATM-OCN with field mirroring NUOPC prototype ------------------------------------------------------- A simple two model system, where one Model component requests field mirroring -on one of its States. +with Namesapces on one of its States. Description: @@ -11,28 +11,29 @@ Description: The ESM driver uses explicitly constructed petLists when adding the two model components. - + The ESM driver component uses the default run sequence to implement coupling - between ATM and OCN components. - + between ATM and OCN components. + The connector components are explicitly added by the driver. - + The OCN component use a simple two-phase initialization, consisting of advertise and realize. - - The ATM component uses a three-phase initialization, consisting of - advertise/request-mirror, modify mirror-advertised fields, realize. - + + The ATM component uses a three-phase initialization, consisting of + advertise/request-mirror with FieldTransferPolicy="transferAllWithNamespace", + modify mirror-advertised fields, realize. + Build: - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF installation. - make - + Execution: - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) - + Output: - PET*.Log files containing compliance checker output if turned on. - The prototype outputs time stepping information to stdout. @@ -44,3 +45,4 @@ Code structure: - esm.F90 - The Earth System Model (ESM) component, specializing generic NUOPC_Driver. Define partial petLists for ATM and OCN. - esmApp.F90 - ESM application. + From 48d60184548b32011a07d41fcc24c4240dd09c30 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Thu, 12 Dec 2024 17:48:21 -0800 Subject: [PATCH 16/16] Add AtmOcnMirrorFieldsWithNamespaceProto artifacts. --- .gitignore | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitignore b/.gitignore index f749a57..218aa50 100644 --- a/.gitignore +++ b/.gitignore @@ -301,6 +301,21 @@ AtmOcnMirrorFieldsProto/field_ocn_import_pmsl.nc AtmOcnMirrorFieldsProto/field_ocn_import_rsns.nc AtmOcnMirrorFieldsProto/ocn.mod AtmOcnMirrorFieldsProto/ocn.o +AtmOcnMirrorFieldsWithNamespaceProto/PET0.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/PET1.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/PET2.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/PET3.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/atm.mod +AtmOcnMirrorFieldsWithNamespaceProto/atm.o +AtmOcnMirrorFieldsWithNamespaceProto/esm.mod +AtmOcnMirrorFieldsWithNamespaceProto/esm.o +AtmOcnMirrorFieldsWithNamespaceProto/esmApp +AtmOcnMirrorFieldsWithNamespaceProto/esmApp.o +AtmOcnMirrorFieldsWithNamespaceProto/esmApp.stdout +AtmOcnMirrorFieldsWithNamespaceProto/field_atm_import_namespace:ocn_sea_surface_temperature.nc +AtmOcnMirrorFieldsWithNamespaceProto/field_ocn_export_sst.nc +AtmOcnMirrorFieldsWithNamespaceProto/ocn.mod +AtmOcnMirrorFieldsWithNamespaceProto/ocn.o AtmOcnPetListProto/PET0.ESMF_LogFile AtmOcnPetListProto/PET1.ESMF_LogFile AtmOcnPetListProto/PET2.ESMF_LogFile