diff --git a/.gitignore b/.gitignore index f749a57..218aa50 100644 --- a/.gitignore +++ b/.gitignore @@ -301,6 +301,21 @@ AtmOcnMirrorFieldsProto/field_ocn_import_pmsl.nc AtmOcnMirrorFieldsProto/field_ocn_import_rsns.nc AtmOcnMirrorFieldsProto/ocn.mod AtmOcnMirrorFieldsProto/ocn.o +AtmOcnMirrorFieldsWithNamespaceProto/PET0.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/PET1.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/PET2.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/PET3.ESMF_LogFile +AtmOcnMirrorFieldsWithNamespaceProto/atm.mod +AtmOcnMirrorFieldsWithNamespaceProto/atm.o +AtmOcnMirrorFieldsWithNamespaceProto/esm.mod +AtmOcnMirrorFieldsWithNamespaceProto/esm.o +AtmOcnMirrorFieldsWithNamespaceProto/esmApp +AtmOcnMirrorFieldsWithNamespaceProto/esmApp.o +AtmOcnMirrorFieldsWithNamespaceProto/esmApp.stdout +AtmOcnMirrorFieldsWithNamespaceProto/field_atm_import_namespace:ocn_sea_surface_temperature.nc +AtmOcnMirrorFieldsWithNamespaceProto/field_ocn_export_sst.nc +AtmOcnMirrorFieldsWithNamespaceProto/ocn.mod +AtmOcnMirrorFieldsWithNamespaceProto/ocn.o AtmOcnPetListProto/PET0.ESMF_LogFile AtmOcnPetListProto/PET1.ESMF_LogFile AtmOcnPetListProto/PET2.ESMF_LogFile diff --git a/AtmOcnMirrorFieldsProto/README b/AtmOcnMirrorFieldsProto/README index fd77dbe..4e70031 100644 --- a/AtmOcnMirrorFieldsProto/README +++ b/AtmOcnMirrorFieldsProto/README @@ -12,28 +12,29 @@ Description: The ESM driver uses explicitly constructed petLists when adding the two model components. - + The ESM driver component uses the default run sequence to implement coupling - between ATM and OCN components. - + between ATM and OCN components. + The connector components are explicitly added by the driver. - + The OCN component use a simple two-phase initialization, consisting of advertise and realize. - - The ATM component uses a three-phase initialization, consisting of - advertise/request-mirror, modify mirror-advertised fields, realize. - + + The ATM component uses a three-phase initialization, consisting of + advertise/request-mirror with FieldTransferPolicy="transferAll", + modify mirror-advertised fields, realize. + Build: - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF installation. - make - + Execution: - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) - + Output: - PET*.Log files containing compliance checker output if turned on. - The prototype outputs time stepping information to stdout. diff --git a/AtmOcnMirrorFieldsProto/atm.F90 b/AtmOcnMirrorFieldsProto/atm.F90 index 462652e..73ff534 100644 --- a/AtmOcnMirrorFieldsProto/atm.F90 +++ b/AtmOcnMirrorFieldsProto/atm.F90 @@ -245,7 +245,13 @@ subroutine ModifyAdvertised(model, rc) !TODO: settings, and go between components on different threading levels. !TODO: This is an edge case, but long term it should be supported. -#if 0 +!TODO: The problem in StateReconcile() comes from the UNIQUE_GEOM_INFO_TREAT_on +!TODO: handling. Once that part is reworked inside StateReconcile() the mixed +!TODO: sharing case should work again! + +!TODO: For now share Grid and Field for both fields. + +#if 1 ! set SharePolicyGeomObject = "share" call NUOPC_SetAttribute(field, name="SharePolicyGeomObject", & value="share", rc=rc) @@ -254,7 +260,7 @@ subroutine ModifyAdvertised(model, rc) file=__FILE__)) & return ! bail out #endif -#if 0 +#if 1 ! set SharePolicyField = "share" call NUOPC_SetAttribute(field, name="SharePolicyField", & value="share", rc=rc) diff --git a/AtmOcnMirrorFieldsProto/ocn.F90 b/AtmOcnMirrorFieldsProto/ocn.F90 index 96c3fed..f5a8a69 100644 --- a/AtmOcnMirrorFieldsProto/ocn.F90 +++ b/AtmOcnMirrorFieldsProto/ocn.F90 @@ -96,6 +96,7 @@ subroutine Advertise(model, rc) StandardName="air_pressure_at_sea_level", name="pmsl", & #ifdef TEST_SHARING SharePolicyGeomObject="share", & + SharePolicyField="share", & #endif rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/Makefile b/AtmOcnMirrorFieldsWithNamespaceProto/Makefile new file mode 100644 index 0000000..991c603 --- /dev/null +++ b/AtmOcnMirrorFieldsWithNamespaceProto/Makefile @@ -0,0 +1,76 @@ +# GNU Makefile template for user ESMF application + +################################################################################ +################################################################################ +## This Makefile must be able to find the "esmf.mk" Makefile fragment in the ## +## 'include' line below. Following the ESMF User's Guide, a complete ESMF ## +## installation should ensure that a single environment variable "ESMFMKFILE" ## +## is made available on the system. This variable should point to the ## +## "esmf.mk" file. ## +## ## +## This example Makefile uses the "ESMFMKFILE" environment variable. ## +## ## +## If you notice that this Makefile cannot find variable ESMFMKFILE then ## +## please contact the person responsible for the ESMF installation on your ## +## system. ## +## As a work-around you can simply hardcode the path to "esmf.mk" in the ## +## include line below. However, doing so will render this Makefile a lot less ## +## flexible and non-portable. ## +################################################################################ + +ifneq ($(origin ESMFMKFILE), environment) +$(error Environment variable ESMFMKFILE was not set.) +endif + +include $(ESMFMKFILE) + +# strip quotes around the ESMF_INTERNAL_MPIRUN value +ESMF_INTERNAL_MPIRUN := $(shell echo $(ESMF_INTERNAL_MPIRUN)) + +################################################################################ +################################################################################ + +.SUFFIXES: .f90 .F90 .c .C + +%.o : %.f90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREENOCPP) $< + +%.o : %.F90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $< + +%.o : %.c + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + +%.o : %.C + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + + +# ----------------------------------------------------------------------------- +esmApp: esmApp.o esm.o atm.o ocn.o + $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS) + +# module dependencies: +esmApp.o: esm.o +esm.o: atm.o ocn.o + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +.PHONY: dust clean distclean info edit +dust: + rm -f PET*.ESMF_LogFile *.nc *.stdout +clean: + rm -f esmApp *.o *.mod +distclean: dust clean + +info: + @echo ================================================================== + @echo ESMFMKFILE=$(ESMFMKFILE) + @echo ================================================================== + @cat $(ESMFMKFILE) + @echo ================================================================== + +edit: + nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & + +run: + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/README b/AtmOcnMirrorFieldsWithNamespaceProto/README new file mode 100644 index 0000000..b5a459c --- /dev/null +++ b/AtmOcnMirrorFieldsWithNamespaceProto/README @@ -0,0 +1,48 @@ +README for ATM-OCN with field mirroring NUOPC prototype +------------------------------------------------------- + +A simple two model system, where one Model component requests field mirroring +with Namesapces on one of its States. + +Description: + + A two-way coupled system with a single driver (ESM) and two model components + (ATM, OCN). + + The ESM driver uses explicitly constructed petLists when adding the two + model components. + + The ESM driver component uses the default run sequence to implement coupling + between ATM and OCN components. + + The connector components are explicitly added by the driver. + + The OCN component use a simple two-phase initialization, consisting of + advertise and realize. + + The ATM component uses a three-phase initialization, consisting of + advertise/request-mirror with FieldTransferPolicy="transferAllWithNamespace", + modify mirror-advertised fields, realize. + + +Build: + - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF + installation. + - make + +Execution: + - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. + - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) + +Output: + - PET*.Log files containing compliance checker output if turned on. + - The prototype outputs time stepping information to stdout. + +Code structure: + - Makefile - Makefile that is based on the standard esmf.mk mechanism. + - atm.F90 - The ATM component, specializing generic NUOPC_Model. + - ocn.F90 - The OCN component, specializing generic NUOPC_Model. + - esm.F90 - The Earth System Model (ESM) component, specializing generic + NUOPC_Driver. Define partial petLists for ATM and OCN. + - esmApp.F90 - ESM application. + diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/atm.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/atm.F90 new file mode 100644 index 0000000..fe63085 --- /dev/null +++ b/AtmOcnMirrorFieldsWithNamespaceProto/atm.F90 @@ -0,0 +1,326 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module ATM + + !----------------------------------------------------------------------------- + ! ATM Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_RealizeAccepted, & + specRoutine=RealizeAccepted, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_SetAttribute(importState, "FieldTransferPolicy", & + "transferAllWithNamespace", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine RealizeAccepted(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + integer :: i, j + type(ESMF_State) :: importState, exportState + integer :: importItemCount, importNestedItemCount + type(ESMF_State) :: importNestedState + character(ESMF_MAXSTR) :: nestedStateName + character(ESMF_MAXSTR), allocatable :: importItemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: importItemTypeList(:) + character(ESMF_MAXSTR), allocatable :: importNestedItemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: importNestedItemTypeList(:) + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query nested States + call ESMF_StateGet(importState, itemCount=importItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! allocate temporary data structures + allocate(importItemNameList(importItemCount)) + allocate(importItemTypeList(importItemCount)) + + ! query importState + call ESMF_StateGet(importState, nestedFlag=.false., & + itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! loop over items in importState + do i = 1, importItemCount + ! query item + if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then + ! pull out nested state + call ESMF_StateGet(importState, itemName=importItemNameList(i), & + nestedState=importNestedState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query nested state + call ESMF_StateGet(importNestedState, name=nestedStateName, & + itemCount=importNestedItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! allocate temporary data structures + allocate(importNestedItemNameList(importNestedItemCount)) + allocate(importNestedItemTypeList(importNestedItemCount)) + + ! query item name and types in the nested state + call ESMF_StateGet(importNestedState, & + itemNameList=importNestedItemNameList, & + itemTypeList=importNestedItemTypeList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! loop over items in importNestedState + do j = 1, importNestedItemCount + call ESMF_LogWrite('realize '//& + trim(importNestedItemNameList(j))//& + ' import field received from '//trim(nestedStateName), & + ESMF_LOGMSG_INFO) + + call NUOPC_Realize(importNestedState, & + fieldName=trim(importNestedItemNameList(j)), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end do + + ! clean memory + deallocate(importNestedItemNameList) + deallocate(importNestedItemTypeList) + end if + end do + + ! clean memory + deallocate(importItemNameList) + deallocate(importItemTypeList) + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_State) :: importNestedState + character(ESMF_MAXSTR) :: namespace + character(ESMF_MAXSTR), allocatable :: importItemNameList(:) + type(ESMF_StateItem_Flag), allocatable :: importItemTypeList(:) + integer :: i, importItemCount + integer, save :: slice=1 + character(len=160) :: msgString + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set by default, + ! its timeStep is equal to the parent timeStep. As a consequence the + ! currTime + timeStep is equal to the stopTime of the internal Clock + ! for this call of the Advance() routine. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="------>Advancing ATM from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockPrint(clock, options="stopTime", & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query nested States + call ESMF_StateGet(importState, nestedFlag=.false., & + itemCount=importItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! query number of item in importState + call ESMF_StateGet(importState, itemCount=importItemCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! allocate temporary data structures + allocate(importItemNameList(importItemCount)) + allocate(importItemTypeList(importItemCount)) + + ! query importState + call ESMF_StateGet(importState, nestedFlag=.false., & + itemNameList=importItemNameList, itemTypeList=importItemTypeList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! loop over items in importState + do i = 1, importItemCount + ! query item + if (importItemTypeList(i) == ESMF_STATEITEM_STATE) then + ! pull out nested state + call ESMF_StateGet(importState, & + itemName=importItemNameList(i), nestedState=importNestedState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_GetAttribute(importNestedState, name="Namespace", & + value=namespace, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! convert state name to lower case + namespace = ESMF_UtilStringLowerCase(namespace) + + ! write out the Fields in the importState and exportState + call NUOPC_Write(importNestedState, & + fileNamePrefix='field_atm_import_namespace:'//trim(namespace)//'_', & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end if + end do + slice = slice+1 + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/esm.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/esm.F90 new file mode 100644 index 0000000..3d88c93 --- /dev/null +++ b/AtmOcnMirrorFieldsWithNamespaceProto/esm.F90 @@ -0,0 +1,204 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +! For testing with partially overlapping components keep one active, other _off +#define WITH_ATM_PETLIST +#define WITH_OCN_PETLIST + +module ESM + + !----------------------------------------------------------------------------- + ! Code that specializes generic ESM Component code. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Driver, & + driverSS => SetServices + + use ATM, only: atmSS => SetServices + use OCN, only: ocnSS => SetServices + + use NUOPC_Connector, only: cplSS => SetServices + + implicit none + + private + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Driver + call NUOPC_CompDerive(driver, driverSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize driver + call NUOPC_CompSpecialize(driver, specLabel=label_SetModelServices, & + specRoutine=SetModelServices, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set driver verbosity + call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetModelServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + ! local variables + type(ESMF_Grid) :: grid + type(ESMF_Field) :: field + type(ESMF_Time) :: startTime + type(ESMF_Time) :: stopTime + type(ESMF_TimeInterval) :: timeStep + type(ESMF_Clock) :: internalClock + type(ESMF_GridComp) :: child + type(ESMF_CplComp) :: connector + integer :: petCount, i + integer, allocatable :: petList(:) + + rc = ESMF_SUCCESS + + ! get the petCount + call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ATM with petList on first half of PETs + allocate(petList(petCount/2)) + do i=1, petCount/2 + petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 + enddo + call NUOPC_DriverAddComp(driver, "ATM", atmSS, & +#ifdef WITH_ATM_PETLIST + petList=petList, & +#endif + comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(petList) + + ! SetServices for OCN with petList on second half of PETs + allocate(petList(petCount/2)) + do i=1, petCount/2 + petList(i) = petCount/2 + i-1 ! PET labeling goes from 0 to petCount-1 + enddo + call NUOPC_DriverAddComp(driver, "OCN", ocnSS, & +#ifdef WITH_OCN_PETLIST + petList=petList, & +#endif + comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(petList) + + ! SetServices for atm2ocn + call NUOPC_DriverAddComp(driver, srcCompLabel="ATM", dstCompLabel="OCN", & + compSetServicesRoutine=cplSS, comp=connector, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(connector, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ocn2atm + call NUOPC_DriverAddComp(driver, srcCompLabel="OCN", dstCompLabel="ATM", & + compSetServicesRoutine=cplSS, comp=connector, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(connector, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set the driver clock + call ESMF_TimeIntervalSet(timeStep, m=15, rc=rc) ! 15 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimeSet(startTime, yy=2010, mm=6, dd=1, h=0, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimeSet(stopTime, yy=2010, mm=6, dd=1, h=1, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + internalClock = ESMF_ClockCreate(name="Application Clock", & + timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_GridCompSet(driver, clock=internalClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/esmApp.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/esmApp.F90 new file mode 100644 index 0000000..8796566 --- /dev/null +++ b/AtmOcnMirrorFieldsWithNamespaceProto/esmApp.F90 @@ -0,0 +1,106 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +program esmApp + + !----------------------------------------------------------------------------- + ! Generic ESM application driver + !----------------------------------------------------------------------------- + + use ESMF + use ESM, only: esmSS => SetServices + + implicit none + + integer :: rc, urc + type(ESMF_GridComp) :: esmComp + + ! Initialize ESMF + call ESMF_Initialize(logkindflag=ESMF_LOGKIND_MULTI, & + defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_LogWrite("esmApp STARTING", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Create the earth system Component + esmComp = ESMF_GridCompCreate(name="esm", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! SetServices for the earth system Component + call ESMF_GridCompSetServices(esmComp, esmSS, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call Initialize for the earth system Component + call ESMF_GridCompInitialize(esmComp, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call Run for earth the system Component + call ESMF_GridCompRun(esmComp, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call Finalize for the earth system Component + call ESMF_GridCompFinalize(esmComp, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Destroy the earth system Component + call ESMF_GridCompDestroy(esmComp, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_LogWrite("esmApp FINISHED", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Finalize ESMF + call ESMF_Finalize() + +end program diff --git a/AtmOcnMirrorFieldsWithNamespaceProto/ocn.F90 b/AtmOcnMirrorFieldsWithNamespaceProto/ocn.F90 new file mode 100644 index 0000000..016677b --- /dev/null +++ b/AtmOcnMirrorFieldsWithNamespaceProto/ocn.F90 @@ -0,0 +1,285 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright (c) 2002-2024, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module OCN + + !----------------------------------------------------------------------------- + ! OCN Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_SetClock, & + specRoutine=SetClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_temperature + call NUOPC_Advertise(exportState, & + StandardName="sea_surface_temperature", name="sst", & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + type(ESMF_TimeInterval) :: stabilityTimeStep + type(ESMF_Field) :: field + type(ESMF_Grid) :: grid + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create a Grid object for Fields + grid = ESMF_GridCreateNoPeriDimUfrm(maxIndex=(/100, 20/), & + minCornerCoord=(/10._ESMF_KIND_R8, 20._ESMF_KIND_R8/), & + maxCornerCoord=(/100._ESMF_KIND_R8, 200._ESMF_KIND_R8/), & + coordSys=ESMF_COORDSYS_CART, staggerLocList=(/ESMF_STAGGERLOC_CENTER/), & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_temperature + field = ESMF_FieldCreate(name="sst", grid=grid, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldFill(field, dataFillScheme="sincos", member=2, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetClock(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_TimeInterval) :: stabilityTimeStep + + rc = ESMF_SUCCESS + + ! query for clock + call NUOPC_ModelGet(model, modelClock=clock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! initialize internal clock + ! here: parent Clock and stability timeStep determine actual model timeStep + !TODO: stabilityTimeStep should be read in from configuation + !TODO: or computed from internal Grid information + call ESMF_TimeIntervalSet(stabilityTimeStep, m=5, rc=rc) ! 5 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSetClock(model, clock, stabilityTimeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_Time) :: currTime + type(ESMF_TimeInterval) :: timeStep + integer, save :: slice=1 + character(len=160) :: msgString + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set in SetClock(), + ! its timeStep is likely smaller than the parent timeStep. As a consequence + ! the time interval covered by a single parent timeStep will result in + ! multiple calls to the Advance() routine. Every time the currTime + ! will come in by one internal timeStep advanced. This goes until the + ! stopTime of the internal Clock has been reached. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="------>Advancing OCN from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockGet(clock, currTime=currTime, timeStep=timeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimePrint(currTime + timeStep, & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! update data in export fields + call ESMF_StateGet(exportState, itemName="sst", field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldFill(field, dataFillScheme="sincos", member=1, & + step=slice, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! write out the Fields in the exportState + call NUOPC_Write(exportState, fileNamePrefix="field_ocn_export_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + slice = slice+1 + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/AtmOcnScalarProto/atm.F90 b/AtmOcnScalarProto/atm.F90 index f607a68..5a80f79 100644 --- a/AtmOcnScalarProto/atm.F90 +++ b/AtmOcnScalarProto/atm.F90 @@ -307,9 +307,9 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out ! access the scalars through a 1D array - scalars => fptr(:,1) if (size(fptr)>0) then ! only use fptr on that PET which holds allocation + scalars => fptr(:,1) do i=lbound(scalars,1), ubound(scalars,1) scalars(i)=(i-1)*10+step enddo diff --git a/ExternalDriverAPIWeakCplDAProto/atmDA.F90 b/ExternalDriverAPIWeakCplDAProto/atmDA.F90 index c0cb3d7..6b7f029 100644 --- a/ExternalDriverAPIWeakCplDAProto/atmDA.F90 +++ b/ExternalDriverAPIWeakCplDAProto/atmDA.F90 @@ -14,7 +14,6 @@ module atmDA ! ATM DA Code !----------------------------------------------------------------------------- - use MPI use ESMF use NUOPC @@ -32,8 +31,8 @@ module atmDA contains !----------------------------------------------------------------------------- - subroutine exec(comm) - integer :: comm + subroutine exec(vm) + type(ESMF_VM) :: vm ! context of this interaction integer :: rc @@ -44,7 +43,8 @@ subroutine exec(comm) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Connect DA code with NUOPC system top component - call nuopc_da_connect(toNuopcTopStandardNames=(/"precipitation_flux"/), & + call nuopc_da_connect(vm=vm, & + toNuopcTopStandardNames=(/"precipitation_flux"/), & fmNuopcTopStandardNames=(/"surface_net_downward_shortwave_flux"/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & diff --git a/ExternalDriverAPIWeakCplDAProto/externalApp.F90 b/ExternalDriverAPIWeakCplDAProto/externalApp.F90 index 46a11c4..94c2625 100644 --- a/ExternalDriverAPIWeakCplDAProto/externalApp.F90 +++ b/ExternalDriverAPIWeakCplDAProto/externalApp.F90 @@ -21,8 +21,9 @@ program externalApp use ESM, only: esmSS => SetServices use nuopc_da, only: & - nuopc_da_init => init, & - nuopc_da_final => final + nuopc_da_init => init, & + nuopc_da_commToVM => commToVM, & + nuopc_da_final => final use atmDA, only: atmDAexec => exec use ocnDA, only: ocnDAexec => exec @@ -31,7 +32,8 @@ program externalApp integer :: rc integer :: size, rank - integer :: splitComm + integer :: commAtmDA, commOcnDA, splitComm + type(ESMF_VM) :: vmAtmDA, vmOcnDA ! Initialize the NUOPC-DA interface call nuopc_da_init(nuopcTopSetServices=esmSS, rc=rc) @@ -41,17 +43,40 @@ program externalApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Split up the MPI_COMM_WORLD into atmDA (first half) and ocnDA (second half) - ! of MPI ranks. Call into the respective DA routine + ! of MPI ranks. call MPI_Comm_size(MPI_COMM_WORLD, size, rc) call MPI_Comm_rank(MPI_COMM_WORLD, rank, rc) if (rank < size/2) then ! atm DA processes call MPI_Comm_split(MPI_COMM_WORLD, 1, rank, splitComm, rc) - call atmDAexec(comm=splitComm) + commAtmDA = splitComm + commOcnDA = MPI_COMM_NULL else ! ocn DA processes call MPI_Comm_split(MPI_COMM_WORLD, 2, rank, splitComm, rc) - call ocnDAexec(comm=splitComm) + commOcnDA = splitComm + commAtmDA = MPI_COMM_NULL + endif + + ! Create corresponding ESMF_VM objects for atmDA and ocnDA + vmAtmDA = nuopc_da_commToVM(commAtmDA, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + vmOcnDA = nuopc_da_commToVM(commOcnDA, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Call into the respective DA routine for ATM and OCN + if (rank < size/2) then + ! atm DA processes + call atmDAexec(vmAtmDA) + else + ! ocn DA processes + call ocnDAexec(vmOcnDA) endif ! Finalize the NUOPC-DA interface diff --git a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 index 910ef35..0c71781 100644 --- a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 +++ b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 @@ -11,7 +11,7 @@ module nuopc_da !----------------------------------------------------------------------------- - ! NUOPC - DA interface code + ! NUOPC - DA interface code - this is generic code that could move into NUOPC !----------------------------------------------------------------------------- use MPI @@ -21,12 +21,12 @@ module nuopc_da implicit none private - + type(ESMF_GridComp) :: nuopcTop type(ESMF_State) :: toNuopcTop, fmNuopcTop type(ESMF_Clock) :: clock - public init, connect, step, final + public init, commToVM, connect, step, final !----------------------------------------------------------------------------- contains @@ -116,10 +116,96 @@ recursive subroutine nuopcTopSetServices(gridcomp, rc) !----------------------------------------------------------------------------- - subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) - character(*), intent(in), optional :: toNuopcTopStandardNames(:) - character(*), intent(in), optional :: fmNuopcTopStandardNames(:) - integer, intent(out) :: rc + ! The following is used in commToVM. Some Mac configurations have issues with this being + ! an internal subroutine to commToVM, so we have pulled it out to a module-level + ! subroutine. (See https://github.com/esmf-org/nuopc-app-prototypes/issues/8.) + recursive subroutine dummySS(gridcomp, rc) + type(ESMF_GridComp) :: gridcomp ! must not be optional + integer, intent(out) :: rc ! must not be optional + rc = ESMF_SUCCESS + end subroutine + + function commToVM(comm, rc) + type(ESMF_VM) :: commToVM + integer, intent(in) :: comm ! MPI communicator + integer, intent(out) :: rc + + type(ESMF_VM) :: vm + integer :: petCount, taskCount, temp_int(1) + integer :: i, j, urc + integer, allocatable :: temp_list(:), petList(:) + type(ESMF_GridComp) :: comp + + call ESMF_VMGetCurrent(vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + call ESMF_VMGet(vm, petCount=petCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + temp_int(1) = 0 ! default indicate task is not in comm + if (comm /= MPI_COMM_NULL) temp_int(1) = 1 ! indicate task is in comm + + allocate(temp_list(petCount)) + + call ESMF_VMAllGather(vm, temp_int, temp_list, 1, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! determine number of tasks in comm + taskCount = 0 + do i=1, petCount + if (temp_list(i)==1) taskCount = taskCount + 1 + enddo + + ! construct petList + allocate(petList(taskCount)) + j = 1 + do i=1, petCount + if (temp_list(i)==1) then + petList(j) = i-1 ! PETs are basis 0 + j = j+1 + endif + enddo + + deallocate(temp_list) + + ! create ESMF component on petList + comp = ESMF_GridCompCreate(petList=petList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + deallocate(petList) + + ! call SetServices with a dummy routine to trigger internal VM creation + call ESMF_GridCompSetServices(comp, dummySS, userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! access the VM to be returned + call ESMF_GridCompGet(comp, vm=commToVM, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + end function commToVM + + !----------------------------------------------------------------------------- + + subroutine connect(vm, toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) + type(ESMF_VM), intent(in) :: vm + character(*), intent(in), optional :: toNuopcTopStandardNames(:) + character(*), intent(in), optional :: fmNuopcTopStandardNames(:) + integer, intent(out) :: rc integer :: urc, phase, i type(ESMF_Field) :: field @@ -128,7 +214,7 @@ subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) if (present(toNuopcTopStandardNames)) then call NUOPC_Advertise(toNuopcTop, StandardNames=toNuopcTopStandardNames, & TransferOfferGeomObject="cannot provide", SharePolicyField="share", & - rc=rc) + vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__, rcToReturn=rc)) return @@ -137,7 +223,7 @@ subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) if (present(fmNuopcTopStandardNames)) then call NUOPC_Advertise(fmNuopcTop, StandardNames=fmNuopcTopStandardNames, & TransferOfferGeomObject="cannot provide", SharePolicyField="share", & - rc=rc) + vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__, rcToReturn=rc)) return diff --git a/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 b/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 index 23877e3..2d1ecbf 100644 --- a/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 +++ b/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 @@ -14,7 +14,6 @@ module ocnDA ! OCN DA Code !----------------------------------------------------------------------------- - use MPI use ESMF use NUOPC @@ -32,8 +31,8 @@ module ocnDA contains !----------------------------------------------------------------------------- - subroutine exec(comm) - integer :: comm + subroutine exec(vm) + type(ESMF_VM) :: vm ! context of this interaction integer :: rc @@ -44,7 +43,8 @@ subroutine exec(comm) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Connect DA code with NUOPC system top component - call nuopc_da_connect(fmNuopcTopStandardNames=(/"sea_surface_temperature"/), & + call nuopc_da_connect(vm=vm, & + fmNuopcTopStandardNames=(/"sea_surface_temperature"/), & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & diff --git a/testProtos.sh b/testProtos.sh index 4ec9f4e..8f98afe 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -26,10 +26,29 @@ then # Darwin systems that use Apple Clang need extra environment variables # to find the libomp installation. These are important for the ESMX # tests where CMake is used under the hood to determine the correct - # compiler and linker flags. Set them here to be available: - export LDFLAGS=-L/opt/homebrew/opt/libomp/lib - export CXXFLAGS=-I/opt/homebrew/opt/libomp/include - export CFLAGS=-I/opt/homebrew/opt/libomp/include + # compiler and linker flags. Set them here to be available. + + # We'll try to get the location from a spack installation of llvm-openmp if possible, + # but if we can't find that, then we'll fall back on assuming that libomp is available + # via homebrew. + homebrew_libomp_dir=/opt/homebrew/opt/libomp + + if command -v spack &>/dev/null; then + # First try getting these from a spack-installed llvm-openmp: + omp_dir=$(spack location -i --first llvm-openmp 2>/dev/null) + if [ $? -ne 0 ]; then + # llvm-openmp isn't installed with spack; fall back on the homebrew location + omp_dir=${homebrew_libomp_dir} + fi + else + # spack isn't found; fall back on the homebrew location + omp_dir=${homebrew_libomp_dir} + fi + + # The spack-based setup requires -Wl,-rpath in addition to -L + export LDFLAGS="-L${omp_dir}/lib -Wl,-rpath,${omp_dir}/lib" + export CXXFLAGS=-I${omp_dir}/include + export CFLAGS=-I${omp_dir}/include fi function TestProto { @@ -399,6 +418,7 @@ TestProto AtmOcnMedPetListTimescalesProto esmApp TestProto AtmOcnMedPetListTimescalesSplitFastProto esmApp TestProto AtmOcnMedProto esmApp TestProto AtmOcnMirrorFieldsProto esmApp +TestProto AtmOcnMirrorFieldsWithNamespaceProto esmApp TestProto AtmOcnPetListProto esmApp TestProto AtmOcnProto esmApp TestProto AtmOcnRtmTwoTimescalesProto esmApp