From 276563041ea6a2b6b4c70cbfa5173fb85db7b47f Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Thu, 12 Oct 2023 12:41:17 -0700 Subject: [PATCH] Add perlmutter gnu, intel, cray port (#882) --- configuration/scripts/cice.batch.csh | 17 ++++++ configuration/scripts/cice.launch.csh | 5 +- .../scripts/machines/Macros.perlmutter_cray | 56 ++++++++++++++++++ .../scripts/machines/Macros.perlmutter_gnu | 57 +++++++++++++++++++ .../scripts/machines/Macros.perlmutter_intel | 57 +++++++++++++++++++ .../scripts/machines/env.perlmutter_cray | 51 +++++++++++++++++ .../scripts/machines/env.perlmutter_gnu | 51 +++++++++++++++++ .../scripts/machines/env.perlmutter_intel | 51 +++++++++++++++++ 8 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 configuration/scripts/machines/Macros.perlmutter_cray create mode 100644 configuration/scripts/machines/Macros.perlmutter_gnu create mode 100644 configuration/scripts/machines/Macros.perlmutter_intel create mode 100644 configuration/scripts/machines/env.perlmutter_cray create mode 100644 configuration/scripts/machines/env.perlmutter_gnu create mode 100644 configuration/scripts/machines/env.perlmutter_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 263b16d02..33b27cbf8 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -168,6 +168,23 @@ cat >> ${jobfile} << EOFB ###SBATCH --mail-user username@domain.com EOFB +else if (${ICE_MACHINE} =~ perlmutter*) then +@ nthrds2 = ${nthrds} * 2 +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -A ${acct} +#SBATCH --qos=${queue} +#SBATCH --time=${batchtime} +#SBATCH --nodes=${nnodes} +#SBATCH --ntasks=${ntasks} +#SBATCH --cpus-per-task=${nthrds2} +#SBATCH --constraint cpu +###SBATCH -e filename +###SBATCH -o filename +###SBATCH --mail-type FAIL +###SBATCH --mail-user username@domain.com +EOFB + else if (${ICE_MACHINE} =~ compy*) then if (${runlength} <= 2) set queue = "short" cat >> ${jobfile} <&! \$ICE_RUNLO EOFR #======= -else if (${ICE_MACHCOMP} =~ cori*) then +else if (${ICE_MACHCOMP} =~ cori* || ${ICE_MACHCOMP} =~ perlmutter*) then if (${ICE_COMMDIR} =~ serial*) then cat >> ${jobfile} << EOFR -./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +srun --cpu-bind=cores ./cice >&! \$ICE_RUNLOG_FILE EOFR else cat >> ${jobfile} << EOFR diff --git a/configuration/scripts/machines/Macros.perlmutter_cray b/configuration/scripts/machines/Macros.perlmutter_cray new file mode 100644 index 000000000..cc4132fa9 --- /dev/null +++ b/configuration/scripts/machines/Macros.perlmutter_cray @@ -0,0 +1,56 @@ +#============================================================================== +# Macros file for NERSC perlmutter, cray compiler +#============================================================================== + +CPP := ftn -e P +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := +FFLAGS := -hbyteswapio +FFLAGS_NOOPT:= -O0 +LDFLAGS := -hbyteswapio + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -hfp0 -g -Rbcdps -Ktrap=fp +else + FFLAGS += -O2 -hfp0 # -eo +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + diff --git a/configuration/scripts/machines/Macros.perlmutter_gnu b/configuration/scripts/machines/Macros.perlmutter_gnu new file mode 100644 index 000000000..220d2dd80 --- /dev/null +++ b/configuration/scripts/machines/Macros.perlmutter_gnu @@ -0,0 +1,57 @@ +#============================================================================== +# Macros file for NERSC perlmutter, gnu compiler +#============================================================================== + +CPP := ftn -E +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c + +FIXEDFLAGS := -ffixed-line-length-132 +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +else + FFLAGS += -O2 + CFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + diff --git a/configuration/scripts/machines/Macros.perlmutter_intel b/configuration/scripts/machines/Macros.perlmutter_intel new file mode 100644 index 000000000..ce781be44 --- /dev/null +++ b/configuration/scripts/machines/Macros.perlmutter_intel @@ -0,0 +1,57 @@ +#============================================================================== +# Macros file for NERSC perlmutter, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -march=core-avx2 + +FIXEDFLAGS := -fixed -132 +FREEFLAGS := -free +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2 +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08 +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.perlmutter_cray b/configuration/scripts/machines/env.perlmutter_cray new file mode 100644 index 000000000..04ee3ce94 --- /dev/null +++ b/configuration/scripts/machines/env.perlmutter_cray @@ -0,0 +1,51 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +#module unload PrgEnv-aocc +#module unload PrgEnv-cray +#module unload PrgEnv-gnu +#module unload PrgEnv-intel +#module unload PrgEnv-nvidia +#module unload gpu +module load cpu +module load PrgEnv-cray +module unload cce +module load cce/15.0.1 +module unload cray-mpich +module load cray-mpich/8.1.25 + +module unload cray-netcdf +module unload cray-hdf5 +module load cray-hdf5/1.12.2.3 +module load cray-netcdf/4.9.0.3 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE + +endif + +setenv ICE_MACHINE_MACHNAME perlmutter +setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect" +setenv ICE_MACHINE_ENVNAME cray +setenv ICE_MACHINE_ENVINFO "Cray clang/Fortran 15.0.1, cray-mpich/8.1.25, netcdf/4.9.0.3" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $SCRATCH/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium +setenv ICE_MACHINE_BASELINE $SCRATCH/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "regular" +setenv ICE_MACHINE_TPNODE 128 +setenv ICE_MACHINE_BLDTHRDS 8 +setenv ICE_MACHINE_QSTAT "squeue --jobs= " diff --git a/configuration/scripts/machines/env.perlmutter_gnu b/configuration/scripts/machines/env.perlmutter_gnu new file mode 100644 index 000000000..42e510e55 --- /dev/null +++ b/configuration/scripts/machines/env.perlmutter_gnu @@ -0,0 +1,51 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +#module unload PrgEnv-aocc +#module unload PrgEnv-cray +#module unload PrgEnv-gnu +#module unload PrgEnv-intel +#module unload PrgEnv-nvidia +#module unload gpu +module load cpu +module load PrgEnv-gnu +module unload gcc +module load gcc/11.2.0 +module unload cray-mpich +module load cray-mpich/8.1.25 + +module unload cray-netcdf +module unload cray-hdf5 +module load cray-hdf5/1.12.2.3 +module load cray-netcdf/4.9.0.3 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE + +endif + +setenv ICE_MACHINE_MACHNAME perlmutter +setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect" +setenv ICE_MACHINE_ENVNAME gnu +setenv ICE_MACHINE_ENVINFO "gnu c/fortran 11.2.0 20210728, cray-mpich/8.1.25, netcdf/4.9.0.3" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $SCRATCH/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium +setenv ICE_MACHINE_BASELINE $SCRATCH/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "regular" +setenv ICE_MACHINE_TPNODE 128 +setenv ICE_MACHINE_BLDTHRDS 8 +setenv ICE_MACHINE_QSTAT "squeue --jobs= " diff --git a/configuration/scripts/machines/env.perlmutter_intel b/configuration/scripts/machines/env.perlmutter_intel new file mode 100644 index 000000000..7ecdc0f96 --- /dev/null +++ b/configuration/scripts/machines/env.perlmutter_intel @@ -0,0 +1,51 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +#module unload PrgEnv-aocc +#module unload PrgEnv-cray +#module unload PrgEnv-gnu +#module unload PrgEnv-intel +#module unload PrgEnv-nvidia +#module unload gpu +module load cpu +module load PrgEnv-intel +module unload intel +module load intel/2023.1.0 +module unload cray-mpich +module load cray-mpich/8.1.25 + +module unload cray-netcdf +module unload cray-hdf5 +module load cray-hdf5/1.12.2.3 +module load cray-netcdf/4.9.0.3 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE + +endif + +setenv ICE_MACHINE_MACHNAME perlmutter +setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "ifort 2021.9.0 20230302, Intel oneAPI DPC++/C++ 2023.1.0 (2023.1.0.20230320), cray-mpich/8.1.25, netcdf/4.9.0.3" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $SCRATCH/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium +setenv ICE_MACHINE_BASELINE $SCRATCH/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "regular" +setenv ICE_MACHINE_TPNODE 128 +setenv ICE_MACHINE_BLDTHRDS 8 +setenv ICE_MACHINE_QSTAT "squeue --jobs= "