Skip to content

Commit

Permalink
Standalone gadi (#2)
Browse files Browse the repository at this point in the history
* Standalone gadi setup (#1)
  • Loading branch information
anton-seaice authored and Anton Steketee committed May 2, 2024
1 parent 3e30553 commit d45ea77
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 0 deletions.
25 changes: 25 additions & 0 deletions configuration/scripts/cice.batch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,31 @@ cat >> ${jobfile} << EOFB
###PBS -m be
EOFB

else if (${ICE_MACHINE} =~ gadi*) then
if (${queue} =~ *sr) then #sapphire rapids
@ memuse = ( $ncores * 481 / 100 )
else if (${queue} =~ *bw) then #broadwell
@ memuse = ( $ncores * 457 / 100 )
else if (${queue} =~ *sl) then
@ memuse = ( $ncores * 6 )
else #normal queues
@ memuse = ( $ncores * 395 / 100 )
endif
cat >> ${jobfile} << EOFB
#PBS -q ${queue}
#PBS -P ${ICE_MACHINE_PROJ}
#PBS -N ${ICE_CASENAME}
#PBS -l storage=gdata/${ICE_MACHINE_PROJ}+scratch/${ICE_MACHINE_PROJ}+gdata/ik11
#PBS -l ncpus=${ncores}
#PBS -l mem=${memuse}gb
#PBS -l walltime=${batchtime}
#PBS -j oe
#PBS -W umask=003
#PBS -o ${ICE_CASEDIR}
source /etc/profile.d/modules.csh
module use `echo ${MODULEPATH} | sed 's/:/ /g'` #copy the users modules
EOFB

else if (${ICE_MACHINE} =~ gust*) then
cat >> ${jobfile} << EOFB
#PBS -q ${queue}
Expand Down
12 changes: 12 additions & 0 deletions configuration/scripts/cice.launch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,18 @@ mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./ci
EOFR
endif

#=======
else if (${ICE_MACHCOMP} =~ gadi*) then
if (${ICE_COMMDIR} =~ serial*) then
cat >> ${jobfile} << EOFR
./cice >&! \$ICE_RUNLOG_FILE
EOFR
else
cat >> ${jobfile} << EOFR
mpirun -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif

#=======
else if (${ICE_MACHCOMP} =~ hobart* || ${ICE_MACHCOMP} =~ izumi*) then
if (${ICE_COMMDIR} =~ serial*) then
Expand Down
73 changes: 73 additions & 0 deletions configuration/scripts/machines/Macros.gadi_intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#==============================================================================
# Makefile macros for NCI Gadi, intel compiler
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE -DREPRODUCIBLE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise -Wno-unused-variable -Wno-unused-parameter

FIXEDFLAGS := -132
FREEFLAGS := -FR

NCI_INTEL_FLAGS := -r8 -i4 -traceback -w -fpe0 -ftz -convert big_endian -assume byterecl -check noarg_temp_created
NCI_REPRO_FLAGS := -fp-model precise -fp-model source -align all

ifeq ($(ICE_BLDDEBUG), true)
NCI_DEBUG_FLAGS := -g3 -O0 -debug all -check all -no-vec -assume nobuffered_io
FFLAGS := $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_DEBUG_FLAGS)
CPPDEFS := $(CPPDEFS) -DDEBUG=$(DEBUG)
else
NCI_OPTIM_FLAGS := -g3 -O2 -axCORE-AVX2 -debug all -check none -qopt-report=5 -qopt-report-annotate -assume buffered_io
FFLAGS := $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_OPTIM_FLAGS)
endif

SCC := icx
SFC := ifort
MPICC := mpicc
MPIFC := mpifort

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

SLIBS := $(SLIBS)
INCLDIR := $(INCLDIR)

# if spack modules loaded, use them. otherwise use system modules
ifndef SPACK_NETCDF_FORTRAN_ROOT
SLIBS += -L$(NETCDF)/lib -lnetcdf -lnetcdff
INCLDIR += -I$(NETCDF)/include
else
SLIBS += -L$(SPACK_NETCDF_C_ROOT)/lib64 -lnetcdf
SLIBS += -L$(SPACK_NETCDF_FORTRAN_ROOT)/lib -lnetcdff
INCLDIR += -I$(SPACK_NETCDF_C_ROOT)/include
INCLDIR += -I$(SPACK_NETCDF_FORTRAN_ROOT)/include
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

ifeq ($(ICE_IOTYPE), pio1)
LIB_PIO := $(PIO_LIBDIR)
SLIBS += -L$(LIB_PIO) -lpio
endif

ifeq ($(ICE_IOTYPE), pio2)
ifndef SPACK_PARALLELIO_ROOT
SLIBS += -L$(PARALLELIO_ROOT)/lib -lpioc -lpiof
else
SLIBS += -L$(SPACK_PARALLELIO_ROOT)/lib -lpioc -lpiof
INCLDIR += -I $(SPACK_PARALLELIO_ROOT)/include
endif

SLIBS += $(SLIBS) -L$(OMPI_BASE)/lib -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh

endif
56 changes: 56 additions & 0 deletions configuration/scripts/machines/env.gadi_intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/bin/csh -f

set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source /etc/profile.d/modules.csh

module load intel-compiler
module load openmpi

if ($?ICE_IOTYPE) then
if ($ICE_IOTYPE =~ pio*) then
if ($ICE_IOTYPE == "pio1") then
# we don't have pio1 installed anywhere
module load pnetcdf
module load netcdf
module load pio
else
module load parallelio
endif
else
module load netcdf
endif
endif

if ($?ICE_BFBTYPE) then
if ($ICE_BFBTYPE =~ qcchk*) then
# conda/analysis has the required librarys, skip building from cice yaml file
module use /g/data/hh5/public/modules
module load conda/analysis
# conda env create -f ../../configuration/scripts/tests/qctest.yml
# conda activate qctest
endif
endif

endif

setenv ICE_MACHINE_MACHNAME gadi
setenv ICE_MACHINE_MACHINFO "Intel Xeon Scalable"
setenv ICE_MACHINE_ENVNAME intel
# setenv ICE_MACHINE_ENVINFO ${module list}
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /scratch/$PROJECT/$USER/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /g/data/ik11/inputs
setenv ICE_MACHINE_BASELINE /scratch/$PROJECT/$USER/CICE_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub"
setenv ICE_MACHINE_PROJ $PROJECT
setenv ICE_MACHINE_ACCT $USER
setenv ICE_MACHINE_QUEUE "normal"
setenv ICE_MACHINE_TPNODE 48
setenv ICE_MACHINE_BLDTHRDS 4
setenv ICE_MACHINE_QSTAT "qstat"

0 comments on commit d45ea77

Please sign in to comment.