From c5a25314e62cc0de35e1ce4d20519ae0cf59d752 Mon Sep 17 00:00:00 2001 From: Daniel Kennedy Date: Tue, 1 Mar 2022 12:52:51 -0700 Subject: [PATCH] add tethering --- jobscripts/casper.template | 15 +++++ jobscripts/cheyenne.template | 15 +++++ jobscripts/collect.sh | 97 ++++++++++++++++++------------ jobscripts/run_ens.sh | 110 +++++++++++++++++------------------ jobscripts/tether.sh | 65 +++++++++++++++++++++ 5 files changed, 208 insertions(+), 94 deletions(-) create mode 100644 jobscripts/casper.template create mode 100644 jobscripts/cheyenne.template create mode 100644 jobscripts/tether.sh diff --git a/jobscripts/casper.template b/jobscripts/casper.template new file mode 100644 index 0000000..9b8078d --- /dev/null +++ b/jobscripts/casper.template @@ -0,0 +1,15 @@ +#!/bin/bash +#PBS -N jobname +#PBS -q casper +#PBS -l walltime=12:00:00 +#PBS -A P93300641 +#PBS -j oe +#PBS -k eod +#PBS -l select=1:ncpus=1 +#PBS -W depend=afterok:JOB_ID + +#tether submits thiscase (affixing finidat), +#and presubmits nextcase with afterok:thiscase +JOBDIR='/glade/u/home/djk2120/clm5ppe/jobscripts/' +bash $JOBDIR"tether.sh" prevcase scratch thiscase joblist template + diff --git a/jobscripts/cheyenne.template b/jobscripts/cheyenne.template new file mode 100644 index 0000000..984189e --- /dev/null +++ b/jobscripts/cheyenne.template @@ -0,0 +1,15 @@ +#!/bin/bash +#PBS -N jobname +#PBS -q premium +#PBS -l walltime=12:00:00 +#PBS -A P93300641 +#PBS -j oe +#PBS -k eod +#PBS -l select=1:ncpus=1 +#PBS -W depend=afterok:JOB_ID + +#tether submits thiscase (affixing finidat), +#and presubmits nextcase with afterok:thiscase +JOBDIR='/glade/u/home/djk2120/clm5ppe/jobscripts/' +bash $JOBDIR"tether.sh" prevcase scratch thiscase joblist template + diff --git a/jobscripts/collect.sh b/jobscripts/collect.sh index 6d8847d..c2966f7 100644 --- a/jobscripts/collect.sh +++ b/jobscripts/collect.sh @@ -3,62 +3,81 @@ #takes three arguments on the command-line # e.g. bash collect.sh PPEn11/configs/CTL2010_s1.config h0 1 # 1 - config file -# 2 - history tape, e.g. h0, h1, ... , r -# 3 - whether to move files, +# 2 - whether to move files, # 0=don't move, just verify existence # 1=move, but don't overwrite # 2=move with overwrite -if [ $# -lt 3 ] +if [ $# -lt 2 ] then - echo "ERROR: please specify format file, htape, and whether to move files" - echo " ex: ./collect.sh spinAD.config h1 0" + echo "ERROR: please specify format file and whether to move files" + echo " ex: ./collect.sh spinAD.config 0" exit 1 fi + + #set up environment variables source $1 echo $1 jobdir=$(pwd)"/" -tape=$2 -moveFiles=$3 -fails=$(basename $1 .config)"_"$tape"_fails.txt" -> $fails +moveFiles=$2 -if [ $tape == 'r' ]; then - NEW_DIR=$RESTARTS -else - NEW_DIR=$HIST_DIR -fi +tapes=('r' 'h0' 'h1' 'h2' 'h3' 'h4' 'h5' 'h7') +for tape in ${tapes[@]};do -#loop through paramlist -while read p; do - thiscase=$casePrefix"_"$p - file=$SCRATCH$thiscase"/run/*.clm2."$tape".*nc" - if [ ! -f $file ]; then - echo $p" MAY HAVE FAILED" - echo $p>>$fails + fails=$(basename $1 .config)"_"$tape"_fails.txt" + :> $fails + echo "checking for "$tape" files" + + if [ $tape == 'r' ]; then + NEW_DIR=$RESTARTS else - echo $p - newfile=$NEW_DIR$(basename $file) - if [ $moveFiles -gt 0 ]; then - already=0 - if [ -f $newfile ]; then - already=1 - if [ $moveFiles -eq 1 ]; then - echo $p" restart already exists, will not copy" - else - echo $p" restart already exists, file overwritten" - fi - fi - if [ $already == 0 ] || [ $2 == 2 ]; then - mv $file $newfile - fi - fi + NEW_DIR=$HIST_DIR fi -done < $paramList + + + if [ $moveFiles -gt 0 ]; then + mkdir -p $NEW_DIR + echo "collecting files in "$NEW_DIR + fi + + #loop through paramlist + while read p; do + thiscase=${cases[-1]}"_"$p + files=$SCRATCH$thiscase"/run/*.clm2."$tape".*nc" + + if compgen -G $files > /dev/null; then + i=0 + for file in $files; do + i=$((i+1)) + if [ $i == 1 ]; then + newfile=$NEW_DIR$(basename $file) + if [ $moveFiles -gt 0 ]; then + already=0 + if [ -f $newfile ]; then + already=1 + if [ $moveFiles -eq 1 ]; then + echo $p" restart already exists, will not copy" + else + echo $p" restart already exists, file overwritten" + fi + fi + if [ $already == 0 ] || [ $3 == 2 ]; then + mv $file $newfile + fi + fi + fi + done + else + echo $p" MAY HAVE FAILED" + echo $p>>$fails + fi + + done < $paramList + nf=$(wc -l < $fails) if [ $nf == 0 ]; then @@ -68,3 +87,5 @@ else echo "ERROR: one or more simulations may have crashed!" echo " see "$fails fi + +done diff --git a/jobscripts/run_ens.sh b/jobscripts/run_ens.sh index b62e9d7..cef2097 100644 --- a/jobscripts/run_ens.sh +++ b/jobscripts/run_ens.sh @@ -7,64 +7,62 @@ fi #set up environment variables source $1 +joblist='tethered.txt' -#loop through paramlist while read p; do - #create the new case name - repcase=$casePrefix"_"$p + for i in "${!cases[@]}"; do + case="${cases[i]}" + exeroot="${exeroots[i]}" + basecase=$SCRIPTS$ensemble"/basecases/"$case + thiscase=$SCRIPTS$ensemble"/"$case"/"$case"_"$p + + #clone case + cd $SCRIPTS + ./create_clone --case $thiscase --clone $basecase + + #setup, point to executable + cd $thiscase + ./case.setup + ./xmlchange BUILD_COMPLETE=TRUE + ./xmlchange EXEROOT=$exeroot + ./xmlchange DOUT_S=FALSE + + #comment out previous paramfile from user_nl_clm + :> user_nl_clm.tmp + while read line; do + if [[ $line != *"paramfile"* ]]; then + echo $line>>user_nl_clm.tmp + else + echo '!'$line>>user_nl_clm.tmp + fi + done> user_nl_clm + + # cat nlmods if needed + if [ "$nlmodsFlag" = true ] + then + nlmods=$NLMODS$p".txt" + cat $nlmods >> user_nl_clm + fi - #clone case - echo "--------------------------------------------" - echo " creating "$repcase - echo "--------------------------------------------" - cd $SCRIPTS_DIR - ./create_clone --case $caseDir$repcase --clone $basecase - cd $caseDir$repcase + #set up job tethering + if (( i == 0 )); then + firstcase=$thiscase + :> $joblist #empty file + else + cd $firstcase + echo $thiscase >> $joblist + fi + done - #setup and point to executable - ./case.setup - if [ "$exerootFlag" = true ] - then - ./xmlchange BUILD_COMPLETE=TRUE - ./xmlchange EXEROOT=$exeroot - else - echo "--------------------------------------------" - echo " building "$repcase - echo "--------------------------------------------" - ./case.build - fi + #submit job, with next jobs tethered via PBS afterok + cd $PPE + prevcase="none" + bash tether.sh $prevcase $SCRATCH $firstcase $joblist $template - #temp, adjust project - ./xmlchange PROJECT="P08010000" - - - # copy user_nl_clm and specify paramfile - cp $nlbase user_nl_clm - pfile=$PARAMS_DIR$p".nc" - pfilestr="paramfile = '"$pfile"'" - echo -e "\n"$pfilestr >> user_nl_clm - - # specify finidat if needed - if [ "$finidatFlag" = true ] - then - rfile=$RESTARTS$prevCase"_"$p*".nc" - rfile=$(echo $rfile) #force wildcard expansion - rfilestr="finidat ='"$rfile"'" - echo $rfilestr >> user_nl_clm - fi - - # cat nlmods if needed - if [ "$nlmodsFlag" = true ] - then - nlmods=$NLMODS_DIR$p".txt" - cat $nlmods >> user_nl_clm - fi - - echo "--------------------------------------------" - echo " submitting "$repcase - echo "--------------------------------------------" - ./case.submit - - - -done <$paramList +done<$paramList diff --git a/jobscripts/tether.sh b/jobscripts/tether.sh new file mode 100644 index 0000000..fdb0318 --- /dev/null +++ b/jobscripts/tether.sh @@ -0,0 +1,65 @@ +prevcase=$1 +scratch=$2 +thiscase=$3 +joblist=$4 +template=$5 + +cd $thiscase + + + +#handle finidat +if [ $prevcase != 'none' ]; then + #comment out any finidat from user_nl_clm + :> user_nl_clm.tmp + while read line; do + if [[ $line != *"finidat"* ]]; then + echo $line>>user_nl_clm.tmp + else + echo '!'$line>>user_nl_clm.tmp + fi + done>user_nl_clm +fi + +#submit case, capturing JOB_ID +./case.submit > submit_out +JOB_ID=$(grep "Submitted job id is" submit_out | tail -1 | cut -f 5 -d' ') + +#presubmit next case, if any are left +njobs=$(wc -l < $joblist) +if (( $njobs > 0 ));then + #read next job from joblist + nextcase=$(head -n 1 $joblist) + echo "nextcase="$nextcase + + #cp joblist to nextcase, removing nextcase + tail -n +2 $joblist > file.tmp + mv file.tmp $nextcase'/'$joblist + + #prepare the next job submission + casename=${nextcase##*/} + nextjob=$nextcase"/"$casename".presubmission" + cp $template $nextjob + + sed -i -e 's:JOB_ID:'$JOB_ID':g' $nextjob + sed -i -e 's:jobname:pre_'$casename':g' $nextjob + sed -i -e 's:prevcase:'$thiscase':g' $nextjob + sed -i -e 's:scratch:'$scratch':g' $nextjob + sed -i -e 's:thiscase:'$nextcase':g' $nextjob + sed -i -e 's:joblist:'$joblist':g' $nextjob + sed -i -e 's:template:'$template':g' $nextjob + + #presubmit the next job + qsub $nextjob + + +fi