add tethering

jobscripts/casper.template

@@ -0,0 +1,15 @@
+#!/bin/bash
+#PBS -N jobname
+#PBS -q casper
+#PBS -l walltime=12:00:00
+#PBS -A P93300641
+#PBS -j oe
+#PBS -k eod
+#PBS -l select=1:ncpus=1
+#PBS -W depend=afterok:JOB_ID
+
+#tether submits thiscase (affixing finidat), 
+#and presubmits nextcase with afterok:thiscase
+JOBDIR='/glade/u/home/djk2120/clm5ppe/jobscripts/'
+bash $JOBDIR"tether.sh" prevcase scratch thiscase joblist template
+

jobscripts/cheyenne.template

@@ -0,0 +1,15 @@
+#!/bin/bash
+#PBS -N jobname
+#PBS -q premium
+#PBS -l walltime=12:00:00
+#PBS -A P93300641
+#PBS -j oe
+#PBS -k eod
+#PBS -l select=1:ncpus=1
+#PBS -W depend=afterok:JOB_ID
+
+#tether submits thiscase (affixing finidat), 
+#and presubmits nextcase with afterok:thiscase
+JOBDIR='/glade/u/home/djk2120/clm5ppe/jobscripts/'
+bash $JOBDIR"tether.sh" prevcase scratch thiscase joblist template
+

jobscripts/collect.sh

@@ -3,62 +3,81 @@
 #takes three arguments on the command-line
 # e.g. bash collect.sh PPEn11/configs/CTL2010_s1.config h0 1
 # 1 - config file
-# 2 - history tape, e.g. h0, h1, ... , r
-# 3 - whether to move files, 
+# 2 - whether to move files, 
 #        0=don't move, just verify existence
 #        1=move, but don't overwrite
 #        2=move with overwrite
 
 
-if [ $# -lt 3 ]
+if [ $# -lt 2 ]
 then
-    echo "ERROR: please specify format file, htape, and whether to move files"
-    echo "   ex: ./collect.sh spinAD.config h1 0"
+    echo "ERROR: please specify format file and whether to move files"
+    echo "   ex: ./collect.sh spinAD.config 0"
     exit 1
 fi
 
+
+
 #set up environment variables
 source $1
 echo $1
 jobdir=$(pwd)"/"
-tape=$2
-moveFiles=$3
-fails=$(basename $1 .config)"_"$tape"_fails.txt"
-> $fails
+moveFiles=$2
 
 
-if [ $tape == 'r' ]; then
-    NEW_DIR=$RESTARTS
-else
-    NEW_DIR=$HIST_DIR
-fi
+tapes=('r' 'h0' 'h1' 'h2' 'h3' 'h4' 'h5' 'h7') 
+for tape in ${tapes[@]};do
 
-#loop through paramlist
-while read p; do
-    thiscase=$casePrefix"_"$p
-    file=$SCRATCH$thiscase"/run/*.clm2."$tape".*nc"
-    if [ ! -f $file ]; then
-	echo $p" MAY HAVE FAILED"
-	echo $p>>$fails
+    fails=$(basename $1 .config)"_"$tape"_fails.txt"
+    :> $fails
+    echo "checking for "$tape" files"
+
+    if [ $tape == 'r' ]; then
+	NEW_DIR=$RESTARTS
     else
-	echo $p
-	newfile=$NEW_DIR$(basename $file)
-	if [ $moveFiles -gt 0 ]; then 
-	    already=0
-	    if [ -f $newfile ]; then
-		already=1
-		if [ $moveFiles -eq 1 ]; then
-		    echo $p" restart already exists, will not copy"
-		else
-		    echo $p" restart already exists, file overwritten"
-		fi
-	    fi
-	    if [ $already == 0 ] || [ $2 == 2 ]; then
-		mv $file $newfile
-	    fi
-	fi 
+	NEW_DIR=$HIST_DIR
     fi
-done < $paramList
+
+
+    if [ $moveFiles -gt 0 ]; then
+	mkdir -p $NEW_DIR
+	echo "collecting files in "$NEW_DIR
+    fi
+
+    #loop through paramlist
+    while read p; do
+	thiscase=${cases[-1]}"_"$p
+	files=$SCRATCH$thiscase"/run/*.clm2."$tape".*nc"
+
+	if compgen -G $files > /dev/null; then
+	    i=0
+	    for file in $files; do
+		i=$((i+1))
+		if [ $i == 1 ]; then
+		    newfile=$NEW_DIR$(basename $file)	
+		    if [ $moveFiles -gt 0 ]; then 
+			already=0
+			if [ -f $newfile ]; then
+			    already=1
+			    if [ $moveFiles -eq 1 ]; then
+				echo $p" restart already exists, will not copy"
+			    else
+				echo $p" restart already exists, file overwritten"
+			    fi
+			fi
+			if [ $already == 0 ] || [ $3 == 2 ]; then
+			    mv $file $newfile
+			fi
+		    fi 
+		fi
+	    done
+	else
+	    echo $p" MAY HAVE FAILED"
+	    echo $p>>$fails
+	fi
+
+    done < $paramList
+
 
 nf=$(wc -l < $fails)
 if [ $nf == 0 ]; then
@@ -68,3 +87,5 @@ else
     echo "ERROR: one or more simulations may have crashed!"
     echo "       see "$fails
 fi
+
+done

jobscripts/run_ens.sh

@@ -7,64 +7,62 @@ fi
 
 #set up environment variables
 source $1
+joblist='tethered.txt'
 
-#loop through paramlist
 while read p; do
-  #create the new case name
-  repcase=$casePrefix"_"$p
+    for i in "${!cases[@]}"; do
+	case="${cases[i]}"
+	exeroot="${exeroots[i]}"
+	basecase=$SCRIPTS$ensemble"/basecases/"$case
+	thiscase=$SCRIPTS$ensemble"/"$case"/"$case"_"$p
+
+	#clone case
+	cd $SCRIPTS
+	./create_clone --case $thiscase --clone $basecase
+
+	#setup, point to executable
+	cd $thiscase
+	./case.setup
+	./xmlchange BUILD_COMPLETE=TRUE
+	./xmlchange EXEROOT=$exeroot
+	./xmlchange DOUT_S=FALSE
+
+	#comment out previous paramfile from user_nl_clm
+	:> user_nl_clm.tmp
+	while read line; do
+	    if [[ $line != *"paramfile"* ]]; then
+		echo $line>>user_nl_clm.tmp
+	    else
+		echo '!'$line>>user_nl_clm.tmp
+	    fi
+	done<user_nl_clm
+	mv user_nl_clm.tmp user_nl_clm
+
+	#append correct paramfile
+	pfile=$PARAMS$p".nc"
+	pfilestr="paramfile = '"$pfile"'"
+	echo -e "\n"$pfilestr >> user_nl_clm
+
+	# cat nlmods if needed
+	if [ "$nlmodsFlag" = true ]
+	then
+	    nlmods=$NLMODS$p".txt"
+	    cat $nlmods >> user_nl_clm
+	fi
 
-  #clone case
-  echo "--------------------------------------------"
-  echo "   creating "$repcase
-  echo "--------------------------------------------" 
-  cd $SCRIPTS_DIR
-  ./create_clone --case $caseDir$repcase --clone $basecase
-  cd $caseDir$repcase
+	#set up job tethering
+	if (( i == 0 )); then
+	    firstcase=$thiscase
+	    :> $joblist  #empty file
+	else
+	    cd $firstcase
+	    echo $thiscase >> $joblist
+	fi
+    done
 
-  #setup and point to executable
-  ./case.setup
-  if [ "$exerootFlag" = true ]
-  then
-      ./xmlchange BUILD_COMPLETE=TRUE
-      ./xmlchange EXEROOT=$exeroot
-  else
-      echo "--------------------------------------------"
-      echo "   building "$repcase
-      echo "--------------------------------------------" 
-      ./case.build
-  fi
+    #submit job, with next jobs tethered via PBS afterok
+    cd $PPE
+    prevcase="none"
+    bash tether.sh $prevcase $SCRATCH $firstcase $joblist $template
 
-  #temp, adjust project
-  ./xmlchange PROJECT="P08010000"
-
-
-  # copy user_nl_clm and specify paramfile
-  cp $nlbase user_nl_clm
-  pfile=$PARAMS_DIR$p".nc"
-  pfilestr="paramfile = '"$pfile"'"
-  echo -e "\n"$pfilestr >> user_nl_clm
-
-  # specify finidat if needed
-  if [ "$finidatFlag" = true ]
-  then
-      rfile=$RESTARTS$prevCase"_"$p*".nc"
-      rfile=$(echo $rfile) #force wildcard expansion
-      rfilestr="finidat ='"$rfile"'"
-      echo $rfilestr >> user_nl_clm
-  fi
-
-  # cat nlmods if needed
-  if [ "$nlmodsFlag" = true ]
-  then
-      nlmods=$NLMODS_DIR$p".txt"
-      cat $nlmods >> user_nl_clm
-  fi
-
-  echo "--------------------------------------------"
-  echo "   submitting "$repcase
-  echo "--------------------------------------------" 
-  ./case.submit
-
-
-
-done <$paramList
+done<$paramList

jobscripts/tether.sh

@@ -0,0 +1,65 @@
+prevcase=$1
+scratch=$2
+thiscase=$3
+joblist=$4
+template=$5
+
+cd $thiscase
+
+
+
+#handle finidat
+if [ $prevcase != 'none' ]; then
+    #comment out any finidat from user_nl_clm
+    :> user_nl_clm.tmp
+    while read line; do
+	if [[ $line != *"finidat"* ]]; then
+	    echo $line>>user_nl_clm.tmp
+	else
+	    echo '!'$line>>user_nl_clm.tmp
+	fi
+	done<user_nl_clm
+    mv user_nl_clm.tmp user_nl_clm
+
+    #find the appropirate restart file
+    casename=${prevcase##*/}
+    restart=$scratch$casename'/run/'$casename".clm2.r."*".nc"
+    restart=$(echo $restart) #expands wildcard
+
+    #append to user_nl_clm
+    echo -e 'finidat="'$restart'"'>>user_nl_clm
+fi
+
+#submit case, capturing JOB_ID
+./case.submit > submit_out
+JOB_ID=$(grep "Submitted job id is" submit_out | tail -1 | cut -f 5 -d' ')
+
+#presubmit next case, if any are left
+njobs=$(wc -l < $joblist)
+if (( $njobs > 0 ));then 
+    #read next job from joblist
+    nextcase=$(head -n 1 $joblist)
+    echo "nextcase="$nextcase
+
+    #cp joblist to nextcase, removing nextcase
+    tail -n +2 $joblist > file.tmp 
+    mv file.tmp $nextcase'/'$joblist
+
+    #prepare the next job submission
+    casename=${nextcase##*/}
+    nextjob=$nextcase"/"$casename".presubmission"
+    cp $template $nextjob
+
+    sed -i -e 's:JOB_ID:'$JOB_ID':g' $nextjob
+    sed -i -e 's:jobname:pre_'$casename':g' $nextjob
+    sed -i -e 's:prevcase:'$thiscase':g' $nextjob
+    sed -i -e 's:scratch:'$scratch':g' $nextjob
+    sed -i -e 's:thiscase:'$nextcase':g' $nextjob
+    sed -i -e 's:joblist:'$joblist':g' $nextjob
+    sed -i -e 's:template:'$template':g' $nextjob
+
+    #presubmit the next job
+    qsub $nextjob
+
+
+fi