--- MITgcm/tools/do_tst_2+2 2011/01/15 19:16:32 1.8 +++ MITgcm/tools/do_tst_2+2 2015/11/18 22:18:41 1.21 @@ -1,6 +1,6 @@ #! /usr/bin/env bash -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/do_tst_2+2,v 1.8 2011/01/15 19:16:32 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/do_tst_2+2,v 1.21 2015/11/18 22:18:41 jmc Exp $ # $Name: $ usage() @@ -14,11 +14,14 @@ echo " -skd LIST_EXP : skip experiments in LIST_EXP" echo " -exe COMMAND : use COMMAND to run the tests" echo " -mpi : run the tests using MPI" + echo " -mf STRING :(MPI) file with list of possible machines to run on" echo " -o STRING : used to build output directory name" echo " (DEF=\"hostname\")" echo " -a,-A STRING : email address to send output to" echo " (DEF=\"\" no email is sent)" echo " (-A: + save each log file)" + echo " -send STRING : sending command (instead of using mpack)" + echo " -sd STRING : location to save output tar file to send (DEF='$SAVDIR')" echo " -clean : clean output & reset" exit } @@ -27,13 +30,18 @@ TESTDIRS= SKIPDIRS= SCRIPT='../tools/tst_2+2' +LOGFILE='tst_2+2_out.log' scrArg='All' +bldDir='build' COMMAND= mpi=0 -OUTDIR=`hostname | sed 's/\..*$//'` +MPI_MFILE= +OUTDIR=`hostname -s | tr '[:upper:]' '[:lower:]'` SAVELOG=0 ADDRESS= MPACK="../tools/mpack-1.6/mpack" +SENDCMD= +SAVDIR='.' here=`pwd` yy= for xx @@ -49,8 +57,11 @@ -skd ) yy=SKIPDIRS ;; -exe ) yy=COMMAND ;; -mpi ) mpi=1 ;; + -mf ) yy=MPI_MFILE ;; -a ) yy=ADDRESS ;; -A ) yy=ADDRESS ; SAVELOG=1 ;; + -send ) yy=SENDCMD ;; + -sd ) yy=SAVDIR ;; -o ) yy=OUTDIR ;; *) echo "Error: unrecognized option: "$xx ; usage ; exit ;; esac @@ -75,15 +86,19 @@ done if test -x $SCRIPT ; then - echo "run script '$SCRIPT' for experiment in:" + if test $CLEAN = 1 ; then + echo "clean output from script '$SCRIPT' for experiment in:" + else + echo "run script '$SCRIPT' for experiment in:" + fi echo " $LIST" + echo "" yy=`echo $SCRIPT | grep -c '^\/'` if test $yy = 0 ; then SCRIPT="../../$SCRIPT" ; fi else echo "ERROR: script '$SCRIPT' not found or not executable" exit fi -echo "" #------------------------ if test $CLEAN = 1 ; then @@ -95,9 +110,9 @@ if test -r $xx/$yy/data.tst ; then echo "clean dir:" $xx/$yy cd $xx/$yy - echo ' ' >> outp.tst_2+2.log - echo $SCRIPT 4 >> outp.tst_2+2.log - $SCRIPT 4 >> outp.tst_2+2.log 2>&1 + echo ' ' >> $LOGFILE + echo $SCRIPT 4 >> $LOGFILE + $SCRIPT 4 >> $LOGFILE 2>&1 cd $here fi done @@ -106,7 +121,19 @@ fi #------------------------ # Create a uniquely named directory to store results -DATE=`date +%Y%m%d` +CMDLINE=$0 +for xx in "$@" ; do nw=`echo $xx | wc -w` + if test $nw = '1' ; then CMDLINE="$CMDLINE $xx" + else CMDLINE="$CMDLINE '$xx'" ; fi +done +DATE='' +if test -f tr_out.txt ; then +# try to use the date corresponding to 'Start time:' in file "tr_out.txt" : + sTime=`grep '^Start time: ' tr_out.txt 2>/dev/null | sed 's/Start time: //'` + if test "x$sTime" != x ; then DATE=`date -d "$sTime" "+%Y%m%d" 2>/dev/null` ; fi +fi +# otherwise, use current date: +if test "x$DATE" = x ; then DATE=`date +%Y%m%d`; fi BASE="rs_"$OUTDIR"_"$DATE"_" xx=0 DRESULTS="$BASE$xx" @@ -125,9 +152,10 @@ printf "Start time: " > $SUMMARY date >> $SUMMARY +echo 'run:' $CMDLINE >> $SUMMARY if test -f tr_out.txt ; then echo ' using output from:' >> $SUMMARY - sed -n '2,/OPTFILE/ p' tr_out.txt >> $SUMMARY + sed -n '2,/^ OPTFILE=/ p' tr_out.txt >> $SUMMARY echo >> $SUMMARY fi echo 'test 2+2=4 summary :' >> $SUMMARY @@ -136,39 +164,53 @@ echo ' 1 2 3' >> $SUMMARY #-- For MPI test: +LOC_MFILE='tr_mpi_mfile' RUNOUTP="output.txt" if [ $mpi -ge 1 ] ; then SCRIPT="$SCRIPT -mpi" RUNOUTP="STDOUT.0000" + if test "x$COMMAND" = x ; then COMMAND="mpirun -v -np TR_NPROC ./mitgcmuv" ; fi fi for xx in $LIST do - echo ============================================================================== - if [ $mpi -ge 1 ] ; then - NP_MPI=2 - mpi_size=$xx/build/SIZE.h.mpi + echo '------------------------------------------------------------------------------' + if [ $mpi -le 0 ] ; then + rCommand=$COMMAND + else + LOC_NPROC=2 + mpi_size=$xx/$bldDir/SIZE.h.mpi if test -f $mpi_size ; then px=`grep '^ & *nPx *=' $mpi_size | sed 's/^ & *nPx *= *//' | sed 's/, *$//'` py=`grep '^ & *nPy *=' $mpi_size | sed 's/^ & *nPy *= *//' | sed 's/, *$//'` pp=`expr $px \* $py` > /dev/null 2>&1 ; out=$? - if test "x$out" = x0 ; then NP_MPI=$pp ; fi + if test "x$out" = x0 ; then LOC_NPROC=$pp ; fi + fi + rCommand=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"` + if test "x$MPI_MFILE" != x ; then + #- create new MPI machine-file with the right number of Procs + rm -f $LOC_MFILE + cat $MPI_MFILE | sort | uniq | head -$LOC_NPROC > $LOC_MFILE + nl=`wc -l $LOC_MFILE | awk '{print $1}'` + if [ $nl -lt $LOC_NPROC ] ; then + rm -f $LOC_MFILE + cat $MPI_MFILE | head -$LOC_NPROC > $LOC_MFILE + fi + rCommand=`echo $rCommand | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"` fi - rCommand=`echo $COMMAND | sed "s/-np * XX /-np $NP_MPI /"` - else - rCommand=$COMMAND fi + listD='run '`(cd $xx/results ; ls output.*.txt 2> /dev/null | sed 's/^output\./tr_run./g' | sed 's/\.txt$//g' )` for yy in $listD do if test -f $xx/$yy/$RUNOUTP ; then - if test "x$OPTFILE" = x -a -f $xx/build/Makefile ; then - comm=`grep '^# OPTFILE=' $xx/build/Makefile 2>/dev/null | head -1 | sed 's/^# //'` - echo "from '$xx/build/Makefile', extract:" > $DRESULTS/genmake_state - sed -n '/^# executed by:/,+1 p' $xx/build/Makefile >> $DRESULTS/genmake_state + if test "x$OPTFILE" = x -a -f $xx/$bldDir/Makefile ; then + comm=`grep '^# OPTFILE=' $xx/$bldDir/Makefile 2>/dev/null | head -1 | sed 's/^# //'` + echo "from '$xx/$bldDir/Makefile', extract:" > $DRESULTS/genmake_state + sed -n '/^# executed by:/,+1 p' $xx/$bldDir/Makefile >> $DRESULTS/genmake_state echo " $comm" >> $DRESULTS/genmake_state eval $comm - gmkLog=$xx/build/genmake.log + gmkLog=$xx/$bldDir/genmake.log grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1 out=$? if test "x$out" = x0 ; then @@ -184,15 +226,15 @@ else nam=$xx`echo $yy | sed 's/tr_run//'` ; fi echo -n "Entering $xx/$yy :" cd $xx/$yy - pwd > outp.tst_2+2.log + pwd > $LOGFILE if test "x$COMMAND" = x ; then - echo $SCRIPT $scrArg >> outp.tst_2+2.log ; echo ' ' >> outp.tst_2+2.log - $SCRIPT $scrArg >> outp.tst_2+2.log 2>&1 + echo $SCRIPT $scrArg >> $LOGFILE ; echo ' ' >> $LOGFILE + $SCRIPT $scrArg >> $LOGFILE 2>&1 out=$? else - echo "$SCRIPT $scrArg -command \"$rCommand\"" >> outp.tst_2+2.log - echo ' ' >> outp.tst_2+2.log - $SCRIPT $scrArg -command "$rCommand" >> outp.tst_2+2.log 2>&1 + echo "$SCRIPT $scrArg -command \"$rCommand\"" >> $LOGFILE + echo ' ' >> $LOGFILE + $SCRIPT $scrArg -command "$rCommand" >> $LOGFILE 2>&1 out=$? fi case $out in @@ -207,45 +249,52 @@ if test $out != '0' ; then echo " test 2+2=4 FAIL (exit $out)" echo " " - tail -5 outp.tst_2+2.log + tail -5 $LOGFILE echo " " - cp -p outp.tst_2+2.log ../../$DRESULTS/$nam.log + cp -p $LOGFILE ../../$DRESULTS/$nam.log elif test $SAVELOG = 1 ; then - cp -p outp.tst_2+2.log ../../$DRESULTS/$nam.log + cp -p $LOGFILE ../../$DRESULTS/$nam.log fi cd $here fi done + if test "x$mpi" != x0 -a "x$MPI_MFILE" != x ; then rm -f $LOC_MFILE ; fi done printf "End time: " >> $SUMMARY date >> $SUMMARY -# If address was supplied and mpack exist, then send email using mpack. +# If address was supplied, then send email using Send-Command or mpack. if test "x$ADDRESS" != xNONE -a "x$ADDRESS" != x ; then - if test -x $MPACK ; then - tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \ - && gzip $DRESULTS".tar" \ - && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESS + if test "x$SENDCMD" = x ; then + # if Send-Command is empty and mpack exist, then send email using mpack. + if test -x $MPACK ; then SENDCMD=$MPACK + else + echo + echo "Warning: $MPACK is not executable => no email was sent" + fi + fi + if test "x$SENDCMD" != x ; then + #echo " run: $SENDCMD -s MITgcm-test -m 3555000 ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESS" + tar -cf ${SAVDIR}/${DRESULTS}".tar" $DRESULTS > /dev/null 2>&1 \ + && gzip ${SAVDIR}/${DRESULTS}".tar" \ + && $SENDCMD -s MITgcm-test -m 3555000 ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESS out=$? if test "x$out" != x0 ; then echo echo "Warning: The tar, gzip, & mpack step failed. Please send email" echo " to for help. You may copy the " echo " summary of results from the directory \"$DRESULTS\"." - echo else echo echo "An email containing results was sent to the following address:" echo " \"$ADDRESS\"" - echo fi - test -f $DRESULTS".tar" && rm -f $DRESULTS".tar" - test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz" - else - echo "Warning: $MPACK is not executable => no email was sent" - echo + test -f ${SAVDIR}/${DRESULTS}".tar" && rm -f ${SAVDIR}/${DRESULTS}".tar" + test -f ${SAVDIR}/${DRESULTS}".tar.gz" && rm -f ${SAVDIR}/${DRESULTS}".tar.gz" fi fi +echo '' ; echo '===== Summary (tst_2+2_out.txt): ======' +cat $SUMMARY # save output and remove output Dir (if no address was provided) # note: to keep the output Dir without sending email, use option "-a NONE" @@ -254,4 +303,3 @@ if test "x$ADDRESS" = x ; then rm -rf $DRESULTS fi -