--- MITgcm/tools/do_tst_2+2	2010/03/25 23:09:29	1.6
+++ MITgcm/tools/do_tst_2+2	2013/09/22 23:37:12	1.18
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/do_tst_2+2,v 1.6 2010/03/25 23:09:29 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/do_tst_2+2,v 1.18 2013/09/22 23:37:12 jmc Exp $
 # $Name:  $
 
 usage()
@@ -14,11 +14,14 @@
         echo "  -skd LIST_EXP : skip experiments in LIST_EXP"
         echo "  -exe  COMMAND : use COMMAND to run the tests"
         echo "  -mpi          : run the tests using MPI"
+        echo "  -mf    STRING :(MPI) file with list of possible machines to run on"
         echo "  -o     STRING : used to build output directory name"
         echo "                      (DEF=\"hostname\")"
         echo "  -a,-A  STRING : email address to send output to"
         echo "                      (DEF=\"\" no email is sent)"
         echo "                      (-A: + save each log file)"
+        echo "  -send  STRING : sending command (instead of using mpack)"
+        echo "  -sd    STRING : location to save output tar file to send (DEF='$SAVDIR')"
         echo "  -clean        : clean output & reset"
         exit
 }
@@ -27,13 +30,17 @@
 TESTDIRS=
 SKIPDIRS=
 SCRIPT='../tools/tst_2+2'
+LOGFILE='tst_2+2_out.log'
 scrArg='All'
 COMMAND=
 mpi=0
+MPI_MFILE=
 OUTDIR=`hostname | sed 's/\..*$//'`
 SAVELOG=0
 ADDRESS=
 MPACK="../tools/mpack-1.6/mpack"
+SENDCMD=
+SAVDIR='.'
 here=`pwd`
 yy=
 for xx
@@ -49,8 +56,11 @@
       -skd  ) yy=SKIPDIRS ;;
       -exe  ) yy=COMMAND ;;
       -mpi  ) mpi=1 ;;
+      -mf   ) yy=MPI_MFILE ;;
       -a    ) yy=ADDRESS ;;
       -A    ) yy=ADDRESS ; SAVELOG=1 ;;
+      -send ) yy=SENDCMD ;;
+      -sd   ) yy=SAVDIR ;;
       -o    ) yy=OUTDIR ;;
         *) echo "Error: unrecognized option: "$xx ; usage ; exit ;;
       esac
@@ -75,15 +85,19 @@
 done
 
 if test -x $SCRIPT ; then
- echo "run script '$SCRIPT' for experiment in:"
+ if test $CLEAN = 1 ; then
+   echo "clean output from script '$SCRIPT' for experiment in:"
+ else
+   echo "run script '$SCRIPT' for experiment in:"
+ fi
  echo " $LIST"
+ echo ""
  yy=`echo $SCRIPT | grep -c '^\/'`
  if test $yy = 0 ; then SCRIPT="../../$SCRIPT" ; fi
 else
  echo "ERROR: script '$SCRIPT' not found or not executable"
  exit
 fi
-echo ""
 #------------------------
 
 if test $CLEAN = 1 ; then
@@ -95,9 +109,9 @@
     if test -r $xx/$yy/data.tst ; then
       echo "clean dir:" $xx/$yy
       cd $xx/$yy
-      echo ' ' >> outp.tst_2+2.log
-      echo $SCRIPT 4 >> outp.tst_2+2.log
-      $SCRIPT 4 >> outp.tst_2+2.log 2>&1
+      echo ' ' >> $LOGFILE
+      echo $SCRIPT 4 >> $LOGFILE
+      $SCRIPT 4 >> $LOGFILE 2>&1
       cd $here
     fi
   done
@@ -106,7 +120,20 @@
 fi
 #------------------------
 #  Create a uniquely named directory to store results
-DATE=`date +%Y%m%d`
+CMDLINE=$0
+for xx in "$@" ; do nw=`echo $xx | wc -w`
+    if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
+                        else CMDLINE="$CMDLINE '$xx'" ; fi
+done
+#for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
+DATE=''
+if test -f tr_out.txt ; then
+#  try to use the date corresponding to 'Start time:' in file "tr_out.txt" :
+  sTime=`grep '^Start time:  ' tr_out.txt 2>/dev/null | sed 's/Start time:  //'`
+  if test "x$sTime" != x ; then DATE=`date -d "$sTime" "+%Y%m%d" 2>/dev/null` ; fi
+fi
+#  otherwise, use current date:
+if test "x$DATE" = x ; then DATE=`date +%Y%m%d`; fi
 BASE="rs_"$OUTDIR"_"$DATE"_"
 xx=0
 DRESULTS="$BASE$xx"
@@ -125,9 +152,10 @@
 
 printf "Start time:  " >  $SUMMARY
 date  >> $SUMMARY
+echo 'run:' $CMDLINE >> $SUMMARY
 if test -f tr_out.txt ; then
  echo ' using output from:' >> $SUMMARY
- sed -n '2,/OPTFILE/ p' tr_out.txt >> $SUMMARY
+ sed -n '2,/^  OPTFILE=/ p' tr_out.txt >> $SUMMARY
  echo >> $SUMMARY
 fi
 echo  'test 2+2=4 summary :' >> $SUMMARY
@@ -136,15 +164,42 @@
 echo '  1 2 3'  >> $SUMMARY
 
 #-- For MPI test:
+LOC_MFILE='tr_mpi_mfile'
 RUNOUTP="output.txt"
 if [ $mpi -ge 1 ] ; then
   SCRIPT="$SCRIPT -mpi"
   RUNOUTP="STDOUT.0000"
+  if test "x$COMMAND" = x ; then COMMAND="mpirun -v -np TR_NPROC ./mitgcmuv" ; fi
 fi
 
 for xx in $LIST
 do
-  echo ==============================================================================
+  echo '------------------------------------------------------------------------------'
+  if [ $mpi -le 0 ] ; then
+    rCommand=$COMMAND
+  else
+    LOC_NPROC=2
+    mpi_size=$xx/build/SIZE.h.mpi
+    if test -f $mpi_size ; then
+      px=`grep '^     & *nPx *=' $mpi_size | sed 's/^     & *nPx *= *//' | sed 's/, *$//'`
+      py=`grep '^     & *nPy *=' $mpi_size | sed 's/^     & *nPy *= *//' | sed 's/, *$//'`
+      pp=`expr $px \* $py` > /dev/null 2>&1 ; out=$?
+      if test "x$out" = x0 ; then LOC_NPROC=$pp ; fi
+    fi
+    rCommand=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"`
+    if test "x$MPI_MFILE" != x ; then
+      #- create new MPI machine-file with the right number of Procs
+      rm -f $LOC_MFILE
+      cat $MPI_MFILE | sort | uniq | head -$LOC_NPROC > $LOC_MFILE
+      nl=`wc -l $LOC_MFILE | awk '{print $1}'`
+      if [ $nl -lt $LOC_NPROC ] ; then
+        rm -f $LOC_MFILE
+        cat $MPI_MFILE | head -$LOC_NPROC > $LOC_MFILE
+      fi
+      rCommand=`echo $rCommand | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"`
+    fi
+  fi
+
   listD='run '`(cd $xx/results ; ls output.*.txt 2> /dev/null | sed 's/^output\./tr_run./g' | sed 's/\.txt$//g' )`
   for yy in $listD
   do
@@ -153,22 +208,33 @@
         comm=`grep '^# OPTFILE=' $xx/build/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
         echo "from '$xx/build/Makefile', extract:" > $DRESULTS/genmake_state
         sed -n '/^# executed by:/,+1 p' $xx/build/Makefile >> $DRESULTS/genmake_state
-        echo $comm >> $DRESULTS/genmake_state
+        echo " $comm" >> $DRESULTS/genmake_state
         eval $comm
+        gmkLog=$xx/build/genmake.log
+        grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1
+        out=$?
+        if test "x$out" = x0 ; then
+            echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state
+            sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \
+                       | grep -v '^... compiler version ' > tr_vers.tmp_log
+            sed -n '1,/^$/p' tr_vers.tmp_log | sed '/^$/d' | sed 's/^./ &/' \
+                      >> $DRESULTS/genmake_state
+            rm -f tr_vers.tmp_log
+        fi
       fi
       if test $yy = 'run' ; then nam=$xx
       else nam=$xx`echo $yy | sed 's/tr_run//'` ; fi
       echo -n "Entering $xx/$yy :"
       cd $xx/$yy
-      pwd > outp.tst_2+2.log
+      pwd > $LOGFILE
       if test "x$COMMAND" = x ; then
-        echo $SCRIPT $scrArg >> outp.tst_2+2.log ; echo ' ' >> outp.tst_2+2.log
-        $SCRIPT $scrArg >> outp.tst_2+2.log 2>&1
+        echo $SCRIPT $scrArg >> $LOGFILE ; echo ' ' >> $LOGFILE
+        $SCRIPT $scrArg >> $LOGFILE 2>&1
         out=$?
       else
-        echo "$SCRIPT $scrArg -command \"$COMMAND\"" >> outp.tst_2+2.log
-        echo ' ' >> outp.tst_2+2.log
-        $SCRIPT $scrArg -command "$COMMAND" >> outp.tst_2+2.log 2>&1
+        echo "$SCRIPT $scrArg -command \"$rCommand\"" >> $LOGFILE
+        echo ' ' >> $LOGFILE
+        $SCRIPT $scrArg -command "$rCommand" >> $LOGFILE 2>&1
         out=$?
       fi
       case $out in
@@ -183,25 +249,35 @@
       if test $out != '0' ; then
         echo " test 2+2=4 FAIL (exit $out)"
         echo " "
-        tail -5 outp.tst_2+2.log
+        tail -5 $LOGFILE
         echo " "
-        cp -p outp.tst_2+2.log ../../$DRESULTS/$nam.log
+        cp -p $LOGFILE ../../$DRESULTS/$nam.log
       elif test $SAVELOG = 1 ; then
-        cp -p outp.tst_2+2.log ../../$DRESULTS/$nam.log
+        cp -p $LOGFILE ../../$DRESULTS/$nam.log
       fi
       cd $here
     fi
   done
+  if test "x$mpi" != x0 -a "x$MPI_MFILE" != x ; then rm -f $LOC_MFILE ; fi
 done
 printf "End time:    " >> $SUMMARY
 date >> $SUMMARY
 
-#  If address was supplied and mpack exist, then send email using mpack.
+#  If address was supplied, then send email using Send-Command or mpack.
 if test "x$ADDRESS" != xNONE -a "x$ADDRESS" != x ; then
-    if test -x $MPACK ; then
-        tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
-            && gzip $DRESULTS".tar" \
-            && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESS
+    if test "x$SENDCMD" = x ; then
+      # if Send-Command is empty and mpack exist, then send email using mpack.
+      if test -x $MPACK ; then SENDCMD=$MPACK
+      else
+        echo "Warning: $MPACK is not executable => no email was sent"
+        echo
+      fi
+    fi
+    if test "x$SENDCMD" != x ; then
+        #echo " run: $SENDCMD -s MITgcm-test -m 3555000  ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESS"
+        tar -cf  ${SAVDIR}/${DRESULTS}".tar" $DRESULTS > /dev/null 2>&1 \
+            && gzip  ${SAVDIR}/${DRESULTS}".tar" \
+            && $SENDCMD -s MITgcm-test -m 3555000  ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESS
         out=$?
         if test "x$out" != x0 ; then
             echo
@@ -215,11 +291,8 @@
             echo "  \"$ADDRESS\""
             echo
         fi
-        test -f $DRESULTS".tar"  &&  rm -f $DRESULTS".tar"
-        test -f $DRESULTS".tar.gz"  &&  rm -f $DRESULTS".tar.gz"
-    else
-        echo "Warning: $MPACK is not executable => no email was sent"
-        echo
+        test -f ${SAVDIR}/${DRESULTS}".tar" &&  rm -f ${SAVDIR}/${DRESULTS}".tar"
+        test -f ${SAVDIR}/${DRESULTS}".tar.gz" &&  rm -f ${SAVDIR}/${DRESULTS}".tar.gz"
     fi
 fi
 
@@ -230,4 +303,3 @@
 if test "x$ADDRESS" = x ; then
   rm -rf $DRESULTS
 fi
-