--- MITgcm/verification/testreport	2003/12/05 17:36:03	1.29
+++ MITgcm/verification/testreport	2006/03/20 21:50:36	1.77
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.29 2003/12/05 17:36:03 adcroft Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.77 2006/03/20 21:50:36 jmc Exp $
 #  $Name:  $
 #
 
@@ -13,7 +13,7 @@
     echo "  (-help|-h)               print usage"
     echo "  (-mpi)                   use MPI input files"
     echo "  (-ieee|-noieee)          if possible, use IEEE compiler flags"
-    echo "                             (DEF=\"noieee\")"
+    echo "                             (DEF=\"-ieee\")"
     echo "  (-optfile=|-of=)STRING   list of optfiles to use"
     echo "  (-a|-addr) STRING        list of email recipients"
     echo "                             (DEF=\"edhill@mitgcm.org\")"
@@ -27,24 +27,34 @@
     echo "                             (DEF=\"make output.txt\")"
     echo "  (-m|-make) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
+    echo "  (-odir) STRING           used to build output directory name"
+    echo "                             (DEF=\"hostname\")"
+    echo "  (-ptracers|-ptr) STRING  specify which ptracers to test"
+    echo "                             (DEF=\"1 2 3 4 5\")"
+    echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\""
     echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
     echo "  (-nogenmake|-ng)         skip the genmake stage"
     echo "  (-noclean|-nc)           skip the \"make clean\" stage"
     echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
+    echo "  (-deldir|-dd)            on success, delete the output directory"
     echo
-    echo "and where STRING follows a whitespace-delimited format"
+    echo "and where STRING can be a whitespace-delimited list"
     echo "such as:"
+    echo
     echo "  -t 'exp0 exp2 exp3' "
     echo "  -addr='abc@123.com testing@home.org'"
     echo
+    echo "provided that the expression is properly quoted within the current"
+    echo "shell (note the use of single quotes to protect white space)."
+    echo
     exit 1
 }
 
 #  build the mpack utility
 build_mpack()
 {
-    echo -n "building the mpack utility...  "
+    printf "building the mpack utility...  "
     if test ! -x "$MPACKDIR/mpack" ; then
 	if test ! -d $MPACKDIR ; then
 	    echo
@@ -54,7 +64,7 @@
 	    echo
 	    HAVE_MPACK=f
 	fi
-	echo -n "building mpack...  "
+	printf "building mpack...  "
 	if test "x$CC" = x ; then
 	    export CC=cc
 	fi
@@ -77,7 +87,7 @@
 
 testoutput_for_prop()
 {
-    # testoutput_for_prop dir s1 label subdir
+    # testoutput_for_prop dir s1 label subdir extension
     #
     #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
     #  using search strings s1 and text label 
@@ -87,8 +97,8 @@
     fi
     if [ -r $1/$4/output.txt ]; then
 	grep "$2" $1/$4/output.txt | sed 's/.*=//' | cat -n > tmp1.txt
-	lncnt=`wc -l tmp1.txt | awk '{print $1}' `
-	if [ $lncnt -lt 3 ]; then
+	lncntA=`wc -l tmp1.txt | awk '{print $1}' `
+	if [ $lncntA -lt 3 ]; then
 	    if [ $verbose -gt 0 ]; then
 		echo Not enough lines of output when searching for "$2" 1>&2
 	    fi
@@ -99,16 +109,32 @@
 	return 99
     fi
     if [ $debug -gt 0 ]; then
-	echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
+	echo testoutput_for_prop: grep "$2" $1/results/output.txt$5 1>&2
     fi
-    grep "$2" $1/results/output.txt | sed 's/.*=//' | cat -n > tmp2.txt
-    lncnt=`wc -l tmp2.txt | awk '{print $1}' `
-    if [ $lncnt -lt 3 ]; then
+    grep "$2" $1/results/output.txt$5 | sed 's/.*=//' | cat -n > tmp2.txt
+    lncntB=`wc -l tmp2.txt | awk '{print $1}' `
+    if [ $lncntB -lt 3 ]; then
 	if [ $verbose -gt 0 ]; then
 	    echo Not enough lines of output when searching for "$2" 1>&2
 	fi
 	return 99
     fi
+    if [ $lncntA -ne $lncntB ]; then
+        if [ $verbose -gt 0 ]; then
+            echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
+        fi
+        return 99
+    fi
+    has_nan=`cat tmp1.txt | grep -i nan | wc -l`
+    if [ $has_nan -gt 0  ] ; then
+	echo testoutput_for_prop: output.txt contains $has_nan NaN values  1>&2
+	return 99
+    fi
+    has_inf=`cat tmp1.txt | grep -i inf | wc -l`
+    if [ $has_inf -gt 0  ] ; then
+	echo testoutput_for_prop: output.txt contains $has_inf Inf values  1>&2
+	return 99
+    fi
     if [ $debug -gt 0 ]; then
 	echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
     fi
@@ -169,37 +195,92 @@
     rm -f t[01][56].txt t[56].txt
 }
 
+check_for_add_mon_output()
+{
+    # Check for additional types of monitor output
+    if test "x$1" = x ; then
+	return
+    fi
+
+    for ii in $PTRACERS_NUM ; do
+	eval "HAVE_PTR0"$ii"=f"
+    done
+
+    ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
+    ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
+    for ii in $PTRACERS_NUM ; do
+	for jj in $ptr_add ; do
+	    name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
+	    tst=`grep $name $1 | wc -l | awk '{print $1}'`
+	    if test ! "x$tst" = x0 ; then
+		eval "HAVE_PTR0"$ii"=t"
+	    fi
+	done
+	#  eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
+    done
+}
+
 testoutput()
 {
-    # testoutput directory subdir
+    # testoutput directory subdir extension
     #
     #  test output in "directory"
     if test "x$ADM" = x ; then
 	if [ $debug -gt 0 ]; then
 	    echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
 	fi
-	testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
+	testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2 $3; cg2dres=$?
 	if [ $debug -gt 0 ]; then
 	    echo testoutput: cg2dres=$cg2dres 1>&2
 	fi
-	testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
-	testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
-	testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
-	testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
-	testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
-	testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
-	testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
-	testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
-	testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
-	testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
-	testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
-	testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
-	testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
-	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
-	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
-	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
-	dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
-	    $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+	testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2 $3; tmin=$?
+	testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2 $3; tmax=$?
+	testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2 $3; tmean=$?
+	testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2 $3; tsd=$?
+	testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2 $3; smin=$?
+	testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2 $3; smax=$?
+	testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2 $3; smean=$?
+	testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2 $3; ssd=$?
+	testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2 $3; umin=$?
+	testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2 $3; umax=$?
+	testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2 $3; umean=$?
+	testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2 $3; usd=$?
+	testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2 $3; vmin=$?
+	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$?
+	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$?
+	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$?
+
+	#  This is for PTRACERS
+	for ii in $PTRACERS_NUM ; do
+	    eval `echo "p0"$ii"_min=99"`
+	    eval `echo "p0"$ii"_max=99"`
+	    eval `echo "p0"$ii"_mean=99"`
+	    eval `echo "p0"$ii"_sd=99"`
+	    tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
+	    #echo 'tst = '$tst
+	    if test "x$tst" = xt ; then
+		a="trcstat_ptracer0"
+		testoutput_for_prop $1 "$a"$ii"_min"  "p0"$ii"_min"  $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_min="$RETVAL`
+		testoutput_for_prop $1 "$a"$ii"_max"  "p0"$ii"_max"  $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_max="$RETVAL`
+		testoutput_for_prop $1 "$a"$ii"_mean" "p0"$ii"_mean" $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_mean="$RETVAL`
+		testoutput_for_prop $1 "$a"$ii"_sd"   "p0"$ii"_sd"   $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_sd="$RETVAL`
+	    fi
+	done
+
+	allargs="$cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd"
+	allargs="$allargs $umin $umax $umean $usd $vmin $vmax $vmean $vsd"
+	allargs="$allargs $p01_min $p01_max $p01_mean $p01_sd"
+	allargs="$allargs $p02_min $p02_max $p02_mean $p02_sd"
+	allargs="$allargs $p03_min $p03_max $p03_mean $p03_sd"
+	allargs="$allargs $p04_min $p04_max $p04_mean $p04_sd"
+	allargs="$allargs $p05_min $p05_max $p05_mean $p05_sd"
+
+	eval "dashnum $allargs"
+
     else
 	testoutput_ad $1 $2 "precision_grdchk_result"
     fi
@@ -211,7 +292,11 @@
     if test "x$NOGENMAKE" = xt ; then
 	echo "genmake skipped!"
     else
-	GENMAKE2="$BASH ../../../tools/genmake2"
+	if test "x$BASH" = x ; then
+	    GENMAKE2="../../../tools/genmake2"
+	else
+	    GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
+	fi
 	(
 	    cd $1;
 	    command="$GENMAKE2  -ds -m $MAKE"
@@ -227,10 +312,14 @@
 	    if test "x$IEEE" != x ; then
 		command="$command -ieee"
 	    fi
+	    if test "x$MPI" = xt ; then
+		command="$command -mpi"
+	    fi
 	    printf 'genmake ... ' 1>&2
 	    $command > make.log 2>&1
 	    RETVAL=$?
-	    cp Makefile $CDIR
+	    #  Reduce the size of the testing emails!
+	    head -100 Makefile > $CDIR/Makefile_head
 	    if test "x$RETVAL" != x0 ; then
 		tail make.log
 		echo "genmakemodel: genmake failed" 1>&2
@@ -271,6 +360,20 @@
     fi
 }
 
+postclean()
+{
+    # postclean directory
+    if test "x$POSTCLEAN" = xt ; then
+	(
+	    cd $1;
+	    if test -r Makefile ; then
+		$MAKE CLEAN >> /dev/null 2>&1
+	    fi
+	    exit 0
+	)
+    fi
+}
+
 makedependmodel()
 {
     # makedependmodel directory
@@ -302,7 +405,11 @@
 	if test -r Makefile ; then
 	    printf 'make ... ' 1>&2
 	    if test "x$ADM" = x ; then
-		$MAKE >> make.log 2>&1
+		if test "x$JOBS" = x ; then
+		    $MAKE >> make.log 2>&1
+		else
+		    $MAKE -j $JOBS >> make.log 2>&1
+		fi
 	    else
 		$MAKE adall >> make.log 2>&1
 	    fi
@@ -329,7 +436,7 @@
     code_dir=$2
     BUILD_DIR=$dir/$3
     CODE_DIR=$dir/$code_dir
-
+    
     # These are files that should replace their counter-part when using -mpi
     MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
 
@@ -344,20 +451,21 @@
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
 		if ! test -f $BUILD_DIR/$i ; then
-	        #echo Linking $name to $i
+	            #echo Linking $name to $i
 		    (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
 		fi
 	    fi
 	done
     else
-    # NO: We undo any _mpi symbolically linked files
+        # NO: We undo any _mpi symbolically linked files
 	for ii in $MPI_FILES ; do
 	    i=`echo $ii | sed 's:^\./::'`
 	    name=`echo $i | sed 's:_mpi::' `
 	    if test -L $BUILD_DIR/$name ; then
-		linktarg=`(cd $BUILD_DIR; readlink $name)`
-		if test $linktarg = "../$code_dir/$name"_mpi ; then
-	        #echo Un-linking $name from $linktarg
+		cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
+		RETVAL=$?
+		if test "x$RETVAL" = x0 ; then
+		    #echo Un-linking $name from $linktarg
 		    rm -f $BUILD_DIR/$name
 		fi
 	    fi
@@ -377,7 +485,7 @@
 	    if test "x$ADM" = x ; then
 		files=`( cd ../input ; ls -1 | grep -v CVS )`
 		for i in $files ; do
-		    if test ! -d "../input/"$i ; then
+		    if test ! -d "../input/"$i -a ! -f $i ; then
 			ln -sf "../input/"$i $i
 		    fi
 		done
@@ -407,22 +515,35 @@
     #  (where "$COMMAND" is relative to "directory")
     (
 	cd $1
-	printf 'runmodel ... ' 1>&2
+	printf 'runmodel in %s ...' $1 1>&2
 	# make output.txt
-	$COMMAND >> run.log 2>&1
-	RETVAL=$?
+	echo
+	rm -f run.log
+        if test $OUTPUTFILE -ot $EXECUTABLE ; then
+	   ( eval $COMMAND ) > run.log 2>&1
+	   RETVAL=$?
+	else
+           echo " $OUTPUTFILE is up to date " > run.log 2>&1
+	   RETVAL=0
+	fi
+	# echo "COMMAND='$COMMAND'"
+	# echo "pwd='"`pwd`"'"
 	if test "x$RETVAL" = x0 ; then
+	    tail run.log
 	    echo successful 1>&2
-	    if test "x$ADM" = x ; then
-		cp output.txt $CDIR"/output.txt"
-	    else
-		cp output.txt_adm $CDIR"/output.txt_adm"
-	    fi
+	    # === Reduce the size of the testing emails!
+	    # if test "x$ADM" = x ; then
+	    #   cp output.txt $CDIR"/output.txt"
+	    # else
+	    #   cp output.txt_adm $CDIR"/output.txt_adm"
+	    # fi
+            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 0
 	else
 	    tail run.log
 	    echo failed 1>&2
 	    cp run.log $CDIR"/run.log"
+            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 1
 	fi
     )
@@ -432,42 +553,41 @@
 {
     # create codelet for comparing model output
 
-    echo -n "creating the comparison code...  "
+    printf "creating the comparison code...  "
     cat > tmp_cmpnum.c <<EOF
 #include <stdio.h>
 #include <math.h>
 int main( int argc, char** argv )  {
-  int linnum,best;
-  double a,b,diff;
-  best = -16;
-  while( 1 )  {
+  int linnum,best,lncnt;
+  double a,b,abave,relerr;
+  best = -22;
+  lncnt = 0;
+  while( 1 & (lncnt+=1) < 999 )  {
     scanf("%d", &linnum);
     if (linnum == -1)  break;
     scanf("%lf", &a);  scanf("%lf", &b);
-    diff = 0.5*(fabs(a)+fabs(b));
-    if (diff > 1.e-12) {
-      diff=fabs(a-b)/diff;
-      if (diff > 0.0) {
-        linnum = (int)log10(diff);
-        best = (best > linnum) ? best : linnum;
-      }
-      else {
-        if (best == -16 && diff != 0)  best = -22;
-      }
+    abave = 0.5*(fabs(a)+fabs(b));
+    if (abave > 0.0) {
+      relerr=fabs(a-b)/abave;
+      if (relerr > 0.0) { linnum = (int)rint(log10(relerr)); }
+      else { linnum = -16 ; }
+      best = (best > linnum) ? best : linnum;
     }
   }
+  if (lncnt == 999) best=-29;
   printf("%d\n", -best);
   return 0;
 }
 EOF
-    cc -o tmp_cmpnum tmp_cmpnum.c -lm
+    $CC -o tmp_cmpnum tmp_cmpnum.c -lm
 
     if [ -x ./tmp_cmpnum ]; then
 	echo "OK"
 	return 0
     else
 	echo
-	echo "ERROR: failed to compile comparison code" 
+	echo "ERROR: failed to compile comparison code -- please specify" 
+	echo "  a C compiler using the CC environment variable."
 	exit 1
     fi
 }
@@ -536,7 +656,7 @@
 expts=''
 # ieee=1
 
-IEEE=
+IEEE=true
 if test "x$MITGCM_IEEE" != x ; then
     IEEE=$MITGCM_IEEE
 fi
@@ -547,6 +667,7 @@
 NOGENMAKE=f
 NOCLEAN=f
 NODEPEND=f
+POSTCLEAN=f
 
 BASH=
 OPTFILE=NONE
@@ -556,12 +677,23 @@
 HAVE_MPACK=
 MPACK="$MPACKDIR/mpack"
 COMMAND=
-MAKE=make
+if test "x$MAKE" = x ; then
+    MAKE=make
+fi
+if test "x$CC" = x ; then
+    CC=cc
+fi
+JOBS=
 MPI=f
+OUTDIR=
+DELDIR=
 
 ADM=
 
-echo -n "parsing options...  "
+# Additional monitor types
+PTRACERS_NUM="1 2 3 4 5"
+
+printf "parsing options...  "
 
 ac_prev=
 for ac_option ; do
@@ -610,8 +742,21 @@
 	-make=* | --make=*)
 	    MAKE=$ac_optarg ;;
 
+        -odir | --odir)
+            ac_prev=OUTDIR ;;
+        -odir=* | --odir=*)
+            OUTDIR=$ac_optarg ;;
+
+	-ptracers | --ptracers | -ptr | --ptr)
+	    ac_prev=PTRACERS_NUM ;;
+	-ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
+	    PTRACERS_NUM=$ac_optarg ;;
+
+	-j) ac_prev=JOBS ;;
+	-j=*) JOBS=$ac_optarg ;;
+
 	-clean | --clean)
-	    CLEANUP=t ;;
+	    CLEANUP=t ; DELDIR=t ;;
 
 	-quick | --quick | -q | --q)
 	    QUICK=t ;;
@@ -622,6 +767,9 @@
 	-nodepend | --nodepend | -nd | --nd)
 	    NODEPEND=t ;;
 
+	-postclean | --postclean | -pc | --pc)
+	    POSTCLEAN=t ;;
+
 	-mpi) MPI=t ;;
 
 	-adm | -ad) ADM=t ;;
@@ -633,6 +781,8 @@
 	-debug) debug=1 ;;
 	-quiet) verbose=0 ;;
 
+	-deldir | -dd) DELDIR=t ;;
+
         -*)
             echo "Error: unrecognized option: "$ac_option
             usage
@@ -661,27 +811,43 @@
     OPTFILE=$MITGCM_OF
 fi
 
+if test "x$ADM" = xt ; then
+    EXECUTABLE="mitgcmuv_ad"
+    OUTPUTFILE="output.txt_adm"
+else
+    EXECUTABLE="mitgcmuv"
+    OUTPUTFILE="output.txt"
+fi
+
 if test "x$ADM" = xt -a "x$COMMAND" = x ; then
-    COMMAND="./mitgcmuv_ad > output.txt_adm 2>&1"
+    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
 fi
 
 if test "x$COMMAND" = x ; then
-    COMMAND="make output.txt"
+    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
 fi
 
-echo "OK"
+#echo "OK"
+echo "OK (COMMAND= $COMMAND )"
 
 #  create the FORTRAN comparison code
 createcodelet
 
 #  build the mpack utility
-build_mpack
+if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
+    echo "skipping mpack build"
+else
+    build_mpack
+fi
 
 #  Create a uniquely named directory to store results
 MACH=`hostname`
 UNAMEA=`uname -a`
 DATE=`date +%Y%m%d`
 BASE="tr_"$MACH"_"$DATE"_"
+if test "x$OUTDIR" != x ; then
+    BASE="tr_"$OUTDIR"_"$DATE"_"
+fi
 DNUM=0
 DRESULTS="$BASE$DNUM"
 while test -e $DRESULTS ; do
@@ -695,7 +861,7 @@
     exit 1
 fi
 SUMMARY="$DRESULTS/summary.txt"
-echo -n "Start time:  " >> $SUMMARY
+printf "Start time:  " >> $SUMMARY
 start_date=`date`
 echo $start_date > $SUMMARY
 
@@ -727,14 +893,27 @@
 echo
 echo >> $SUMMARY
 if test "x$ADM" = x ; then
-    cat << EOF | tee -a $SUMMARY
-                T           S           U           V
-G D M    c        m  s        m  s        m  s        m  s
-E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
-N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
-2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
-
-EOF
+    line_0="            ----T-----  ----S-----  ----U-----  ----V-----"
+    line_1="G D M    c        m  s        m  s        m  s        m  s"
+    line_2="E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
+    line_3="N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
+    line_4="2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  ."
+    for ii in $PTRACERS_NUM ; do
+	#  tst=`eval 'echo $HAVE_PTR0'$ii`
+	#  if test "x$tst" = xt ; then
+	line_0="$line_0  --PTR 0"$ii"--"
+	line_1="$line_1        m  s"
+	line_2="$line_2  m  m  e  ."
+	line_3="$line_3  i  a  a  d"
+	line_4="$line_4  n  x  n  ."
+	#  fi
+    done
+    echo "$line_0" | tee -a $SUMMARY
+    echo "$line_1" | tee -a $SUMMARY
+    echo "$line_2" | tee -a $SUMMARY
+    echo "$line_3" | tee -a $SUMMARY
+    echo "$line_4" | tee -a $SUMMARY
+    echo " "       | tee -a $SUMMARY
 else
     echo "ADJOINT=true" >> $SUMMARY
     echo >> $SUMMARY
@@ -758,6 +937,10 @@
 	if test -r $dir/input/Makefile ; then
 	    ( cd $dir/input ; make CLEAN )
 	fi
+	(
+	    cd $dir
+	    rm -rf tr_run.*
+	)
 	continue
     fi
 
@@ -773,6 +956,11 @@
 	echo "can't read \"$fout\" -- skipping $dir"
 	continue
     fi
+    if test "x$ADM" = x ; then
+	check_for_add_mon_output  $fout
+    fi
+
+    # Check for additional types of monitor output
 
     builddir="input"
     rundir="input"
@@ -784,6 +972,20 @@
 	linkdata $use_seperate_build $dir/$rundir
     fi
     
+    #  Check whether there are "extra runs" for this testdir
+    extra_runs=
+    if test "x$ADM" = x -a "x$use_seperate_build" = x1 ; then
+	ex_run_dirs=`( cd $dir ; echo input.* )`
+	echo "ex_run_dirs='$ex_run_dirs'"
+	for exd in $ex_run_dirs ; do
+	    name=`echo $exd | sed -e 's/input.//g'`
+	    outf="$dir/results/output.txt.$name"
+	    if test -f $outf -a -r $outf ; then
+		extra_runs="$extra_runs $name"
+	    fi
+	done
+    fi
+
     if test "x$ADM" = x ; then
 	code_dir=code
 	CODE_DIR=$dir/code
@@ -793,7 +995,7 @@
     fi
     BUILD_DIR=$dir/$builddir
 
-    if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" == "xt" ; then
+    if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
 	echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
 	continue
     fi
@@ -803,7 +1005,7 @@
     echo "Experiment:  $dir"
     echo
     unset genmake makedepend make run
-    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
 
     #  Create an output dir for each OPTFILE/tdir combination
     rel_CDIR=$DRESULTS"/"$dir
@@ -826,25 +1028,79 @@
     echo
     if test "x$ADM" = x ; then
 	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
+	echo
+	echo "$fres" >> $SUMMARY
+	echo "fresults='$fres'" > $CDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+
+	for ex in $extra_runs ; do
+	    unset run
+	    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+	    #  Create an output dir for each OPTFILE/tdir.ex combination
+	    rel_CDIR=$DRESULTS"/"$dir"."$ex
+	    mkdir $rel_CDIR
+	    CDIR=`pwd`"/$rel_CDIR"
+	    test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
+	    for ldir in input.$ex input ; do
+		(
+		    cd "$dir/$ldir" > /dev/null 2>&1
+		    ls -1 2>/dev/null \
+			| sed -e 's|^CVS$||g' | sed -e 's|^output.txt$||g'
+		) > tr_exrun_links
+		(
+		    cd "$dir/tr_run.$ex"
+		    cat ../../tr_exrun_links | while read i ; do
+			if test ! "x$i" = x ; then
+			    test ! -r $i  &&  ln -s "../"$ldir"/"$i $i
+			fi
+		    done
+		)
+		test -e tr_exrun_links  &&  rm -f tr_exrun_links
+	    done
+	    ldir=build
+	    (
+		cd $dir/tr_run.$ex
+		test ! -e mitgcmuv  &&  ln -s "../"$ldir"/"mitgcmuv .
+	    )
+	    runmodel $dir/tr_run.$ex && run=Y \
+		&& results=`testoutput $dir tr_run.$ex "."$ex`
+	    fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
+	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
+	    fres="$fres.$ex"
+	    echo
+	    echo "$fres" >> $SUMMARY
+	    echo "fresults='$fres'" > $CDIR"/summary.txt"
+	    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	    echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
+	done
     else
 	fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
 	fres=$fres"$results   $dir"
+	echo
+	echo "$fres" >> $SUMMARY
+	echo "fresults='$fres'" > $CDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+        grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
+			   >> $CDIR"/summary.txt"
     fi
-    echo
-    echo "$fres" >> $SUMMARY
-    echo "fresults='$fres'" > $CDIR"/summary.txt"
-    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
-    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
-    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-    echo "tdir='$dir'" >> $CDIR"/summary.txt"
+
+    postclean $dir/$builddir
     
     echo "-------------------------------------------------------------------------------"
     
 done
 
-echo -n "Start time:  " >> $SUMMARY
+printf "Start time:  " >> $SUMMARY
 echo $start_date >> $SUMMARY
-echo -n "End time:    " >> $SUMMARY
+printf "End time:    " >> $SUMMARY
 date >> $SUMMARY
 
 #  If addresses were supplied and mpack built successfully, then try
@@ -855,7 +1111,7 @@
     if test "x$HAVE_MPACK" = xt ; then
 	tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
 	    && gzip $DRESULTS".tar" \
-	    && $MPACK -s MITgcm-test -m 1500000 $DRESULTS".tar.gz" $ADDRESSES
+	    && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
 	RETVAL=$?
 	if test "x$RETVAL" != x0 ; then
 	    echo
@@ -878,10 +1134,14 @@
 rm -f tmp_cmpnum.c tmp_cmpnum
 
 if test "x$CLEANUP" != xt ; then
-    cat $SUMMARY
+    cat $SUMMARY | sed 's/ -- -- -- --//g'
     if test -e tr_out.txt ; then
         mv tr_out.txt tr_out.txt.old
     fi
-    cat $SUMMARY > tr_out.txt
+    cat $SUMMARY | sed 's/ -- -- -- --//g' > tr_out.txt
+fi
+
+if test "x$DELDIR" = xt ; then
+    rm -rf $DRESULTS
 fi