--- MITgcm/verification/testreport	2004/07/06 20:14:32	1.41
+++ MITgcm/verification/testreport	2004/11/24 17:06:07	1.50
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.41 2004/07/06 20:14:32 edhill Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.50 2004/11/24 17:06:07 edhill Exp $
 #  $Name:  $
 #
 
@@ -27,6 +27,9 @@
     echo "                             (DEF=\"make output.txt\")"
     echo "  (-m|-make) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
+    echo "  (-ptracers|-ptr) STRING  specify which ptracers to test"
+    echo "                             (DEF=\"1 2 3 4 5\")"
+    echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\""
     echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
     echo "  (-nogenmake|-ng)         skip the genmake stage"
@@ -34,11 +37,15 @@
     echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
     echo "  (-deldir|-dd)            on success, delete the output directory"
     echo
-    echo "and where STRING follows a whitespace-delimited format"
+    echo "and where STRING can be a whitespace-delimited list"
     echo "such as:"
+    echo
     echo "  -t 'exp0 exp2 exp3' "
     echo "  -addr='abc@123.com testing@home.org'"
     echo
+    echo "provided that the expression is properly quoted within the current"
+    echo "shell (note the use of single quotes to protect white space)."
+    echo
     exit 1
 }
 
@@ -170,6 +177,31 @@
     rm -f t[01][56].txt t[56].txt
 }
 
+check_for_add_mon_output()
+{
+    # Check for additional types of monitor output
+    if test "x$1" = x ; then
+	return
+    fi
+
+    for ii in $PTRACERS_NUM ; do
+	eval "HAVE_PTR0"$ii"=f"
+    done
+
+    ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
+    ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
+    for ii in $PTRACERS_NUM ; do
+	for jj in $ptr_add ; do
+	    name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
+	    tst=`grep $name $1 | wc -l | awk '{print $1}'`
+	    if test ! "x$tst" = x0 ; then
+		eval "HAVE_PTR0"$ii"=t"
+	    fi
+	done
+	#  eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
+    done
+}
+
 testoutput()
 {
     # testoutput directory subdir extension
@@ -199,8 +231,38 @@
 	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$?
 	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$?
 	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$?
-	dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
-	    $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+
+	#  This is for PTRACERS
+	for ii in $PTRACERS_NUM ; do
+	    eval `echo "p0"$ii"_min=99"`
+	    eval `echo "p0"$ii"_max=99"`
+	    eval `echo "p0"$ii"_mean=99"`
+	    eval `echo "p0"$ii"_sd=99"`
+	    tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
+	    #echo 'tst = '$tst
+	    if test "x$tst" = xt ; then
+		a="trcstat_ptracer0"
+		testoutput_for_prop $1 "$a"$ii"_min"  "p0"$ii"_min"  $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_min="$RETVAL`
+		testoutput_for_prop $1 "$a"$ii"_max"  "p0"$ii"_max"  $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_max="$RETVAL`
+		testoutput_for_prop $1 "$a"$ii"_mean" "p0"$ii"_mean" $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_mean="$RETVAL`
+		testoutput_for_prop $1 "$a"$ii"_sd"   "p0"$ii"_sd"   $2 $3
+		RETVAL=$? ; eval `echo "p0"$ii"_sd="$RETVAL`
+	    fi
+	done
+
+	allargs="$cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd"
+	allargs="$allargs $umin $umax $umean $usd $vmin $vmax $vmean $vsd"
+	allargs="$allargs $p01_min $p01_max $p01_mean $p01_sd"
+	allargs="$allargs $p02_min $p02_max $p02_mean $p02_sd"
+	allargs="$allargs $p03_min $p03_max $p03_mean $p03_sd"
+	allargs="$allargs $p04_min $p04_max $p04_mean $p04_sd"
+	allargs="$allargs $p05_min $p05_max $p05_mean $p05_sd"
+
+	eval "dashnum $allargs"
+
     else
 	testoutput_ad $1 $2 "precision_grdchk_result"
     fi
@@ -235,7 +297,8 @@
 	    printf 'genmake ... ' 1>&2
 	    $command > make.log 2>&1
 	    RETVAL=$?
-	    cp Makefile $CDIR
+	    #  Reduce the size of the testing emails!
+	    head -100 Makefile > $CDIR/Makefile_head
 	    if test "x$RETVAL" != x0 ; then
 		tail make.log
 		echo "genmakemodel: genmake failed" 1>&2
@@ -307,7 +370,11 @@
 	if test -r Makefile ; then
 	    printf 'make ... ' 1>&2
 	    if test "x$ADM" = x ; then
-		$MAKE >> make.log 2>&1
+		if test "x$JOBS" = x ; then
+		    $MAKE >> make.log 2>&1
+		else
+		    $MAKE -j $JOBS >> make.log 2>&1
+		fi
 	    else
 		$MAKE adall >> make.log 2>&1
 	    fi
@@ -334,7 +401,7 @@
     code_dir=$2
     BUILD_DIR=$dir/$3
     CODE_DIR=$dir/$code_dir
-
+    
     # These are files that should replace their counter-part when using -mpi
     MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
 
@@ -349,20 +416,21 @@
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
 		if ! test -f $BUILD_DIR/$i ; then
-	        #echo Linking $name to $i
+	            #echo Linking $name to $i
 		    (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
 		fi
 	    fi
 	done
     else
-    # NO: We undo any _mpi symbolically linked files
+        # NO: We undo any _mpi symbolically linked files
 	for ii in $MPI_FILES ; do
 	    i=`echo $ii | sed 's:^\./::'`
 	    name=`echo $i | sed 's:_mpi::' `
 	    if test -L $BUILD_DIR/$name ; then
-		linktarg=`(cd $BUILD_DIR; readlink $name)`
-		if test $linktarg = "../$code_dir/$name"_mpi ; then
-	        #echo Un-linking $name from $linktarg
+		cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
+		RETVAL=$?
+		if test "x$RETVAL" = x0 ; then
+		    #echo Un-linking $name from $linktarg
 		    rm -f $BUILD_DIR/$name
 		fi
 	    fi
@@ -415,22 +483,27 @@
 	printf 'runmodel ... ' 1>&2
 	# make output.txt
 	echo
+	rm -f run.log
 	# echo "COMMAND='$COMMAND'"
 	# echo "pwd='"`pwd`"'"
-	( eval $COMMAND ) >> run.log 2>&1
+	( eval $COMMAND ) > run.log 2>&1
 	RETVAL=$?
 	if test "x$RETVAL" = x0 ; then
+	    tail run.log
 	    echo successful 1>&2
-	    if test "x$ADM" = x ; then
-		cp output.txt $CDIR"/output.txt"
-	    else
-		cp output.txt_adm $CDIR"/output.txt_adm"
-	    fi
+	    # === Reduce the size of the testing emails!
+	    # if test "x$ADM" = x ; then
+	    #   cp output.txt $CDIR"/output.txt"
+	    # else
+	    #   cp output.txt_adm $CDIR"/output.txt_adm"
+	    # fi
+            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 0
 	else
 	    tail run.log
 	    echo failed 1>&2
 	    cp run.log $CDIR"/run.log"
+            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 1
 	fi
     )
@@ -567,11 +640,15 @@
 MPACK="$MPACKDIR/mpack"
 COMMAND=
 MAKE=make
+JOBS=
 MPI=f
 DELDIR=
 
 ADM=
 
+# Additional monitor types
+PTRACERS_NUM="1 2 3 4 5"
+
 printf "parsing options...  "
 
 ac_prev=
@@ -621,6 +698,14 @@
 	-make=* | --make=*)
 	    MAKE=$ac_optarg ;;
 
+	-ptracers | --ptracers | -ptr | --ptr)
+	    ac_prev=PTRACERS_NUM ;;
+	-ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
+	    PTRACERS_NUM=$ac_optarg ;;
+
+	-j) ac_prev=JOBS ;;
+	-j=*) JOBS=$ac_optarg ;;
+
 	-clean | --clean)
 	    CLEANUP=t ;;
 
@@ -679,7 +764,7 @@
 fi
 
 if test "x$COMMAND" = x ; then
-    COMMAND="make output.txt"
+    COMMAND="$MAKE output.txt"
 fi
 
 echo "OK"
@@ -744,14 +829,27 @@
 echo
 echo >> $SUMMARY
 if test "x$ADM" = x ; then
-    cat << EOF | tee -a $SUMMARY
-                T           S           U           V
-G D M    c        m  s        m  s        m  s        m  s
-E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
-N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
-2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
-
-EOF
+    line_0="            ----T-----  ----S-----  ----U-----  ----V-----"
+    line_1="G D M    c        m  s        m  s        m  s        m  s"
+    line_2="E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
+    line_3="N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
+    line_4="2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  ."
+    for ii in $PTRACERS_NUM ; do
+	#  tst=`eval 'echo $HAVE_PTR0'$ii`
+	#  if test "x$tst" = xt ; then
+	line_0="$line_0  --PTR 0"$ii"--"
+	line_1="$line_1        m  s"
+	line_2="$line_2  m  m  e  ."
+	line_3="$line_3  i  a  a  d"
+	line_4="$line_4  n  x  n  ."
+	#  fi
+    done
+    echo "$line_0" | tee -a $SUMMARY
+    echo "$line_1" | tee -a $SUMMARY
+    echo "$line_2" | tee -a $SUMMARY
+    echo "$line_3" | tee -a $SUMMARY
+    echo "$line_4" | tee -a $SUMMARY
+    echo " "       | tee -a $SUMMARY
 else
     echo "ADJOINT=true" >> $SUMMARY
     echo >> $SUMMARY
@@ -783,6 +881,7 @@
     fout=
     if test "x$ADM" = x ; then
 	fout=$dir"/results/output.txt"
+	check_for_add_mon_output  $fout
     else
 	fout=$dir"/results_ad/output.txt_adm"
     fi
@@ -791,6 +890,8 @@
 	continue
     fi
 
+    # Check for additional types of monitor output
+
     builddir="input"
     rundir="input"
     use_seperate_build=0
@@ -868,26 +969,26 @@
 	OLD_COMMAND=$COMMAND
 	COMMAND="./mitgcmuv > output.txt"
 	for ex in $extra_runs ; do
-	    test -e "$dir/tr_run" && rm -rf "$dir/tr_run"
-	    mkdir "$dir/tr_run"
+	    test -e "$dir/tr_run.$ex" && rm -rf "$dir/tr_run.$ex"
+	    mkdir "$dir/tr_run.$ex"
 	    links=`( cd "$dir/input" > /dev/null 2>&1 ; ls -1 | grep -v CVS )`
 	    (
-		cd "$dir/tr_run"
+		cd "$dir/tr_run.$ex"
 		for i in $links; do
 		    ln -s ../input/$i $i
 		done
 	    )
 	    links=`( cd "$dir/input.$ex" > /dev/null 2>&1 ; ls -1 | grep -v CVS )`
 	    ( 
-		cd "$dir/tr_run"
+		cd "$dir/tr_run.$ex"
 		for i in $links; do
 		    test -e $i  &&  rm -f $i
 		    ln -s ../input.$ex/$i $i
 		done
 		ln -s ../$builddir/mitgcmuv mitgcmuv
 	    )
-	    runmodel $dir/tr_run && run=Y \
-		&& results=`testoutput $dir tr_run "."$ex`
+	    runmodel $dir/tr_run.$ex && run=Y \
+		&& results=`testoutput $dir tr_run.$ex "."$ex`
 	    fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
 	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	    fres="$fres.$ex"
@@ -929,7 +1030,7 @@
     if test "x$HAVE_MPACK" = xt ; then
 	tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
 	    && gzip $DRESULTS".tar" \
-	    && $MPACK -s MITgcm-test -m 1500000 $DRESULTS".tar.gz" $ADDRESSES
+	    && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
 	RETVAL=$?
 	if test "x$RETVAL" != x0 ; then
 	    echo