--- MITgcm/verification/testreport	2003/12/07 07:06:13	1.32
+++ MITgcm/verification/testreport	2004/07/08 20:58:24	1.43
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.32 2003/12/07 07:06:13 edhill Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.43 2004/07/08 20:58:24 edhill Exp $
 #  $Name:  $
 #
 
@@ -13,7 +13,7 @@
     echo "  (-help|-h)               print usage"
     echo "  (-mpi)                   use MPI input files"
     echo "  (-ieee|-noieee)          if possible, use IEEE compiler flags"
-    echo "                             (DEF=\"noieee\")"
+    echo "                             (DEF=\"-ieee\")"
     echo "  (-optfile=|-of=)STRING   list of optfiles to use"
     echo "  (-a|-addr) STRING        list of email recipients"
     echo "                             (DEF=\"edhill@mitgcm.org\")"
@@ -27,11 +27,13 @@
     echo "                             (DEF=\"make output.txt\")"
     echo "  (-m|-make) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
+    echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\""
     echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
     echo "  (-nogenmake|-ng)         skip the genmake stage"
     echo "  (-noclean|-nc)           skip the \"make clean\" stage"
     echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
+    echo "  (-deldir|-dd)            on success, delete the output directory"
     echo
     echo "and where STRING follows a whitespace-delimited format"
     echo "such as:"
@@ -44,7 +46,7 @@
 #  build the mpack utility
 build_mpack()
 {
-    echo -n "building the mpack utility...  "
+    printf "building the mpack utility...  "
     if test ! -x "$MPACKDIR/mpack" ; then
 	if test ! -d $MPACKDIR ; then
 	    echo
@@ -54,7 +56,7 @@
 	    echo
 	    HAVE_MPACK=f
 	fi
-	echo -n "building mpack...  "
+	printf "building mpack...  "
 	if test "x$CC" = x ; then
 	    export CC=cc
 	fi
@@ -77,7 +79,7 @@
 
 testoutput_for_prop()
 {
-    # testoutput_for_prop dir s1 label subdir
+    # testoutput_for_prop dir s1 label subdir extension
     #
     #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
     #  using search strings s1 and text label 
@@ -99,9 +101,9 @@
 	return 99
     fi
     if [ $debug -gt 0 ]; then
-	echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
+	echo testoutput_for_prop: grep "$2" $1/results/output.txt$5 1>&2
     fi
-    grep "$2" $1/results/output.txt | sed 's/.*=//' | cat -n > tmp2.txt
+    grep "$2" $1/results/output.txt$5 | sed 's/.*=//' | cat -n > tmp2.txt
     lncnt=`wc -l tmp2.txt | awk '{print $1}' `
     if [ $lncnt -lt 3 ]; then
 	if [ $verbose -gt 0 ]; then
@@ -171,33 +173,33 @@
 
 testoutput()
 {
-    # testoutput directory subdir
+    # testoutput directory subdir extension
     #
     #  test output in "directory"
     if test "x$ADM" = x ; then
 	if [ $debug -gt 0 ]; then
 	    echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
 	fi
-	testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
+	testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2 $3; cg2dres=$?
 	if [ $debug -gt 0 ]; then
 	    echo testoutput: cg2dres=$cg2dres 1>&2
 	fi
-	testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
-	testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
-	testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
-	testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
-	testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
-	testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
-	testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
-	testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
-	testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
-	testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
-	testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
-	testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
-	testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
-	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
-	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
-	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
+	testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2 $3; tmin=$?
+	testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2 $3; tmax=$?
+	testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2 $3; tmean=$?
+	testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2 $3; tsd=$?
+	testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2 $3; smin=$?
+	testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2 $3; smax=$?
+	testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2 $3; smean=$?
+	testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2 $3; ssd=$?
+	testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2 $3; umin=$?
+	testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2 $3; umax=$?
+	testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2 $3; umean=$?
+	testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2 $3; usd=$?
+	testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2 $3; vmin=$?
+	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$?
+	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$?
+	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$?
 	dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
 	    $umin $umax $umean $usd $vmin $vmax $vmean $vsd
     else
@@ -211,7 +213,11 @@
     if test "x$NOGENMAKE" = xt ; then
 	echo "genmake skipped!"
     else
-	GENMAKE2="$BASH ../../../tools/genmake2"
+	if test "x$BASH" = x ; then
+	    GENMAKE2="../../../tools/genmake2"
+	else
+	    GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
+	fi
 	(
 	    cd $1;
 	    command="$GENMAKE2  -ds -m $MAKE"
@@ -302,7 +308,11 @@
 	if test -r Makefile ; then
 	    printf 'make ... ' 1>&2
 	    if test "x$ADM" = x ; then
-		$MAKE >> make.log 2>&1
+		if test "x$JOBS" = x ; then
+		    $MAKE >> make.log 2>&1
+		else
+		    $MAKE -j $JOBS >> make.log 2>&1
+		fi
 	    else
 		$MAKE adall >> make.log 2>&1
 	    fi
@@ -409,7 +419,10 @@
 	cd $1
 	printf 'runmodel ... ' 1>&2
 	# make output.txt
-	$COMMAND >> run.log 2>&1
+	echo
+	# echo "COMMAND='$COMMAND'"
+	# echo "pwd='"`pwd`"'"
+	( eval $COMMAND ) >> run.log 2>&1
 	RETVAL=$?
 	if test "x$RETVAL" = x0 ; then
 	    echo successful 1>&2
@@ -432,15 +445,16 @@
 {
     # create codelet for comparing model output
 
-    echo -n "creating the comparison code...  "
+    printf "creating the comparison code...  "
     cat > tmp_cmpnum.c <<EOF
 #include <stdio.h>
 #include <math.h>
 int main( int argc, char** argv )  {
-  int linnum,best;
+  int linnum,best,lncnt;
   double a,b,diff;
   best = -16;
-  while( 1 )  {
+  lncnt = 0;
+  while( 1 & (lncnt+=1) < 999 )  {
     scanf("%d", &linnum);
     if (linnum == -1)  break;
     scanf("%lf", &a);  scanf("%lf", &b);
@@ -456,6 +470,7 @@
       }
     }
   }
+  if (lncnt == 999) best=-29;
   printf("%d\n", -best);
   return 0;
 }
@@ -536,7 +551,7 @@
 expts=''
 # ieee=1
 
-IEEE=
+IEEE=true
 if test "x$MITGCM_IEEE" != x ; then
     IEEE=$MITGCM_IEEE
 fi
@@ -557,11 +572,13 @@
 MPACK="$MPACKDIR/mpack"
 COMMAND=
 MAKE=make
+JOBS=
 MPI=f
+DELDIR=
 
 ADM=
 
-echo -n "parsing options...  "
+printf "parsing options...  "
 
 ac_prev=
 for ac_option ; do
@@ -610,6 +627,9 @@
 	-make=* | --make=*)
 	    MAKE=$ac_optarg ;;
 
+	-j) ac_prev=JOBS ;;
+	-j=*) JOBS=$ac_optarg ;;
+
 	-clean | --clean)
 	    CLEANUP=t ;;
 
@@ -633,6 +653,8 @@
 	-debug) debug=1 ;;
 	-quiet) verbose=0 ;;
 
+	-deldir | -dd) DELDIR=t ;;
+
         -*)
             echo "Error: unrecognized option: "$ac_option
             usage
@@ -699,7 +721,7 @@
     exit 1
 fi
 SUMMARY="$DRESULTS/summary.txt"
-echo -n "Start time:  " >> $SUMMARY
+printf "Start time:  " >> $SUMMARY
 start_date=`date`
 echo $start_date > $SUMMARY
 
@@ -788,6 +810,20 @@
 	linkdata $use_seperate_build $dir/$rundir
     fi
     
+    #  Check whether there are "extra runs" for this testdir
+    extra_runs=
+    if test "x$ADM" = x -a "x$use_seperate_build" = x1 ; then
+	ex_run_dirs=`( cd $dir ; echo input.* )`
+	echo "ex_run_dirs='$ex_run_dirs'"
+	for exd in $ex_run_dirs ; do
+	    name=`echo $exd | sed -e 's/input.//g'`
+	    outf="$dir/results/output.txt.$name"
+	    if test -f $outf -a -r $outf ; then
+		extra_runs="$extra_runs $name"
+	    fi
+	done
+    fi
+
     if test "x$ADM" = x ; then
 	code_dir=code
 	CODE_DIR=$dir/code
@@ -830,25 +866,68 @@
     echo
     if test "x$ADM" = x ; then
 	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
+	echo
+	echo "$fres" >> $SUMMARY
+	echo "fresults='$fres'" > $CDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+
+	OLD_COMMAND=$COMMAND
+	COMMAND="./mitgcmuv > output.txt"
+	for ex in $extra_runs ; do
+	    test -e "$dir/tr_run.$ex" && rm -rf "$dir/tr_run.$ex"
+	    mkdir "$dir/tr_run.$ex"
+	    links=`( cd "$dir/input" > /dev/null 2>&1 ; ls -1 | grep -v CVS )`
+	    (
+		cd "$dir/tr_run.$ex"
+		for i in $links; do
+		    ln -s ../input/$i $i
+		done
+	    )
+	    links=`( cd "$dir/input.$ex" > /dev/null 2>&1 ; ls -1 | grep -v CVS )`
+	    ( 
+		cd "$dir/tr_run.$ex"
+		for i in $links; do
+		    test -e $i  &&  rm -f $i
+		    ln -s ../input.$ex/$i $i
+		done
+		ln -s ../$builddir/mitgcmuv mitgcmuv
+	    )
+	    runmodel $dir/tr_run.$ex && run=Y \
+		&& results=`testoutput $dir tr_run.$ex "."$ex`
+	    fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
+	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
+	    fres="$fres.$ex"
+	    echo
+	    echo "$fres" >> $SUMMARY
+	    echo "fresults='$fres'" > $CDIR"/summary.txt"
+	    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	    echo "tdir='$dir'" >> $CDIR"/summary.txt"
+	done
+	COMMAND=$OLD_COMMAND
     else
 	fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
 	fres=$fres"$results   $dir"
+	echo
+	echo "$fres" >> $SUMMARY
+	echo "fresults='$fres'" > $CDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	echo "tdir='$dir'" >> $CDIR"/summary.txt"
     fi
-    echo
-    echo "$fres" >> $SUMMARY
-    echo "fresults='$fres'" > $CDIR"/summary.txt"
-    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
-    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
-    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-    echo "tdir='$dir'" >> $CDIR"/summary.txt"
     
     echo "-------------------------------------------------------------------------------"
     
 done
 
-echo -n "Start time:  " >> $SUMMARY
+printf "Start time:  " >> $SUMMARY
 echo $start_date >> $SUMMARY
-echo -n "End time:    " >> $SUMMARY
+printf "End time:    " >> $SUMMARY
 date >> $SUMMARY
 
 #  If addresses were supplied and mpack built successfully, then try
@@ -889,3 +968,7 @@
     cat $SUMMARY > tr_out.txt
 fi
 
+if test "x$DELDIR" = xt ; then
+    rm -rf $DRESULTS
+fi
+