--- MITgcm/verification/testreport	2003/10/10 03:29:44	1.11
+++ MITgcm/verification/testreport	2004/07/17 02:08:45	1.45
@@ -1,6 +1,6 @@
-#!/bin/bash
+#! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.11 2003/10/10 03:29:44 edhill Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.45 2004/07/17 02:08:45 edhill Exp $
 #  $Name:  $
 #
 
@@ -13,22 +13,27 @@
     echo "  (-help|-h)               print usage"
     echo "  (-mpi)                   use MPI input files"
     echo "  (-ieee|-noieee)          if possible, use IEEE compiler flags"
+    echo "                             (DEF=\"-ieee\")"
     echo "  (-optfile=|-of=)STRING   list of optfiles to use"
-    echo "  (-a|-addr)STRING         list of email recipients"
+    echo "  (-a|-addr) STRING        list of email recipients"
     echo "                             (DEF=\"edhill@mitgcm.org\")"
-    echo "  (-t|-tdir)STRING         list of test dirs to use"
+    echo "  (-t|-tdir) STRING        list of test dirs to use"
     echo "                             (DEF=\"\" which builds all)"
-    echo "  (-b|-bash)STRING         location of \"bash\" executable"
-    echo "                             (DEF=\"\" for \"/bin/bash\")"
-    echo "  (-command)STRING         command to run"
+    echo "  (-b|-bash) STRING        preferred location of a \"bash\" or"
+    echo "                             Bourne-compatible \"sh\" shell"
+    echo "                             (DEF=\"\" for \"bash\")"
+    echo "  (-adm|-ad)               perform an adjoint run"
+    echo "  (-command) STRING        command to run"
     echo "                             (DEF=\"make output.txt\")"
-    echo "  (-m|-make)STRING         command to use for \"make\""
+    echo "  (-m|-make) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
+    echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\""
     echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
     echo "  (-nogenmake|-ng)         skip the genmake stage"
     echo "  (-noclean|-nc)           skip the \"make clean\" stage"
     echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
+    echo "  (-deldir|-dd)            on success, delete the output directory"
     echo
     echo "and where STRING follows a whitespace-delimited format"
     echo "such as:"
@@ -41,38 +46,40 @@
 #  build the mpack utility
 build_mpack()
 {
-    echo -n "building the mpack utility...  "
+    printf "building the mpack utility...  "
     if test ! -x "$MPACKDIR/mpack" ; then
 	if test ! -d $MPACKDIR ; then
+	    echo
 	    echo "Error: can't find \"$MPACKDIR\""
 	    echo "  are you sure this program is being run in the correct "
 	    echo "  (that is, \"MITGCM_ROOT\verification\") directory?"
-	    exit 1
+	    echo
+	    HAVE_MPACK=f
+	fi
+	printf "building mpack...  "
+	if test "x$CC" = x ; then
+	    export CC=cc
 	fi
-	echo -n "building mpack...  "
-	( cd $MPACKDIR && ./configure && $MAKE ) > build_mpack.out 2>&1
+	( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
 	RETVAL=$?
 	if test "x$RETVAL" != x0 ; then
 	    echo
 	    echo "Error building the mpack tools at: $MPACK_DIR"
-	    exit 1
+	    echo
+	    HAVE_MPACK=f
+	else
+	    rm -f tr_build_mpack.out
+	    HAVE_MPACK=t
 	fi
+    else
+	HAVE_MPACK=t
     fi
     echo "OK"
 }
 
-compare_lines()
-{
-    # use codelet to compare lines
-    if [ $verbose -gt 1 ]; then
-	cat tmp3.txt 1>&2
-    fi
-    return `./a.out < tmp3.txt`
-}
-
 testoutput_for_prop()
 {
-    # testoutput_for_prop dir s1 label subdir
+    # testoutput_for_prop dir s1 label subdir extension
     #
     #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
     #  using search strings s1 and text label 
@@ -81,7 +88,7 @@
 	echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
     fi
     if [ -r $1/$4/output.txt ]; then
-	grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
+	grep "$2" $1/$4/output.txt | sed 's/.*=//' | cat -n > tmp1.txt
 	lncnt=`wc -l tmp1.txt | awk '{print $1}' `
 	if [ $lncnt -lt 3 ]; then
 	    if [ $verbose -gt 0 ]; then
@@ -94,9 +101,9 @@
 	return 99
     fi
     if [ $debug -gt 0 ]; then
-	echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
+	echo testoutput_for_prop: grep "$2" $1/results/output.txt$5 1>&2
     fi
-    grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
+    grep "$2" $1/results/output.txt$5 | sed 's/.*=//' | cat -n > tmp2.txt
     lncnt=`wc -l tmp2.txt | awk '{print $1}' `
     if [ $lncnt -lt 3 ]; then
 	if [ $verbose -gt 0 ]; then
@@ -111,8 +118,13 @@
     if [ $debug -gt 0 ]; then
 	echo testoutput_for_prop: compare_lines 1>&2
     fi
-    compare_lines
-    digits_of_similarity=$?
+    if [ $verbose -gt 1 ]; then
+	cat tmp3.txt 1>&2
+    fi
+    echo "-1" >> tmp3.txt
+    # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
+    cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
+    digits_of_similarity=`./tmp_cmpnum < tmp4.txt`
     if [ $digits_of_similarity -eq 99 ]; then
 	if [ $verbose -gt 0 ]; then
 	    echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
@@ -123,7 +135,7 @@
 	    echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
 	fi
     fi
-    rm tmp1.txt tmp2.txt tmp3.txt
+    rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
     
     return $digits_of_similarity
 }
@@ -143,39 +155,56 @@
     done
 }
 
+testoutput_ad()
+{
+    grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $5}' > t05.txt
+    grep $3 $1/$2/output.txt_adm | awk '{print NR " " $5}' > t15.txt
+    grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $6}' > t06.txt
+    grep $3 $1/$2/output.txt_adm | awk '{print NR " " $6}' > t16.txt
+    join t05.txt t15.txt > t5.txt
+    join t06.txt t16.txt > t6.txt
+    echo "-1" >> t5.txt
+    echo "-1" >> t6.txt
+    digits_5=`./tmp_cmpnum < t5.txt`
+    digits_6=`./tmp_cmpnum < t6.txt`
+    dashnum $digits_5 $digits_6
+    rm -f t[01][56].txt t[56].txt
+}
+
 testoutput()
 {
-    # testoutput diretory subdir
+    # testoutput directory subdir extension
     #
     #  test output in "directory"
-
-    if [ $debug -gt 0 ]; then
-	echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
-    fi
-    testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
-    if [ $debug -gt 0 ]; then
-	echo testoutput: cg2dres=$cg2dres 1>&2
+    if test "x$ADM" = x ; then
+	if [ $debug -gt 0 ]; then
+	    echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
+	fi
+	testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2 $3; cg2dres=$?
+	if [ $debug -gt 0 ]; then
+	    echo testoutput: cg2dres=$cg2dres 1>&2
+	fi
+	testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2 $3; tmin=$?
+	testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2 $3; tmax=$?
+	testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2 $3; tmean=$?
+	testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2 $3; tsd=$?
+	testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2 $3; smin=$?
+	testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2 $3; smax=$?
+	testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2 $3; smean=$?
+	testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2 $3; ssd=$?
+	testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2 $3; umin=$?
+	testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2 $3; umax=$?
+	testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2 $3; umean=$?
+	testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2 $3; usd=$?
+	testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2 $3; vmin=$?
+	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$?
+	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$?
+	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$?
+	dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+	    $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+    else
+	testoutput_ad $1 $2 "precision_grdchk_result"
     fi
-    
-    testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
-    testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
-    testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
-    testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
-    testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
-    testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
-    testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
-    testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
-    testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
-    testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
-    testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
-    testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
-    testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
-    testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
-    testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
-    testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
-    
-    dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
-        $umin $umax $umean $usd $vmin $vmax $vmean $vsd
 }
 
 genmakemodel()
@@ -184,33 +213,38 @@
     if test "x$NOGENMAKE" = xt ; then
 	echo "genmake skipped!"
     else
-	GENMAKE2="$BASH ../../../tools/genmake2"
+	if test "x$BASH" = x ; then
+	    GENMAKE2="../../../tools/genmake2"
+	else
+	    GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
+	fi
 	(
 	    cd $1;
-	    command="$GENMAKE2  -ds -m $MAKE --mods=../code"
+	    command="$GENMAKE2  -ds -m $MAKE"
+	    if test "x$ADM" = x ; then
+		command="$command --mods=../code"
+	    else
+		command="$command --mods=../code_ad"
+		command="$command -adof=../../../tools/adjoint_options/adjoint_staf"
+	    fi
 	    if test "x$OPTFILE" != xNONE ; then
 		command="$command --optfile=$OPTFILE"
-		# echo "  command=\"$command\""
 	    fi
 	    if test "x$IEEE" != x ; then
 		command="$command -ieee"
 	    fi
-	    # echo "command: \"$command\""
 	    printf 'genmake ... ' 1>&2
 	    $command > make.log 2>&1
 	    RETVAL=$?
-	    for i in genmake_state genmake_optfile genmake_local Makefile ; do
-		if test -r $i ; then
-		    cp $i $CDIR
-		fi
-	    done
+	    #  Reduce the size of the testing emails!
+	    #  cp Makefile $CDIR
 	    if test "x$RETVAL" != x0 ; then
 		tail make.log
 		echo "genmakemodel: genmake failed" 1>&2
-		cp make.log $CDIR
+		cp genmake_* make.log $CDIR
 		return 1
 	    else
-		echo "succesful" 1>&2
+		echo "successful" 1>&2
 	    fi
 	)
     fi
@@ -238,7 +272,7 @@
 		    return 1
 		fi
 	    fi
-	    echo succesful 1>&2
+	    echo successful 1>&2
 	    exit 0
 	)
     fi
@@ -261,7 +295,7 @@
 		cp make.log $CDIR"/make.log"
 		return 1
 	    else
-		echo succesful 1>&2
+		echo successful 1>&2
 	    fi
 	)
     fi
@@ -274,7 +308,15 @@
 	cd $1;
 	if test -r Makefile ; then
 	    printf 'make ... ' 1>&2
-	    $MAKE >> make.log 2>&1
+	    if test "x$ADM" = x ; then
+		if test "x$JOBS" = x ; then
+		    $MAKE >> make.log 2>&1
+		else
+		    $MAKE -j $JOBS >> make.log 2>&1
+		fi
+	    else
+		$MAKE adall >> make.log 2>&1
+	    fi
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
 		tail make.log
@@ -282,19 +324,90 @@
 		cp make.log $CDIR"/make.log"
 		return 1
 	    else
-		echo succesful 1>&2
+		echo successful 1>&2
 	    fi
 	fi
     )
 }
 
+symlink_mpifiles()
+{
+    # Put special links so that MPI specific files are used
+    # This MUST be invoked between makeclean and makelinks because
+    # the Makefile will link to non-mpi files by default
+
+    dir=$1
+    code_dir=$2
+    BUILD_DIR=$dir/$3
+    CODE_DIR=$dir/$code_dir
+    
+    # These are files that should replace their counter-part when using -mpi
+    MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
+
+    #  Is this an MPI run?
+    if test "x$MPI" = xt ; then
+	# YES: We symbolically link these files to the build
+	# dir so long as there is no real file in place
+	for ii in $MPI_FILES ; do
+	    i=`echo $ii | sed 's:^\./::'`
+	    name=`echo $i | sed 's:_mpi::' `
+	    cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
+	    RETVAL=$?
+	    if test "x$RETVAL" != x0 ; then
+		if ! test -f $BUILD_DIR/$i ; then
+	            #echo Linking $name to $i
+		    (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
+		fi
+	    fi
+	done
+    else
+        # NO: We undo any _mpi symbolically linked files
+	for ii in $MPI_FILES ; do
+	    i=`echo $ii | sed 's:^\./::'`
+	    name=`echo $i | sed 's:_mpi::' `
+	    if test -L $BUILD_DIR/$name ; then
+		cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
+		RETVAL=$?
+		if test "x$RETVAL" = x0 ; then
+		    #echo Un-linking $name from $linktarg
+		    rm -f $BUILD_DIR/$name
+		fi
+	    fi
+	done
+    fi
+    
+}
+
 linkdata()
 {
     # linkdata flag
     #
     # symbolically link data files to run directory
-    if [ $1 -ne 0 ]; then
-	( cd $2 ;  ln -sf ../input/* . )
+    if test "x$1" = x1 ; then
+	(
+	    cd $2
+	    if test "x$ADM" = x ; then
+		files=`( cd ../input ; ls -1 | grep -v CVS )`
+		for i in $files ; do
+		    if test ! -d "../input/"$i ; then
+			ln -sf "../input/"$i $i
+		    fi
+		done
+	    else
+		files=`( cd ../input ; ls -1 *.bin | grep -v CVS )`
+		for i in $files ; do
+		    if test ! -d "../input/"$i ; then
+			ln -sf "../input/"$i $i
+		    fi
+		done
+		files=`( cd ../input_ad ; ls -1 | grep -v CVS )`
+		for i in $files ; do
+		    if test ! -d "../input_ad/"$i ; then
+			ln -sf "../input_ad/"$i $i
+		    fi
+		done
+	    fi
+	)
     fi
 }
 
@@ -302,18 +415,30 @@
 {
     # runmodel directory
     #
-    #  runs "$COMMAND" in "directory" 
+    #  runs "$COMMAND in "directory" 
     #  (where "$COMMAND" is relative to "directory")
     (
 	cd $1
-	printf 'runmodel: ' 1>&2
+	printf 'runmodel ... ' 1>&2
 	# make output.txt
-	$COMMAND
+	echo
+	# echo "COMMAND='$COMMAND'"
+	# echo "pwd='"`pwd`"'"
+	( eval $COMMAND ) >> run.log 2>&1
 	RETVAL=$?
 	if test "x$RETVAL" = x0 ; then
-	    cp output.txt $CDIR"/output.txt"
+	    echo successful 1>&2
+	    # === Reduce the size of the testing emails!
+	    # if test "x$ADM" = x ; then
+	    #   cp output.txt $CDIR"/output.txt"
+	    # else
+	    #   cp output.txt_adm $CDIR"/output.txt_adm"
+	    # fi
 	    return 0
 	else
+	    tail run.log
+	    echo failed 1>&2
+	    cp run.log $CDIR"/run.log"
 	    return 1
 	fi
     )
@@ -323,39 +448,44 @@
 {
     # create codelet for comparing model output
 
-    echo -n "creating the comparison code...  "
-    cat > tmp_cmpnum.f <<EOFA
-      program cmpnum
-      implicit none
-      real*8 a,b,diff
-      integer linnum,best
-      best=-16
-  99  read(*,*,end=70,err=60) linnum,a,b
-      diff=0.5*(abs(a)+abs(b))
-c     print *,a,b,diff,abs(a-b)/diff
-      if (diff.gt.1.e-12) then
-        diff=abs(a-b)/diff
-        if (diff.gt.0.) then
-c         print *,int(log10(diff)),diff
-          linnum=int(log10(diff))
-          best=max(best,linnum)
-        endif
-      else
-        if (best.eq.-16.and.diff.ne.0.) best=-22
-      endif
-      goto 99
-  60  stop 'cmpnum: An error occured reading a,b'
-  70  print *,-best
-      end
-EOFA
+    printf "creating the comparison code...  "
+    cat > tmp_cmpnum.c <<EOF
+#include <stdio.h>
+#include <math.h>
+int main( int argc, char** argv )  {
+  int linnum,best,lncnt;
+  double a,b,diff;
+  best = -16;
+  lncnt = 0;
+  while( 1 & (lncnt+=1) < 999 )  {
+    scanf("%d", &linnum);
+    if (linnum == -1)  break;
+    scanf("%lf", &a);  scanf("%lf", &b);
+    diff = 0.5*(fabs(a)+fabs(b));
+    if (diff > 1.e-12) {
+      diff=fabs(a-b)/diff;
+      if (diff > 0.0) {
+        linnum = (int)log10(diff);
+        best = (best > linnum) ? best : linnum;
+      }
+      else {
+        if (best == -16 && diff != 0)  best = -22;
+      }
+    }
+  }
+  if (lncnt == 999) best=-29;
+  printf("%d\n", -best);
+  return 0;
+}
+EOF
+    cc -o tmp_cmpnum tmp_cmpnum.c -lm
 
-    f77 tmp_cmpnum.f
-    if [ -x ./a.out ]; then
+    if [ -x ./tmp_cmpnum ]; then
 	echo "OK"
 	return 0
     else
 	echo
-	echo "createcodelet: failed to compile codelet" 
+	echo "ERROR: failed to compile comparison code" 
 	exit 1
     fi
 }
@@ -415,8 +545,6 @@
 
 
 ###############################################################################
-###############################################################################
-###############################################################################
 
 
 #  Default properties
@@ -426,7 +554,7 @@
 expts=''
 # ieee=1
 
-IEEE=
+IEEE=true
 if test "x$MITGCM_IEEE" != x ; then
     IEEE=$MITGCM_IEEE
 fi
@@ -443,12 +571,17 @@
 ADDRESSES=
 TESTDIRS=
 MPACKDIR="../tools/mpack-1.6"
+HAVE_MPACK=
 MPACK="$MPACKDIR/mpack"
-COMMAND="make output.txt"
+COMMAND=
 MAKE=make
+JOBS=
 MPI=f
+DELDIR=
 
-echo -n "parsing options...  "
+ADM=
+
+printf "parsing options...  "
 
 ac_prev=
 for ac_option ; do
@@ -497,6 +630,9 @@
 	-make=* | --make=*)
 	    MAKE=$ac_optarg ;;
 
+	-j) ac_prev=JOBS ;;
+	-j=*) JOBS=$ac_optarg ;;
+
 	-clean | --clean)
 	    CLEANUP=t ;;
 
@@ -509,7 +645,9 @@
 	-nodepend | --nodepend | -nd | --nd)
 	    NODEPEND=t ;;
 
-	-mpi) MPI=true ;;
+	-mpi) MPI=t ;;
+
+	-adm | -ad) ADM=t ;;
 
 	-ieee) IEEE=true ;;
 	-noieee) IEEE= ;;
@@ -518,6 +656,8 @@
 	-debug) debug=1 ;;
 	-quiet) verbose=0 ;;
 
+	-deldir | -dd) DELDIR=t ;;
+
         -*)
             echo "Error: unrecognized option: "$ac_option
             usage
@@ -546,19 +686,31 @@
     OPTFILE=$MITGCM_OF
 fi
 
+if test "x$ADM" = xt -a "x$COMMAND" = x ; then
+    COMMAND="./mitgcmuv_ad > output.txt_adm 2>&1"
+fi
+
+if test "x$COMMAND" = x ; then
+    COMMAND="$MAKE output.txt"
+fi
+
 echo "OK"
 
 #  create the FORTRAN comparison code
 createcodelet
 
 #  build the mpack utility
-build_mpack
+if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
+    echo "skipping mpack build"
+else
+    build_mpack
+fi
 
 #  Create a uniquely named directory to store results
 MACH=`hostname`
 UNAMEA=`uname -a`
 DATE=`date +%Y%m%d`
-BASE=$MACH"_"$DATE"_"
+BASE="tr_"$MACH"_"$DATE"_"
 DNUM=0
 DRESULTS="$BASE$DNUM"
 while test -e $DRESULTS ; do
@@ -568,21 +720,13 @@
 mkdir $DRESULTS
 RETVAL=$?
 if test "x$RETVAL" != x0 ; then
-    echo "Error: can't create results directory \"./$DRESULTS\""
+    echo "ERROR: Can't create results directory \"./$DRESULTS\""
     exit 1
 fi
 SUMMARY="$DRESULTS/summary.txt"
-date > $SUMMARY
-cat << EOF >> $SUMMARY
-                T           S           U           V
-G D M    c        m  s        m  s        m  s        m  s
-E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
-N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
-2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
-
-EOF
-
-NDIR=0
+printf "Start time:  " >> $SUMMARY
+start_date=`date`
+echo $start_date > $SUMMARY
 
 of_path=
 if test "x$OPTFILE" != xNONE ; then
@@ -596,14 +740,41 @@
 	fi
 	file=${OPTFILE##*/}
 	OPTFILE=$of_path/$file
+	cp $OPTFILE $DRESULTS
+	echo >> $SUMMARY
+	echo "  OPTFILE=$OPTFILE" >> $SUMMARY
     else
-	echo
-	echo "WARNING: can't read OPTFILE=\"$OPTFILE\" but will try to use it..."
+	echo | tee $SUMMARY
+	echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
+	exit 1
     fi
+else
+    echo >> $SUMMARY
+    echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
+    echo "   so the genmake default will be used." >> $SUMMARY
 fi
 echo
-echo "OPTFILE=$OPTFILE" >> $SUMMARY
 echo >> $SUMMARY
+if test "x$ADM" = x ; then
+    cat << EOF | tee -a $SUMMARY
+                T           S           U           V
+G D M    c        m  s        m  s        m  s        m  s
+E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
+N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
+2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
+
+EOF
+else
+    echo "ADJOINT=true" >> $SUMMARY
+    echo >> $SUMMARY
+    cat << EOF | tee -a $SUMMARY
+G D M    C  G
+E p a R  o  r
+N n k u  s  a
+2 d e n  t  d
+
+EOF
+fi
 
 #  ...and each test directory...
 for dir in $TESTDIRS ; do
@@ -621,8 +792,52 @@
 
     #  Verify that the testdir exists and contains previous 
     #  results in the correct location--or skip this directory!
-    if test ! -r $dir"/results/output.txt" ; then
-	echo "can't read \"$dir/results/output.txt\" -- skipping $dir"
+    fout=
+    if test "x$ADM" = x ; then
+	fout=$dir"/results/output.txt"
+    else
+	fout=$dir"/results_ad/output.txt_adm"
+    fi
+    if test ! -r $fout ; then
+	echo "can't read \"$fout\" -- skipping $dir"
+	continue
+    fi
+
+    builddir="input"
+    rundir="input"
+    use_seperate_build=0
+    if test -d $dir/build -a -r $dir/build ; then
+	builddir="build"
+	rundir="build"
+	use_seperate_build=1
+	linkdata $use_seperate_build $dir/$rundir
+    fi
+    
+    #  Check whether there are "extra runs" for this testdir
+    extra_runs=
+    if test "x$ADM" = x -a "x$use_seperate_build" = x1 ; then
+	ex_run_dirs=`( cd $dir ; echo input.* )`
+	echo "ex_run_dirs='$ex_run_dirs'"
+	for exd in $ex_run_dirs ; do
+	    name=`echo $exd | sed -e 's/input.//g'`
+	    outf="$dir/results/output.txt.$name"
+	    if test -f $outf -a -r $outf ; then
+		extra_runs="$extra_runs $name"
+	    fi
+	done
+    fi
+
+    if test "x$ADM" = x ; then
+	code_dir=code
+	CODE_DIR=$dir/code
+    else
+	code_dir=code_ad
+	CODE_DIR=$dir/code_ad
+    fi
+    BUILD_DIR=$dir/$builddir
+
+    if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
+	echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
 	continue
     fi
 
@@ -633,114 +848,130 @@
     unset genmake makedepend make run
     results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
 
-    if [ -r $dir/build ]; then
-	seperatebuilddir=1
-	builddir=build
-	rundir=build
-	( cd $dir/build; ln -sf ../input/* . )
-    else
-	seperatebuilddir=0
-	builddir=input
-	rundir=input
-    fi
-    
-    CODE_DIR=$dir/code
-    BUILD_DIR=$dir/$builddir
-    MPI_FILES="CPP_EEOPTIONS.h_mpi SIZE.h_mpi"
-    NOMPI_FILES="CPP_EEOPTIONS.h_nompi SIZE.h_nompi"
-
-    #  Is this an MPI run?
-    if test "x$MPI" = xt ; then
-	FILES=$MPI_FILES
-	endings="_mpi"
-    else
-	FILES=$NOMPI_FILES
-	endings="_nompi"
-    fi
-    
-    #  Check to see that we have the files
-    have_files=t
-    for i in $FILES ; do
-	if test ! -r $CODE_DIR/$i ; then
-	    echo "Warning: can't read file $CODE_DIR/$i"
-	    have_files=f
-	fi
-    done
-    if test "x$have_files" != xt -a "x$MPI" = xt ; then
-	echo "Skipping $dir due to lack of input files (see above warning)"
-	continue
-    fi
-    
-    #  If we have the $FILES and they differ, copy the $FILES to $BUILD_DIR
-    if test "x$have_files" = xt ; then
-	for i in $FILES ; do
-	    sstr="s|$endings||"
-	    name=`echo $i | sed -e $sstr `
-	    cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
-	    RETVAL=$?
-	    if test "x$RETVAL" != x0 ; then
-		cp $CODE_DIR/$i $BUILD_DIR/$name
-	    fi
-	done
-    fi
-    
     #  Create an output dir for each OPTFILE/tdir combination
-    CDIR=$DRESULTS"/"$DRESULTS"_"$NDIR
-    mkdir $CDIR
-    CDIR=`pwd`"/$CDIR"
+    rel_CDIR=$DRESULTS"/"$dir
+    mkdir $rel_CDIR
+    CDIR=`pwd`"/$rel_CDIR"
     
     if test "x$CLEANUP" = xt ; then
 	makeclean $dir/$builddir
     else
 	genmakemodel $dir/$builddir && genmake=Y \
 	    && makeclean $dir/$builddir \
+            && symlink_mpifiles $dir $code_dir $builddir \
 	    && makedependmodel $dir/$builddir && makedepend=Y \
 	    && makemodel $dir/$builddir && make=Y \
-	    && linkdata $seperatebuilddir $dir/$rundir \
-	    && runmodel $dir/$builddir && run=Y \
+	    && linkdata $use_seperate_build $dir/$rundir \
+	    && runmodel $dir/$rundir && run=Y \
 	    && results=`testoutput $dir $rundir`
     fi
     
     echo
-    formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
-	${run:-N} $results
-    echo
-    formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
-	${run:-N} $results >> $SUMMARY
-    echo "fresults='" > $CDIR"/summary.txt"
-    formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
-	${run:-N} $results >> $CDIR"/summary.txt"
-    echo "'" >> $CDIR"/summary.txt"
-    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
-    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
-    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-    echo "tdir='$dir'" >> $CDIR"/summary.txt"
+    if test "x$ADM" = x ; then
+	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
+	echo
+	echo "$fres" >> $SUMMARY
+	echo "fresults='$fres'" > $CDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+
+	OLD_COMMAND=$COMMAND
+	COMMAND="./mitgcmuv > output.txt"
+	for ex in $extra_runs ; do
+	    test -e "$dir/tr_run.$ex" && rm -rf "$dir/tr_run.$ex"
+	    mkdir "$dir/tr_run.$ex"
+	    links=`( cd "$dir/input" > /dev/null 2>&1 ; ls -1 | grep -v CVS )`
+	    (
+		cd "$dir/tr_run.$ex"
+		for i in $links; do
+		    ln -s ../input/$i $i
+		done
+	    )
+	    links=`( cd "$dir/input.$ex" > /dev/null 2>&1 ; ls -1 | grep -v CVS )`
+	    ( 
+		cd "$dir/tr_run.$ex"
+		for i in $links; do
+		    test -e $i  &&  rm -f $i
+		    ln -s ../input.$ex/$i $i
+		done
+		ln -s ../$builddir/mitgcmuv mitgcmuv
+	    )
+	    runmodel $dir/tr_run.$ex && run=Y \
+		&& results=`testoutput $dir tr_run.$ex "."$ex`
+	    fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
+	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
+	    fres="$fres.$ex"
+	    echo
+	    echo "$fres" >> $SUMMARY
+	    echo "fresults='$fres'" > $CDIR"/summary.txt"
+	    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	    echo "tdir='$dir'" >> $CDIR"/summary.txt"
+	done
+	COMMAND=$OLD_COMMAND
+    else
+	fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
+	fres=$fres"$results   $dir"
+	echo
+	echo "$fres" >> $SUMMARY
+	echo "fresults='$fres'" > $CDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
+	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+    fi
     
-    (
-	cd $DRESULTS
-	tar -cf $NDIR".tar" $DRESULTS"_"$NDIR > /dev/null 2>&1
-	gzip $NDIR".tar"
-    )
+    echo "-------------------------------------------------------------------------------"
     
-    if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
-	echo "No mail sent"
-    else
-	$MPACK -s MITgcm-test -m 1000000 $DRESULTS"/"$NDIR".tar.gz" $ADDRESSES
+done
+
+printf "Start time:  " >> $SUMMARY
+echo $start_date >> $SUMMARY
+printf "End time:    " >> $SUMMARY
+date >> $SUMMARY
+
+#  If addresses were supplied and mpack built successfully, then try
+#  to send email using mpack.
+if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
+    echo "No results email was sent."
+else
+    if test "x$HAVE_MPACK" = xt ; then
+	tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
+	    && gzip $DRESULTS".tar" \
+	    && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
 	RETVAL=$?
 	if test "x$RETVAL" != x0 ; then
-	    echo "Warning: \"$MPACK\" failed -- please contact <edhill@mitgcm.org>"
+	    echo
+	    echo "Warning: The tar, gzip, & mpack step failed.  Please send email"
+	    echo "  to <MITgcm-support@mitgcm.org> for help.  You may copy the "
+	    echo "  summary of results from the directory \"$DRESULTS\"."
+	    echo
 	else
-	    rm -f $DRESULTS"/"$NDIR".tar*"
+	    echo
+	    echo "An email containing results was sent to the following addresses:"
+	    echo "  \"$ADDRESSES\""
+	    echo
 	fi
+	test -f $DRESULTS".tar"  &&  rm -f $DRESULTS".tar"
+	test -f $DRESULTS".tar.gz"  &&  rm -f $DRESULTS".tar.gz"
     fi
+fi
 
-    echo "-------------------------------------------------------------------------------"
-    
-    NDIR=$(( $NDIR + 1 ))
-    
-done
+# rm -f tmp_cmpnum.f a.out
+rm -f tmp_cmpnum.c tmp_cmpnum
 
-rm tmp_cmpnum.f a.out
+if test "x$CLEANUP" != xt ; then
+    cat $SUMMARY
+    if test -e tr_out.txt ; then
+        mv tr_out.txt tr_out.txt.old
+    fi
+    cat $SUMMARY > tr_out.txt
+fi
 
-cat $SUMMARY
+if test "x$DELDIR" = xt ; then
+    rm -rf $DRESULTS
+fi