Annotation of /MITgcm/verification/testreport

#! /usr/bin/env bash
#
#  $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.80 2006/06/08 21:39:35 jmc Exp $
#  $Name:  $
#

usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-help|-h)               print usage"
    echo "  (-mpi)                   use MPI input files"
    echo "  (-ieee|-noieee)          if possible, use IEEE compiler flags"
    echo "                             (DEF=\"-ieee\")"
    echo "  (-optfile=|-of=)STRING   list of optfiles to use"
    echo "  (-a|-addr) STRING        list of email recipients"
    echo "                             (DEF=\"edhill@mitgcm.org\")"
    echo "  (-t|-tdir) STRING        list of test dirs to use"
    echo "                             (DEF=\"\" which builds all)"
    echo "  (-b|-bash) STRING        preferred location of a \"bash\" or"
    echo "                             Bourne-compatible \"sh\" shell"
    echo "                             (DEF=\"\" for \"bash\")"
    echo "  (-adm|-ad)               perform an adjoint run"
    echo "  (-command) STRING        command to run"
    echo "                             (DEF=\"make output.txt\")"
    echo "  (-m|-make) STRING        command to use for \"make\""
    echo "                             (DEF=\"make\")"
    echo "  (-odir) STRING           used to build output directory name"
    echo "                             (DEF=\"hostname\")"
    echo "  (-ptracers|-ptr) STRING  specify which ptracers to test"
    echo "                             (DEF=\"1 2 3 4 5\")"
    echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
    echo "  (-clean)                 *ONLY* run \"make CLEAN\""
    echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
    echo "  (-nogenmake|-ng)         skip the genmake stage"
    echo "  (-noclean|-nc)           skip the \"make clean\" stage"
    echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
    echo "  (-deldir|-dd)            on success, delete the output directory"
    echo
    echo "and where STRING can be a whitespace-delimited list"
    echo "such as:"
    echo
    echo "  -t 'exp0 exp2 exp3' "
    echo "  -addr='abc@123.com testing@home.org'"
    echo
    echo "provided that the expression is properly quoted within the current"
    echo "shell (note the use of single quotes to protect white space)."
    echo
    exit 1
}

#  build the mpack utility
build_mpack()
{
    printf "building the mpack utility...  "
    if test ! -x "$MPACKDIR/mpack" ; then
	if test ! -d $MPACKDIR ; then
	    echo
	    echo "Error: can't find \"$MPACKDIR\""
	    echo "  are you sure this program is being run in the correct "
	    echo "  (that is, \"MITGCM_ROOT\verification\") directory?"
	    echo
	    HAVE_MPACK=f
	fi
	printf "building mpack...  "
	if test "x$CC" = x ; then
	    export CC=cc
	fi
	( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
	RETVAL=$?
	if test "x$RETVAL" != x0 ; then
	    echo
	    echo "Error building the mpack tools at: $MPACK_DIR"
	    echo
	    HAVE_MPACK=f
	else
	    rm -f tr_build_mpack.out
	    HAVE_MPACK=t
	fi
    else
	HAVE_MPACK=t
    fi
    echo "OK"
}

testoutput_for_prop()
{
    # testoutput_for_prop dir s1 label subdir extension
    #
    #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
    #  using search strings s1 and text label 

    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
    fi
    if [ -r $1/$4/output.txt ]; then
	grep "$2" $1/$4/output.txt | sed 's/.*=//' | cat -n > tmp1.txt
	lncntA=`wc -l tmp1.txt | awk '{print $1}' `
	if [ $lncntA -lt 3 ]; then
	    if [ $verbose -gt 0 ]; then
		echo Not enough lines of output when searching for "$2" 1>&2
	    fi
	    return 99
	fi
    else
	echo testoutput_for_prop: output.txt from model run was not readable 1>&2
	return 99
    fi
    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: grep "$2" $1/results/output.txt$5 1>&2
    fi
    grep "$2" $1/results/output.txt$5 | sed 's/.*=//' | cat -n > tmp2.txt
    lncntB=`wc -l tmp2.txt | awk '{print $1}' `
    if [ $lncntB -lt 3 ]; then
	if [ $verbose -gt 0 ]; then
	    echo Not enough lines of output when searching for "$2" 1>&2
	fi
	return 99
    fi
    if [ $lncntA -ne $lncntB ]; then
        if [ $verbose -gt 0 ]; then
            echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
        fi
        return 99
    fi
    has_nan=`cat tmp1.txt | grep -i nan | wc -l`
    if [ $has_nan -gt 0  ] ; then
	echo testoutput_for_prop: output.txt contains $has_nan NaN values  1>&2
	return 99
    fi
    has_inf=`cat tmp1.txt | grep -i inf | wc -l`
    if [ $has_inf -gt 0  ] ; then
	echo testoutput_for_prop: output.txt contains $has_inf Inf values  1>&2
	return 99
    fi
    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
    fi
    join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: compare_lines 1>&2
    fi
    if [ $verbose -gt 1 ]; then
	cat tmp3.txt 1>&2
    fi
    echo "-1" >> tmp3.txt
    # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
    cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
    digits_of_similarity=`./tmp_cmpnum < tmp4.txt`
    if [ $digits_of_similarity -eq 99 ]; then
	if [ $verbose -gt 0 ]; then
	    echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
	fi
	digits_of_similarity=99
    else
	if [ $verbose -gt 0 ]; then
	    echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
	fi
    fi
    rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
    
    return $digits_of_similarity
}

dashnum()
{
    # dashnum n1 n2 n3 ...
    #
    #  print numbers using %3i format or "--" if number = 99

    for num in $@ ; do
	if [ $num = 99 ]; then
	    printf ' --'
	else
	    printf '%3i' $num
	fi
    done
}

testoutput_ad()
{
    grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt
    grep $3 $1/$2/output_adm.txt | awk '{print NR " " $5}' > t15.txt
    grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt
    grep $3 $1/$2/output_adm.txt | awk '{print NR " " $6}' > t16.txt
    join t05.txt t15.txt > t5.txt
    join t06.txt t16.txt > t6.txt
    echo "-1" >> t5.txt
    echo "-1" >> t6.txt
    digits_5=`./tmp_cmpnum < t5.txt`
    digits_6=`./tmp_cmpnum < t6.txt`
    dashnum $digits_5 $digits_6
    rm -f t[01][56].txt t[56].txt
}

check_for_add_mon_output()
{
    # Check for additional types of monitor output
    if test "x$1" = x ; then
	return
    fi

    for ii in $PTRACERS_NUM ; do
	eval "HAVE_PTR0"$ii"=f"
    done

    ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
    ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
    for ii in $PTRACERS_NUM ; do
	for jj in $ptr_add ; do
	    name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
	    tst=`grep $name $1 | wc -l | awk '{print $1}'`
	    if test ! "x$tst" = x0 ; then
		eval "HAVE_PTR0"$ii"=t"
	    fi
	done
	#  eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
    done
}

testoutput()
{
    # testoutput directory subdir extension
    #
    #  test output in "directory"
    if test "x$ADM" = x ; then
	if [ $debug -gt 0 ]; then
	    echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
	fi
	testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2 $3; cg2dres=$?
	if [ $debug -gt 0 ]; then
	    echo testoutput: cg2dres=$cg2dres 1>&2
	fi
	testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2 $3; tmin=$?
	testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2 $3; tmax=$?
	testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2 $3; tmean=$?
	testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2 $3; tsd=$?
	testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2 $3; smin=$?
	testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2 $3; smax=$?
	testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2 $3; smean=$?
	testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2 $3; ssd=$?
	testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2 $3; umin=$?
	testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2 $3; umax=$?
	testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2 $3; umean=$?
	testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2 $3; usd=$?
	testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2 $3; vmin=$?
	testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$?
	testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$?
	testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$?

	#  This is for PTRACERS
	for ii in $PTRACERS_NUM ; do
	    eval `echo "p0"$ii"_min=99"`
	    eval `echo "p0"$ii"_max=99"`
	    eval `echo "p0"$ii"_mean=99"`
	    eval `echo "p0"$ii"_sd=99"`
	    tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
	    #echo 'tst = '$tst
	    if test "x$tst" = xt ; then
		a="trcstat_ptracer0"
		testoutput_for_prop $1 "$a"$ii"_min"  "p0"$ii"_min"  $2 $3
		RETVAL=$? ; eval `echo "p0"$ii"_min="$RETVAL`
		testoutput_for_prop $1 "$a"$ii"_max"  "p0"$ii"_max"  $2 $3
		RETVAL=$? ; eval `echo "p0"$ii"_max="$RETVAL`
		testoutput_for_prop $1 "$a"$ii"_mean" "p0"$ii"_mean" $2 $3
		RETVAL=$? ; eval `echo "p0"$ii"_mean="$RETVAL`
		testoutput_for_prop $1 "$a"$ii"_sd"   "p0"$ii"_sd"   $2 $3
		RETVAL=$? ; eval `echo "p0"$ii"_sd="$RETVAL`
	    fi
	done

	allargs="$cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd"
	allargs="$allargs $umin $umax $umean $usd $vmin $vmax $vmean $vsd"
	allargs="$allargs $p01_min $p01_max $p01_mean $p01_sd"
	allargs="$allargs $p02_min $p02_max $p02_mean $p02_sd"
	allargs="$allargs $p03_min $p03_max $p03_mean $p03_sd"
	allargs="$allargs $p04_min $p04_max $p04_mean $p04_sd"
	allargs="$allargs $p05_min $p05_max $p05_mean $p05_sd"

	eval "dashnum $allargs"

    else
	testoutput_ad $1 $2 "precision_grdchk_result"
    fi
}

genmakemodel()
{
    # genmakemodel directory
    if test "x$NOGENMAKE" = xt ; then
	echo "genmake skipped!"
    else
	if test "x$BASH" = x ; then
	    GENMAKE2="../../../tools/genmake2"
	else
	    GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
	fi
	(
	    cd $1;
	    command="$GENMAKE2  -ds -m $MAKE"
	    if test "x$ADM" = x ; then
		command="$command --mods=../code"
	    else
		command="$command --mods=../code_ad"
	    fi
	    if test "x$OPTFILE" != xNONE ; then
		command="$command --optfile=$OPTFILE"
	    fi
	    if test "x$IEEE" != x ; then
		command="$command -ieee"
	    fi
	    if test "x$MPI" = xt ; then
		command="$command -mpi"
	    fi
	    printf 'genmake ... ' 1>&2
	    $command > make.log 2>&1
	    RETVAL=$?
	    #  Reduce the size of the testing emails!
	    head -100 Makefile > $CDIR/Makefile_head
	    if test "x$RETVAL" != x0 ; then
		tail make.log
		echo "genmakemodel: genmake failed" 1>&2
		cp genmake_* make.log $CDIR
		return 1
	    else
		echo "successful" 1>&2
	    fi
	)
    fi
}

makeclean()
{
    # makeclean directory
    if test "x$NOCLEAN" = xt ; then
	echo "make CLEAN skipped!"
    else
	(
	    cd $1;
	    if test -e output.txt ; then
		rm -f output.txt
	    fi
	    printf 'make CLEAN ... ' 2>&1
	    if test -r Makefile ; then
		$MAKE CLEAN >> make.log 2>&1
		RETVAL=$?
		if test "x$RETVAL" != x0 ; then
		    tail make.log
		    echo "makeclean: \"make CLEAN\" failed" 1>&2
		    cp make.log $CDIR"/make.log"
		    return 1
		fi
	    fi
	    echo successful 1>&2
	    exit 0
	)
    fi
}

postclean()
{
    # postclean directory
    if test "x$POSTCLEAN" = xt ; then
	(
	    cd $1;
	    if test -r Makefile ; then
		$MAKE CLEAN >> /dev/null 2>&1
	    fi
	    exit 0
	)
    fi
}

makedependmodel()
{
    # makedependmodel directory
    if test "x$NODEPEND" = xt ; then
	echo "make depend skipped!"
    else
	(
	    cd $1;
	    printf 'make depend ... ' 1>&2
	    $MAKE depend >> make.log 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		tail make.log
		echo "makedependmodel: make depend failed" 1>&2
		cp make.log $CDIR"/make.log"
		return 1
	    else
		echo successful 1>&2
	    fi
	)
    fi
}

makemodel()
{
    # makemodel directory
    (
	cd $1;
	if test -r Makefile ; then
	    printf 'make ... ' 1>&2
	    if test "x$ADM" = x ; then
		if test "x$JOBS" = x ; then
		    $MAKE >> make.log 2>&1
		else
		    $MAKE -j $JOBS >> make.log 2>&1
		fi
	    else
		$MAKE adall >> make.log 2>&1
	    fi
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		tail make.log
		echo failed 1>&2
		cp make.log $CDIR"/make.log"
		return 1
	    else
		echo successful 1>&2
	    fi
	fi
    )
}

symlink_mpifiles()
{
    # Put special links so that MPI specific files are used
    # This MUST be invoked between makeclean and makelinks because
    # the Makefile will link to non-mpi files by default

    dir=$1
    code_dir=$2
    BUILD_DIR=$dir/$3
    CODE_DIR=$dir/$code_dir
    
    # These are files that should replace their counter-part when using -mpi
    MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`

    #  Is this an MPI run?
    if test "x$MPI" = xt ; then
	# YES: We symbolically link these files to the build
	# dir so long as there is no real file in place
	for ii in $MPI_FILES ; do
	    i=`echo $ii | sed 's:^\./::'`
	    name=`echo $i | sed 's:_mpi::' `
	    cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		if ! test -f $BUILD_DIR/$i ; then
	            #echo Linking $name to $i
		    (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
		fi
	    fi
	done
    else
        # NO: We undo any _mpi symbolically linked files
	for ii in $MPI_FILES ; do
	    i=`echo $ii | sed 's:^\./::'`
	    name=`echo $i | sed 's:_mpi::' `
	    if test -L $BUILD_DIR/$name ; then
		cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
		RETVAL=$?
		if test "x$RETVAL" = x0 ; then
		    #echo Un-linking $name from $linktarg
		    rm -f $BUILD_DIR/$name
		fi
	    fi
	done
    fi
    
}

linkdata()
{
    # linkdata flag
    #
    # symbolically link data files to run directory
    if test "x$1" = x1 ; then
	(
	    cd $2
	    if test "x$ADM" = x ; then
		files=`( cd ../input ; ls -1 | grep -v CVS )`
		for i in $files ; do
		    if test ! -d "../input/"$i -a ! -f $i ; then
			ln -sf "../input/"$i $i
		    fi
		done
	    else
		files=`( cd ../input_ad ; ls -1 | grep -v CVS )`
		for i in $files ; do
		    if test ! -d "../input_ad/"$i ; then
			ln -sf "../input_ad/"$i $i
		    fi
		done
		files=`( cd ../input ; ls -1 | grep -v CVS )`
		for i in $files ; do
		    if test ! -d "../input/"$i ; then
		     if test ! -r $i  ; then
			ln -sf "../input/"$i $i
		     fi
		    fi
		done
	    fi
	)
    fi
}

runmodel()
{
    # runmodel directory
    #
    #  runs "$COMMAND in "directory" 
    #  (where "$COMMAND" is relative to "directory")
    (
	cd $1
	printf 'runmodel in %s ...' $1 1>&2
	# make output.txt
	echo
	rm -f run.log
        if test $OUTPUTFILE -ot $EXECUTABLE ; then
	   ( eval $COMMAND ) > run.log 2>&1
	   RETVAL=$?
	else
           echo " $OUTPUTFILE is up to date " > run.log 2>&1
	   RETVAL=0
	fi
	# echo "COMMAND='$COMMAND'"
	# echo "pwd='"`pwd`"'"
	if test "x$RETVAL" = x0 ; then
	    tail run.log
	    echo successful 1>&2
	    # === Reduce the size of the testing emails!
	    # if test "x$ADM" = x ; then
	    #   cp output.txt $CDIR"/output.txt"
	    # else
	    #   cp output_adm.txt $CDIR"/output_adm.txt"
	    # fi
            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
	    return 0
	else
	    tail run.log
	    echo failed 1>&2
	    cp run.log $CDIR"/run.log"
            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
	    return 1
	fi
    )
}

createcodelet()
{
    # create codelet for comparing model output

    printf "creating the comparison code...  "
    cat > tmp_cmpnum.c <<EOF
#include <stdio.h>
#include <math.h>
int main( int argc, char** argv )  {
  int linnum,best,lncnt;
  double a,b,abave,relerr;
  best = -22;
  lncnt = 0;
  while( 1 & (lncnt+=1) < 999 )  {
    scanf("%d", &linnum);
    if (linnum == -1)  break;
    scanf("%lf", &a);  scanf("%lf", &b);
    abave = 0.5*(fabs(a)+fabs(b));
    if (abave > 0.0) {
      relerr=fabs(a-b)/abave;
      if (relerr > 0.0) { linnum = (int)rint(log10(relerr)); }
      else { linnum = -16 ; }
      best = (best > linnum) ? best : linnum;
    }
  }
  if (lncnt == 999) best=-29;
  printf("%d\n", -best);
  return 0;
}
EOF
    $CC -o tmp_cmpnum tmp_cmpnum.c -lm

    if [ -x ./tmp_cmpnum ]; then
	echo "OK"
	return 0
    else
	echo
	echo "ERROR: failed to compile comparison code -- please specify" 
	echo "  a C compiler using the CC environment variable."
	exit 1
    fi
}

formatresults()
{
    # formatresults expt genmake depend make run results*

    nm=$1
    printf '%s %s %s %s' $2 $3 $4 $5
    shift; shift; shift; shift; shift;
    printf '%3s' $@
    
    if [ $1 = '--' ]; then
	printf ' N/O '
    else
	if [ $1 -gt 12 ]; then
	    printf ' pass'
	else
	    printf ' FAIL'
	fi
    fi
    printf '  %s' $nm
    printf '\n'
    
}

show_help()
{
    cat - << EOF
$0 [-help] [-quick] [-verbose] dir1 [dir2] [...]

 -help|-h      Show this help message
 -quiet     Reduce the amount of output
 -verbose   Produce copious amounts of output
 -debug     Produce even more output which will mean nothing to most
 -force     Do "make CLEAN" before compiling. This forces a complete rebuild.
 -clean     Do "make CLEAN" after compiling and testing.
 -cleanup   Aggresively removes all model output, executables and object files
            and then exits. Use with care.

Normal usage:
 $0 *       Configure, compile, run and analyze in all experiment directories
EOF
}

scandirs()
{
    if [ $# -eq 1 ]; then
 	for arg in * ; do
 	    test -d $arg/$1 && echo $arg
 	done
    else
 	echo $*
    fi
}


###############################################################################


#  Default properties
debug=0
verbose=1
clean=0
expts=''
# ieee=1

IEEE=true
if test "x$MITGCM_IEEE" != x ; then
    IEEE=$MITGCM_IEEE
fi


CLEANUP=f
QUICK=f
NOGENMAKE=f
NOCLEAN=f
NODEPEND=f
POSTCLEAN=f

BASH=
OPTFILE=NONE
ADDRESSES=
TESTDIRS=
MPACKDIR="../tools/mpack-1.6"
HAVE_MPACK=
MPACK="$MPACKDIR/mpack"
COMMAND=
if test "x$MAKE" = x ; then
    MAKE=make
fi
if test "x$CC" = x ; then
    CC=cc
fi
JOBS=
MPI=f
OUTDIR=
DELDIR=

ADM=

# Additional monitor types
PTRACERS_NUM="1 2 3 4 5"

printf "parsing options...  "

ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
    
    case $ac_option in
	
	-help | --help | -h | --h)
	    usage ;;
	
	-optfile | --optfile | -of | --of)
            ac_prev=OPTFILE ;;
        -optfile=* | --optfile=* | -of=* | --of=*)
            OPTFILE=$ac_optarg ;;
	
	-addr | --addr | -a | --a)
            ac_prev=ADDRESSES ;;
        -addr=* | --addr=*)
            ADDRESSES=$ac_optarg ;;

	-tdir | --tdir | -t | --t)
            ac_prev=TESTDIRS ;;
        -tdir=* | --tdir=*)
            TESTDIRS=$ac_optarg ;;

	-bash | --bash | -b | --b)
            ac_prev=BASH ;;
        -bash=* | --bash=*)
            BASH=$ac_optarg ;;

	-command | --command | -c | --c)
	    ac_prev=COMMAND ;;
	-command=* | --command=*)
	    COMMAND=$ac_optarg ;;

	-make | --make | -m | --m)
	    ac_prev=MAKE ;;
	-make=* | --make=*)
	    MAKE=$ac_optarg ;;

        -odir | --odir)
            ac_prev=OUTDIR ;;
        -odir=* | --odir=*)
            OUTDIR=$ac_optarg ;;

	-ptracers | --ptracers | -ptr | --ptr)
	    ac_prev=PTRACERS_NUM ;;
	-ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
	    PTRACERS_NUM=$ac_optarg ;;

	-j) ac_prev=JOBS ;;
	-j=*) JOBS=$ac_optarg ;;

	-clean | --clean)
	    CLEANUP=t ; DELDIR=t ;;

	-quick | --quick | -q | --q)
	    QUICK=t ;;
	-nogenmake | --nogenmake | -ng | --ng)
	    NOGENMAKE=t ;;
	-noclean | --noclean | -nc | --nc)
	    NOCLEAN=t ;;
	-nodepend | --nodepend | -nd | --nd)
	    NODEPEND=t ;;

	-postclean | --postclean | -pc | --pc)
	    POSTCLEAN=t ;;

	-mpi) MPI=t ;;

	-adm | -ad) ADM=t ;;

	-ieee) IEEE=true ;;
	-noieee) IEEE= ;;

	-verbose) verbose=2 ;;
	-debug) debug=1 ;;
	-quiet) verbose=0 ;;

	-deldir | -dd) DELDIR=t ;;

        -*)
            echo "Error: unrecognized option: "$ac_option
            usage
            ;;
        
        *)
            echo "Error: unrecognized argument: "$ac_option
            usage
            ;;
        
     esac
     
done

if test "x$QUICK" = xt ; then
    NOGENMAKE=t
    NOCLEAN=t
    NODEPEND=t
fi

if test "x$TESTDIRS" = x ; then
    if test "x$ADM" = xt ; then
	TESTDIRS=`scandirs results_ad`
    else
	TESTDIRS=`scandirs results`
    fi
fi

if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
    OPTFILE=$MITGCM_OF
fi

if test "x$ADM" = xt ; then
    EXECUTABLE="mitgcmuv_ad"
    OUTPUTFILE="output_adm.txt"
else
    EXECUTABLE="mitgcmuv"
    OUTPUTFILE="output.txt"
fi

if test "x$ADM" = xt -a "x$COMMAND" = x ; then
    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
fi

if test "x$COMMAND" = x ; then
    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
fi

#echo "OK"
echo "OK (COMMAND= $COMMAND )"

#  create the FORTRAN comparison code
createcodelet

#  build the mpack utility
if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
    echo "skipping mpack build"
else
    build_mpack
fi

#  Create a uniquely named directory to store results
MACH=`hostname`
UNAMEA=`uname -a`
DATE=`date +%Y%m%d`
BASE="tr_"$MACH"_"$DATE"_"
if test "x$OUTDIR" != x ; then
    BASE="tr_"$OUTDIR"_"$DATE"_"
fi
DNUM=0
DRESULTS="$BASE$DNUM"
while test -e $DRESULTS ; do
    DNUM=$(( $DNUM + 1 ))
    DRESULTS="$BASE$DNUM"
done
mkdir $DRESULTS
RETVAL=$?
if test "x$RETVAL" != x0 ; then
    echo "ERROR: Can't create results directory \"./$DRESULTS\""
    exit 1
fi
SUMMARY="$DRESULTS/summary.txt"
printf "Start time:  " >> $SUMMARY
start_date=`date`
echo $start_date > $SUMMARY

of_path=
if test "x$OPTFILE" != xNONE ; then
    if test -r $OPTFILE ; then
	# get the path
	path=${OPTFILE%/*}
	if test "x$path" = x ; then
	    of_path=`pwd`
	else
	    of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
	fi
	file=${OPTFILE##*/}
	OPTFILE=$of_path/$file
	cp $OPTFILE $DRESULTS
	echo >> $SUMMARY
	echo "  OPTFILE=$OPTFILE" >> $SUMMARY
    else
	echo | tee $SUMMARY
	echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
	exit 1
    fi
else
    echo >> $SUMMARY
    echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
    echo "   so the genmake default will be used." >> $SUMMARY
fi
echo
echo >> $SUMMARY
if test "x$ADM" = x ; then
    line_0="            ----T-----  ----S-----  ----U-----  ----V-----"
    line_1="G D M    c        m  s        m  s        m  s        m  s"
    line_2="E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
    line_3="N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
    line_4="2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  ."
    for ii in $PTRACERS_NUM ; do
	#  tst=`eval 'echo $HAVE_PTR0'$ii`
	#  if test "x$tst" = xt ; then
	line_0="$line_0  --PTR 0"$ii"--"
	line_1="$line_1        m  s"
	line_2="$line_2  m  m  e  ."
	line_3="$line_3  i  a  a  d"
	line_4="$line_4  n  x  n  ."
	#  fi
    done
    echo "$line_0" | tee -a $SUMMARY
    echo "$line_1" | tee -a $SUMMARY
    echo "$line_2" | tee -a $SUMMARY
    echo "$line_3" | tee -a $SUMMARY
    echo "$line_4" | tee -a $SUMMARY
    echo " "       | tee -a $SUMMARY
else
    echo "ADJOINT=true" >> $SUMMARY
    echo >> $SUMMARY
    cat << EOF | tee -a $SUMMARY
G D M    C  G
E p a R  o  r
N n k u  s  a
2 d e n  t  d

EOF
fi

#  ...and each test directory...
for dir in $TESTDIRS ; do
    
    #  Cleanup only!
    if test "x$CLEANUP" = xt ; then
	if test -r $dir/build/Makefile ; then
	    ( cd $dir/build ; make CLEAN )
	fi
	if test -r $dir/input/Makefile ; then
	    ( cd $dir/input ; make CLEAN )
	fi
	(
	    cd $dir
	    rm -rf tr_run.*
	)
	continue
    fi

    #  Verify that the testdir exists and contains previous 
    #  results in the correct location--or skip this directory!
    fout=
    if test "x$ADM" = x ; then
	fout=$dir"/results/output.txt"
    else
	fout=$dir"/results_ad/output_adm.txt"
    fi
    if test ! -r $fout ; then
	echo "can't read \"$fout\" -- skipping $dir"
	continue
    fi
    if test "x$ADM" = x ; then
	check_for_add_mon_output  $fout
    fi

    # Check for additional types of monitor output

    builddir="input"
    rundir="input"
    use_seperate_build=0
    if test -d $dir/build -a -r $dir/build ; then
	builddir="build"
	rundir="build"
	use_seperate_build=1
	linkdata $use_seperate_build $dir/$rundir
    fi
    
    #  Check whether there are "extra runs" for this testdir
    extra_runs=
    if test "x$ADM" = x -a "x$use_seperate_build" = x1 ; then
	ex_run_dirs=`( cd $dir ; echo input.* )`
	echo "ex_run_dirs='$ex_run_dirs'"
	for exd in $ex_run_dirs ; do
	    name=`echo $exd | sed -e 's/input.//g'`
	    outf="$dir/results/output.txt.$name"
	    if test -f $outf -a -r $outf ; then
		extra_runs="$extra_runs $name"
	    fi
	done
    fi

    if test "x$ADM" = x ; then
	code_dir=code
	CODE_DIR=$dir/code
    else
	code_dir=code_ad
	CODE_DIR=$dir/code_ad
    fi
    BUILD_DIR=$dir/$builddir

    if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
	echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
	continue
    fi

    echo "-------------------------------------------------------------------------------"
    echo
    echo "Experiment:  $dir"
    echo
    unset genmake makedepend make run
    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'

    #  Create an output dir for each OPTFILE/tdir combination
    rel_CDIR=$DRESULTS"/"$dir
    mkdir $rel_CDIR
    CDIR=`pwd`"/$rel_CDIR"
    
    if test "x$CLEANUP" = xt ; then
	makeclean $dir/$builddir
    else
	genmakemodel $dir/$builddir && genmake=Y \
	    && makeclean $dir/$builddir \
            && symlink_mpifiles $dir $code_dir $builddir \
	    && makedependmodel $dir/$builddir && makedepend=Y \
	    && makemodel $dir/$builddir && make=Y \
	    && linkdata $use_seperate_build $dir/$rundir \
	    && runmodel $dir/$rundir && run=Y \
	    && results=`testoutput $dir $rundir`
    fi
    
    echo
    if test "x$ADM" = x ; then
	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
	echo
	echo "$fres" >> $SUMMARY
	echo "fresults='$fres'" > $CDIR"/summary.txt"
	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
	echo "tdir='$dir'" >> $CDIR"/summary.txt"

	for ex in $extra_runs ; do
	    unset run
	    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
	    #  Create an output dir for each OPTFILE/tdir.ex combination
	    rel_CDIR=$DRESULTS"/"$dir"."$ex
	    mkdir $rel_CDIR
	    CDIR=`pwd`"/$rel_CDIR"
	    test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
	    for ldir in input.$ex input ; do
		(
		    cd "$dir/$ldir" > /dev/null 2>&1
		    ls -1 2>/dev/null \
			| sed -e 's|^CVS$||g' | sed -e 's|^output.txt$||g'
		) > tr_exrun_links
		(
		    cd "$dir/tr_run.$ex"
		    cat ../../tr_exrun_links | while read i ; do
			if test ! "x$i" = x ; then
			    test ! -r $i  &&  ln -s "../"$ldir"/"$i $i
			fi
		    done
		)
		test -e tr_exrun_links  &&  rm -f tr_exrun_links
	    done
	    ldir=build
	    (
		cd $dir/tr_run.$ex
		test ! -e mitgcmuv  &&  ln -s "../"$ldir"/"mitgcmuv .
	    )
	    runmodel $dir/tr_run.$ex && run=Y \
		&& results=`testoutput $dir tr_run.$ex "."$ex`
	    fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
	    fres="$fres.$ex"
	    echo
	    echo "$fres" >> $SUMMARY
	    echo "fresults='$fres'" > $CDIR"/summary.txt"
	    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
	    echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
	done
    else
	fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
	fres=$fres"$results   $dir"
	echo
	echo "$fres" >> $SUMMARY
	echo "fresults='$fres'" > $CDIR"/summary.txt"
	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
	echo "tdir='$dir'" >> $CDIR"/summary.txt"
        grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
			   >> $CDIR"/summary.txt"
    fi

    postclean $dir/$builddir
    
    echo "-------------------------------------------------------------------------------"
    
done

printf "Start time:  " >> $SUMMARY
echo $start_date >> $SUMMARY
printf "End time:    " >> $SUMMARY
date >> $SUMMARY

#  If addresses were supplied and mpack built successfully, then try
#  to send email using mpack.
if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
    echo "No results email was sent."
else
    if test "x$HAVE_MPACK" = xt ; then
	tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
	    && gzip $DRESULTS".tar" \
	    && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
	RETVAL=$?
	if test "x$RETVAL" != x0 ; then
	    echo
	    echo "Warning: The tar, gzip, & mpack step failed.  Please send email"
	    echo "  to <MITgcm-support@mitgcm.org> for help.  You may copy the "
	    echo "  summary of results from the directory \"$DRESULTS\"."
	    echo
	else
	    echo
	    echo "An email containing results was sent to the following addresses:"
	    echo "  \"$ADDRESSES\""
	    echo
	fi
	test -f $DRESULTS".tar"  &&  rm -f $DRESULTS".tar"
	test -f $DRESULTS".tar.gz"  &&  rm -f $DRESULTS".tar.gz"
    fi
fi

# rm -f tmp_cmpnum.f a.out
rm -f tmp_cmpnum.c tmp_cmpnum

if test "x$CLEANUP" != xt ; then
    cat $SUMMARY | sed 's/ -- -- -- --//g'
    if test -e tr_out.txt ; then
        mv tr_out.txt tr_out.txt.old
    fi
    cat $SUMMARY | sed 's/ -- -- -- --//g' > tr_out.txt
fi

if test "x$DELDIR" = xt ; then
    rm -rf $DRESULTS
fi
