Annotation of /MITgcm/verification/testreport

#!/bin/bash
#
#  $Header: /u/u3/gcmpack/MITgcm/verification/testreport,v 1.9.2.9 2003/10/04 14:21:58 edhill Exp $
#

usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-help|-h)               print usage"
    echo "  (-mpi)                   use MPI input files"
    echo "  (-ieee|-noieee)          if possible, use IEEE compiler flags"
    echo "  (-optfile=|-of=)STRING   list of optfiles to use"
    echo "  (-a|-addr)STRING         list of email recipients"
    echo "                             (DEF=\"edhill@mitgcm.org\")"
    echo "  (-t|-tdir)STRING         list of test dirs to use"
    echo "                             (DEF=\"\" which builds all)"
    echo "  (-b|-bash)STRING         location of \"bash\" executable"
    echo "                             (DEF=\"\" for \"/bin/bash\")"
    echo "  (-command)STRING         command to run"
    echo "                             (DEF=\"make output.txt\")"
    echo "  (-m|-make)STRING         command to use for \"make\""
    echo "                             (DEF=\"make\")"
    echo "  (-clean)                 *ONLY* run \"make CLEAN\""
    echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
    echo "  (-nogenmake|-ng)         skip the genmake stage"
    echo "  (-noclean|-nc)           skip the \"make clean\" stage"
    echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
    echo
    echo "and where STRING follows a whitespace-delimited format"
    echo "such as:"
    echo "  -t 'exp0 exp2 exp3' "
    echo "  -addr='abc@123.com testing@home.org'"
    echo
    exit 1
}

#  build the mpack utility
build_mpack()
{
    echo -n "building the mpack utility...  "
    if test ! -x "$MPACKDIR/mpack" ; then
	if test ! -d $MPACKDIR ; then
	    echo "Error: can't find \"$MPACKDIR\""
	    echo "  are you sure this program is being run in the correct "
	    echo "  (that is, \"MITGCM_ROOT\verification\") directory?"
	    exit 1
	fi
	echo -n "building mpack...  "
	( cd $MPACKDIR && ./configure && $MAKE ) > build_mpack.out 2>&1
	RETVAL=$?
	if test "x$RETVAL" != x0 ; then
	    echo
	    echo "Error building the mpack tools at: $MPACK_DIR"
	    exit 1
	fi
    fi
    echo "OK"
}

compare_lines()
{
    # use codelet to compare lines
    if [ $verbose -gt 1 ]; then
	cat tmp3.txt 1>&2
    fi
    return `./a.out < tmp3.txt`
}

testoutput_for_prop()
{
    # testoutput_for_prop dir s1 label subdir
    #
    #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
    #  using search strings s1 and text label 

    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
    fi
    if [ -r $1/$4/output.txt ]; then
	grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
	lncnt=`wc -l tmp1.txt | awk '{print $1}' `
	if [ $lncnt -lt 3 ]; then
	    if [ $verbose -gt 0 ]; then
		echo Not enough lines of output when searching for "$2" 1>&2
	    fi
	    return 99
	fi
    else
	echo testoutput_for_prop: output.txt from model run was not readable 1>&2
	return 99
    fi
    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
    fi
    grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
    lncnt=`wc -l tmp2.txt | awk '{print $1}' `
    if [ $lncnt -lt 3 ]; then
	if [ $verbose -gt 0 ]; then
	    echo Not enough lines of output when searching for "$2" 1>&2
	fi
	return 99
    fi
    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
    fi
    join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
    if [ $debug -gt 0 ]; then
	echo testoutput_for_prop: compare_lines 1>&2
    fi
    compare_lines
    digits_of_similarity=$?
    if [ $digits_of_similarity -eq 99 ]; then
	if [ $verbose -gt 0 ]; then
	    echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
	fi
	digits_of_similarity=99
    else
	if [ $verbose -gt 0 ]; then
	    echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
	fi
    fi
    rm tmp1.txt tmp2.txt tmp3.txt
    
    return $digits_of_similarity
}

dashnum()
{
    # dashnum n1 n2 n3 ...
    #
    #  print numbers using %3i format or "--" if number = 99

    for num in $@ ; do
	if [ $num = 99 ]; then
	    printf ' --'
	else
	    printf '%3i' $num
	fi
    done
}

testoutput()
{
    # testoutput diretory subdir
    #
    #  test output in "directory"

    if [ $debug -gt 0 ]; then
	echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
    fi
    testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
    if [ $debug -gt 0 ]; then
	echo testoutput: cg2dres=$cg2dres 1>&2
    fi
    
    testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
    testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
    testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
    testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
    testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
    testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
    testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
    testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
    testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
    testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
    testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
    testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
    testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
    testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
    testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
    testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
    
    dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
        $umin $umax $umean $usd $vmin $vmax $vmean $vsd
}

genmakemodel()
{
    # genmakemodel directory
    if test "x$NOGENMAKE" = xt ; then
	echo "genmake skipped!"
    else
	GENMAKE2="$BASH ../../../tools/genmake2"
	(
	    cd $1;
	    printf 'genmake ... ' 1>&2
	    command="$GENMAKE2  -ds -m $MAKE --mods=../code"
	    if test "x$OPTFILE" != xNONE ; then
		command="$command --optfile=$OPTFILE"
		# echo "  command=\"$command\""
	    fi
	    if test "x$IEEE" != x ; then
		command="$command -ieee"
	    fi
	    $command > make.log 2>&1
	    RETVAL=$?
	    for i in genmake_state genmake_optfile genmake_local Makefile ; do
		if test -r $i ; then
		    cp $i $CDIR
		fi
	    done
	    if test "x$RETVAL" != x0 ; then
		tail make.log
		echo "genmakemodel: genmake failed" 1>&2
		cp make.log $CDIR
		return 1
	    else
		echo "succesful" 1>&2
	    fi
	)
    fi
}

makeclean()
{
    # makeclean directory
    if test "x$NOCLEAN" = xt ; then
	echo "make CLEAN skipped!"
    else
	(
	    cd $1;
	    if test -e output.txt ; then
		rm -f output.txt
	    fi
	    printf 'make CLEAN ... ' 2>&1
	    if test -r Makefile ; then
		$MAKE CLEAN >> make.log 2>&1
		RETVAL=$?
		if test "x$RETVAL" != x0 ; then
		    tail make.log
		    echo "makeclean: \"make CLEAN\" failed" 1>&2
		    cp make.log $CDIR"/make.log"
		    return 1
		fi
	    fi
	    echo succesful 1>&2
	    exit 0
	)
    fi
}

makedependmodel()
{
    # makedependmodel directory
    if test "x$NODEPEND" = xt ; then
	echo "make depend skipped!"
    else
	(
	    cd $1;
	    printf 'make depend ... ' 1>&2
	    $MAKE depend >> make.log 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		tail make.log
		echo "makedependmodel: make depend failed" 1>&2
		cp make.log $CDIR"/make.log"
		return 1
	    else
		echo succesful 1>&2
	    fi
	)
    fi
}

makemodel()
{
    # makemodel directory
    (
	cd $1;
	if test -r Makefile ; then
	    printf 'make ... ' 1>&2
	    $MAKE >> make.log 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		tail make.log
		echo failed 1>&2
		cp make.log $CDIR"/make.log"
		return 1
	    else
		echo succesful 1>&2
	    fi
	fi
    )
}

linkdata()
{
    # linkdata flag
    #
    # symbolically link data files to run directory
    if [ $1 -ne 0 ]; then
	( cd $2 ;  ln -sf ../input/* . )
    fi
}

runmodel()
{
    # runmodel directory
    #
    #  runs "$COMMAND" in "directory" 
    #  (where "$COMMAND" is relative to "directory")
    (
	cd $1
	printf 'runmodel: ' 1>&2
	# make output.txt
	$COMMAND
	RETVAL=$?
	if test "x$RETVAL" = x0 ; then
	    cp output.txt $CDIR"/output.txt"
	    return 0
	else
	    return 1
	fi
    )
}

createcodelet()
{
    # create codelet for comparing model output

    echo -n "creating the comparison code...  "
    cat > tmp_cmpnum.f <<EOFA
      program cmpnum
      implicit none
      real*8 a,b,diff
      integer linnum,best
      best=-16
  99  read(*,*,end=70,err=60) linnum,a,b
      diff=0.5*(abs(a)+abs(b))
c     print *,a,b,diff,abs(a-b)/diff
      if (diff.gt.1.e-12) then
        diff=abs(a-b)/diff
        if (diff.gt.0.) then
c         print *,int(log10(diff)),diff
          linnum=int(log10(diff))
          best=max(best,linnum)
        endif
      else
        if (best.eq.-16.and.diff.ne.0.) best=-22
      endif
      goto 99
  60  stop 'cmpnum: An error occured reading a,b'
  70  print *,-best
      end
EOFA

    f77 tmp_cmpnum.f
    if [ -x ./a.out ]; then
	echo "OK"
	return 0
    else
	echo
	echo "createcodelet: failed to compile codelet" 
	exit 1
    fi
}

formatresults()
{
    # formatresults expt genmake depend make run results*

    nm=$1
    printf '%s %s %s %s' $2 $3 $4 $5
    shift; shift; shift; shift; shift;
    printf '%3s' $@
    
    if [ $1 = '--' ]; then
	printf ' N/O '
    else
	if [ $1 -gt 12 ]; then
	    printf ' pass'
	else
	    printf ' FAIL'
	fi
    fi
    printf '  %s' $nm
    printf '\n'
    
}

show_help()
{
    cat - << EOF
$0 [-help] [-quick] [-verbose] dir1 [dir2] [...]

 -help|-h      Show this help message
 -quiet     Reduce the amount of output
 -verbose   Produce copious amounts of output
 -debug     Produce even more output which will mean nothing to most
 -force     Do "make CLEAN" before compiling. This forces a complete rebuild.
 -clean     Do "make CLEAN" after compiling and testing.
 -cleanup   Aggresively removes all model output, executables and object files
            and then exits. Use with care.

Normal usage:
 $0 *       Configure, compile, run and analyze in all experiment directories
EOF
}

scandirs()
{
    if [ $# -eq 0 ]; then
	for arg in * ; do
	    test -d $arg/input && echo $arg
	done
    else
	echo $*
    fi
}


###############################################################################
###############################################################################
###############################################################################


#  Default properties
debug=0
verbose=1
clean=0
expts=''
# ieee=1

IEEE=
if test "x$MITGCM_IEEE" != x ; then
    IEEE=$MITGCM_IEEE
fi


CLEANUP=f
QUICK=f
NOGENMAKE=f
NOCLEAN=f
NODEPEND=f

BASH=
OPTFILE=NONE
ADDRESSES=
TESTDIRS=
MPACKDIR="../tools/mpack-1.6"
MPACK="$MPACKDIR/mpack"
COMMAND="make output.txt"
MAKE=make
MPI=f

echo -n "parsing options...  "

ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
    
    case $ac_option in
	
	-help | --help | -h | --h)
	    usage ;;
	
	-optfile | --optfile | -of | --of)
            ac_prev=OPTFILE ;;
        -optfile=* | --optfile=* | -of=* | --of=*)
            OPTFILE=$ac_optarg ;;
	
	-addr | --addr | -a | --a)
            ac_prev=ADDRESSES ;;
        -addr=* | --addr=*)
            ADDRESSES=$ac_optarg ;;

	-tdir | --tdir | -t | --t)
            ac_prev=TESTDIRS ;;
        -tdir=* | --tdir=*)
            TESTDIRS=$ac_optarg ;;

	-bash | --bash | -b | --b)
            ac_prev=BASH ;;
        -bash=* | --bash=*)
            BASH=$ac_optarg ;;

	-command | --command | -c | --c)
	    ac_prev=COMMAND ;;
	-command=* | --command=*)
	    COMMAND=$ac_optarg ;;

	-make | --make | -m | --m)
	    ac_prev=MAKE ;;
	-make=* | --make=*)
	    MAKE=$ac_optarg ;;

	-clean | --clean)
	    CLEANUP=t ;;

	-quick | --quick | -q | --q)
	    QUICK=t ;;
	-nogenmake | --nogenmake | -ng | --ng)
	    NOGENMAKE=t ;;
	-noclean | --noclean | -nc | --nc)
	    NOCLEAN=t ;;
	-nodepend | --nodepend | -nd | --nd)
	    NODEPEND=t ;;

	-mpi) MPI=true ;;

	-ieee) IEEE=true ;;
	-noieee) IEEE= ;;

	-verbose) verbose=2 ;;
	-debug) debug=1 ;;
	-quiet) verbose=0 ;;

        -*)
            echo "Error: unrecognized option: "$ac_option
            usage
            ;;
        
        *)
            echo "Error: unrecognized argument: "$ac_option
            usage
            ;;
        
     esac
     
done

if test "x$QUICK" = xt ; then
    NOGENMAKE=t
    NOCLEAN=t
    NODEPEND=t
fi

if test "x$TESTDIRS" = x ; then
    TESTDIRS=`scandirs`
fi

if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
    OPTFILE=$MITGCM_OF
fi

echo "OK"

#  create the FORTRAN comparison code
createcodelet

#  build the mpack utility
build_mpack

#  Create a uniquely named directory to store results
MACH=`hostname`
UNAMEA=`uname -a`
DATE=`date +%Y%m%d`
BASE=$MACH"_"$DATE"_"
DNUM=0
DRESULTS="$BASE$DNUM"
while test -e $DRESULTS ; do
    DNUM=$(( $DNUM + 1 ))
    DRESULTS="$BASE$DNUM"
done
mkdir $DRESULTS
RETVAL=$?
if test "x$RETVAL" != x0 ; then
    echo "Error: can't create results directory \"./$DRESULTS\""
    exit 1
fi
SUMMARY="$DRESULTS/summary.txt"
date > $SUMMARY
cat << EOF >> $SUMMARY
                T           S           U           V
G D M    c        m  s        m  s        m  s        m  s
E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .

EOF

NDIR=0

if test "x$OPTFILE" != xNONE ; then
    if test -r $OPTFILE ; then
	OPTFILE=`pwd`"/$OPTFILE"
    fi
fi
echo
echo "OPTFILE=$OPTFILE" >> $SUMMARY
echo >> $SUMMARY

#  ...and each test directory...
for dir in $TESTDIRS ; do
    
    #  Cleanup only!
    if test "x$CLEANUP" = xt ; then
	if test -r $dir/build/Makefile ; then
	    ( cd $dir/build ; make CLEAN )
	fi
	if test -r $dir/input/Makefile ; then
	    ( cd $dir/input ; make CLEAN )
	fi
	continue
    fi

    #  Verify that the testdir exists and contains previous 
    #  results in the correct location--or skip this directory!
    if test ! -r $dir"/results/output.txt" ; then
	echo "can't read \"$dir/results/output.txt\" -- skipping $dir"
	continue
    fi

    echo "-------------------------------------------------------------------------------"
    echo
    echo "Experiment:  $dir"
    echo
    unset genmake makedepend make run
    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'

    if [ -r $dir/build ]; then
	seperatebuilddir=1
	builddir=build
	rundir=build
	( cd $dir/build; ln -sf ../input/* . )
    else
	seperatebuilddir=0
	builddir=input
	rundir=input
    fi
    
    CODE_DIR=$dir/code
    BUILD_DIR=$dir/$builddir
    MPI_FILES="CPP_EEOPTIONS.h_mpi SIZE.h_mpi"
    NOMPI_FILES="CPP_EEOPTIONS.h_nompi SIZE.h_nompi"

    #  Is this an MPI run?
    if test "x$MPI" = xt ; then
	FILES=$MPI_FILES
    else
	FILES=$NOMPI_FILES
    fi
    
    #  Check to see that we have the files
    have_files=t
    for i in $FILES ; do
	if test ! -r $CODE_DIR/$i ; then
	    echo "Warning: can't read file $CODE_DIR/$i"
	    have_files=f
	fi
    done
    if test "x$have_files" != xt -a "x$MPI" = xt ; then
	echo "Skipping $dir due to lack of input files (see above warning)"
	continue
    fi
    
    #  If we have the $FILES and they differ, copy the $FILES to $BUILD_DIR
    if test "x$have_files" = xt ; then
	for i in $FILES ; do
	    cmp $CODE_DIR/$i $BUILD_DIR/$i > /dev/null 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		cp $CODE_DIR/$i $BUILD_DIR/$i
	    fi
	done
    fi
    
    #  Create an output dir for each OPTFILE/tdir combination
    CDIR=$DRESULTS"/"$DRESULTS"_"$NDIR
    mkdir $CDIR
    CDIR=`pwd`"/$CDIR"
    
    if test "x$CLEANUP" = xt ; then
	makeclean $dir/$builddir
    else
	genmakemodel $dir/$builddir && genmake=Y \
	    && makeclean $dir/$builddir \
	    && makedependmodel $dir/$builddir && makedepend=Y \
	    && makemodel $dir/$builddir && make=Y \
	    && linkdata $seperatebuilddir $dir/$rundir \
	    && runmodel $dir/$builddir && run=Y \
	    && results=`testoutput $dir $rundir`
    fi
    
    echo
    formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
	${run:-N} $results
    echo
    formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
	${run:-N} $results >> $SUMMARY
    echo "fresults='" > $CDIR"/summary.txt"
    formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
	${run:-N} $results >> $CDIR"/summary.txt"
    echo "'" >> $CDIR"/summary.txt"
    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
    echo "tdir='$dir'" >> $CDIR"/summary.txt"
    
    (
	cd $DRESULTS
	tar -cf $NDIR".tar" $DRESULTS"_"$NDIR > /dev/null 2>&1
	gzip $NDIR".tar"
    )
    
    if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
	echo "No mail sent"
    else
	$MPACK -s MITgcm-test -m 1000000 $DRESULTS"/"$NDIR".tar.gz" $ADDRESSES
	RETVAL=$?
	if test "x$RETVAL" != x0 ; then
	    echo "Warning: \"$MPACK\" failed -- please contact <edhill@mitgcm.org>"
	else
	    rm -f $DRESULTS"/"$NDIR".tar*"
	fi
    fi

    echo "-------------------------------------------------------------------------------"
    
    NDIR=$(( $NDIR + 1 ))
    
done

rm tmp_cmpnum.f a.out

cat $SUMMARY

1	edhill	1.1	#!/bin/bash
2			#
3	edhill	1.9.2.10	# $Header: /u/u3/gcmpack/MITgcm/verification/testreport,v 1.9.2.9 2003/10/04 14:21:58 edhill Exp $
4	edhill	1.1	#
5
6			usage()
7			{
8			echo
9			echo "Usage: $0 [OPTIONS]"
10			echo
11			echo "where possible OPTIONS are:"
12			echo " (-help\|-h) print usage"
13	edhill	1.7	echo " (-mpi) use MPI input files"
14	edhill	1.9.2.9	echo " (-ieee\|-noieee) if possible, use IEEE compiler flags"
15	edhill	1.6	echo " (-optfile=\|-of=)STRING list of optfiles to use"
16	edhill	1.1	echo " (-a\|-addr)STRING list of email recipients"
17	edhill	1.6	echo " (DEF=\"edhill@mitgcm.org\")"
18			echo " (-t\|-tdir)STRING list of test dirs to use"
19			echo " (DEF=\"\" which builds all)"
20			echo " (-b\|-bash)STRING location of \"bash\" executable"
21			echo " (DEF=\"\" for \"/bin/bash\")"
22	edhill	1.9.2.8	echo " (-command)STRING command to run"
23	edhill	1.6	echo " (DEF=\"make output.txt\")"
24	edhill	1.8	echo " (-m\|-make)STRING command to use for \"make\""
25			echo " (DEF=\"make\")"
26	edhill	1.9.2.8	echo " (-clean) ONLY run \"make CLEAN\""
27	edhill	1.9.2.1	echo " (-quick\|-q) same as \"-nogenmake -noclean -nodepend\""
28			echo " (-nogenmake\|-ng) skip the genmake stage"
29			echo " (-noclean\|-nc) skip the \"make clean\" stage"
30			echo " (-nodepend\|-nd) skip the \"make depend\" stage"
31	edhill	1.6	echo
32			echo "and where STRING follows a whitespace-delimited format"
33			echo "such as:"
34			echo " -t 'exp0 exp2 exp3' "
35			echo " -addr='abc@123.com testing@home.org'"
36	edhill	1.1	echo
37			exit 1
38			}
39
40			# build the mpack utility
41			build_mpack()
42			{
43			echo -n "building the mpack utility... "
44			if test ! -x "$MPACKDIR/mpack" ; then
45			if test ! -d $MPACKDIR ; then
46			echo "Error: can't find \"$MPACKDIR\""
47			echo " are you sure this program is being run in the correct "
48			echo " (that is, \"MITGCM_ROOT\verification\") directory?"
49			exit 1
50			fi
51			echo -n "building mpack... "
52	edhill	1.8	( cd $MPACKDIR && ./configure && $MAKE ) > build_mpack.out 2>&1
53	edhill	1.1	RETVAL=$?
54			if test "x$RETVAL" != x0 ; then
55			echo
56			echo "Error building the mpack tools at: $MPACK_DIR"
57			exit 1
58			fi
59			fi
60			echo "OK"
61			}
62
63			compare_lines()
64			{
65			# use codelet to compare lines
66			if [ $verbose -gt 1 ]; then
67			cat tmp3.txt 1>&2
68			fi
69			return `./a.out < tmp3.txt`
70			}
71
72			testoutput_for_prop()
73			{
74			# testoutput_for_prop dir s1 label subdir
75			#
76			# compares files in $dir/$subdir/output.txt and $dir/results/output.txt
77			# using search strings s1 and text label
78
79			if [ $debug -gt 0 ]; then
80			echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
81			fi
82			if [ -r $1/$4/output.txt ]; then
83			grep "$2" $1/$4/output.txt \| sed 's/.*=//' \| nl > tmp1.txt
84			lncnt=`wc -l tmp1.txt \| awk '{print $1}' `
85			if [ $lncnt -lt 3 ]; then
86			if [ $verbose -gt 0 ]; then
87			echo Not enough lines of output when searching for "$2" 1>&2
88			fi
89			return 99
90			fi
91			else
92			echo testoutput_for_prop: output.txt from model run was not readable 1>&2
93			return 99
94			fi
95			if [ $debug -gt 0 ]; then
96			echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
97			fi
98			grep "$2" $1/results/output.txt \| sed 's/.*=//' \| nl > tmp2.txt
99			lncnt=`wc -l tmp2.txt \| awk '{print $1}' `
100			if [ $lncnt -lt 3 ]; then
101			if [ $verbose -gt 0 ]; then
102			echo Not enough lines of output when searching for "$2" 1>&2
103			fi
104			return 99
105			fi
106			if [ $debug -gt 0 ]; then
107			echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
108			fi
109			join tmp1.txt tmp2.txt \| awk '{print $1 " " $2 " " $3}' > tmp3.txt
110			if [ $debug -gt 0 ]; then
111			echo testoutput_for_prop: compare_lines 1>&2
112			fi
113			compare_lines
114			digits_of_similarity=$?
115			if [ $digits_of_similarity -eq 99 ]; then
116			if [ $verbose -gt 0 ]; then
117			echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
118			fi
119			digits_of_similarity=99
120			else
121			if [ $verbose -gt 0 ]; then
122			echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
123			fi
124			fi
125			rm tmp1.txt tmp2.txt tmp3.txt
126
127			return $digits_of_similarity
128			}
129
130			dashnum()
131			{
132			# dashnum n1 n2 n3 ...
133			#
134			# print numbers using %3i format or "--" if number = 99
135
136			for num in $@ ; do
137			if [ $num = 99 ]; then
138			printf ' --'
139			else
140			printf '%3i' $num
141			fi
142			done
143			}
144
145			testoutput()
146			{
147			# testoutput diretory subdir
148			#
149			# test output in "directory"
150
151			if [ $debug -gt 0 ]; then
152			echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
153			fi
154			testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
155			if [ $debug -gt 0 ]; then
156			echo testoutput: cg2dres=$cg2dres 1>&2
157			fi
158
159			testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
160			testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
161			testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
162			testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
163			testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
164			testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
165			testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
166			testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
167			testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
168			testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
169			testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
170			testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
171			testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
172			testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
173			testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
174			testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
175
176			dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
177			$umin $umax $umean $usd $vmin $vmax $vmean $vsd
178			}
179
180			genmakemodel()
181			{
182			# genmakemodel directory
183	edhill	1.9.2.1	if test "x$NOGENMAKE" = xt ; then
184			echo "genmake skipped!"
185			else
186			GENMAKE2="$BASH ../../../tools/genmake2"
187			(
188			cd $1;
189			printf 'genmake ... ' 1>&2
190	edhill	1.9.2.6	command="$GENMAKE2 -ds -m $MAKE --mods=../code"
191			if test "x$OPTFILE" != xNONE ; then
192			command="$command --optfile=$OPTFILE"
193			# echo " command=\"$command\""
194			fi
195	edhill	1.9.2.9	if test "x$IEEE" != x ; then
196	edhill	1.9.2.10	command="$command -ieee"
197	edhill	1.9.2.9	fi
198	edhill	1.9.2.6	$command > make.log 2>&1
199	edhill	1.9.2.1	RETVAL=$?
200	edhill	1.9.2.8	for i in genmake_state genmake_optfile genmake_local Makefile ; do
201	edhill	1.9.2.1	if test -r $i ; then
202			cp $i $CDIR
203			fi
204			done
205			if test "x$RETVAL" != x0 ; then
206			tail make.log
207			echo "genmakemodel: genmake failed" 1>&2
208			cp make.log $CDIR
209			return 1
210			else
211			echo "succesful" 1>&2
212	edhill	1.1	fi
213	edhill	1.9.2.1	)
214			fi
215	edhill	1.1	}
216
217			makeclean()
218			{
219			# makeclean directory
220	edhill	1.9.2.1	if test "x$NOCLEAN" = xt ; then
221			echo "make CLEAN skipped!"
222			else
223			(
224			cd $1;
225			if test -e output.txt ; then
226			rm -f output.txt
227	edhill	1.1	fi
228	edhill	1.9.2.1	printf 'make CLEAN ... ' 2>&1
229			if test -r Makefile ; then
230			$MAKE CLEAN >> make.log 2>&1
231			RETVAL=$?
232			if test "x$RETVAL" != x0 ; then
233			tail make.log
234			echo "makeclean: \"make CLEAN\" failed" 1>&2
235			cp make.log $CDIR"/make.log"
236			return 1
237			fi
238			fi
239			echo succesful 1>&2
240			exit 0
241			)
242			fi
243	edhill	1.1	}
244
245			makedependmodel()
246			{
247			# makedependmodel directory
248	edhill	1.9.2.1	if test "x$NODEPEND" = xt ; then
249			echo "make depend skipped!"
250			else
251			(
252			cd $1;
253			printf 'make depend ... ' 1>&2
254			$MAKE depend >> make.log 2>&1
255			RETVAL=$?
256			if test "x$RETVAL" != x0 ; then
257			tail make.log
258			echo "makedependmodel: make depend failed" 1>&2
259			cp make.log $CDIR"/make.log"
260			return 1
261			else
262			echo succesful 1>&2
263			fi
264			)
265			fi
266	edhill	1.1	}
267
268			makemodel()
269			{
270			# makemodel directory
271			(
272			cd $1;
273			if test -r Makefile ; then
274			printf 'make ... ' 1>&2
275	edhill	1.8	$MAKE >> make.log 2>&1
276	edhill	1.1	RETVAL=$?
277			if test "x$RETVAL" != x0 ; then
278			tail make.log
279			echo failed 1>&2
280			cp make.log $CDIR"/make.log"
281			return 1
282			else
283			echo succesful 1>&2
284			fi
285			fi
286			)
287			}
288
289			linkdata()
290			{
291			# linkdata flag
292			#
293			# symbolically link data files to run directory
294			if [ $1 -ne 0 ]; then
295			( cd $2 ; ln -sf ../input/* . )
296			fi
297			}
298
299			runmodel()
300			{
301	edhill	1.6	# runmodel directory
302	edhill	1.1	#
303	edhill	1.6	# runs "$COMMAND" in "directory"
304			# (where "$COMMAND" is relative to "directory")
305	edhill	1.1	(
306			cd $1
307	edhill	1.6	printf 'runmodel: ' 1>&2
308			# make output.txt
309			$COMMAND
310			RETVAL=$?
311			if test "x$RETVAL" = x0 ; then
312			cp output.txt $CDIR"/output.txt"
313			return 0
314			else
315			return 1
316	edhill	1.1	fi
317			)
318			}
319
320			createcodelet()
321			{
322			# create codelet for comparing model output
323
324			echo -n "creating the comparison code... "
325			cat > tmp_cmpnum.f <<EOFA
326			program cmpnum
327			implicit none
328			real*8 a,b,diff
329			integer linnum,best
330			best=-16
331			99 read(,,end=70,err=60) linnum,a,b
332			diff=0.5*(abs(a)+abs(b))
333			c print *,a,b,diff,abs(a-b)/diff
334			if (diff.gt.1.e-12) then
335			diff=abs(a-b)/diff
336			if (diff.gt.0.) then
337			c print *,int(log10(diff)),diff
338			linnum=int(log10(diff))
339			best=max(best,linnum)
340			endif
341			else
342			if (best.eq.-16.and.diff.ne.0.) best=-22
343			endif
344			goto 99
345			60 stop 'cmpnum: An error occured reading a,b'
346			70 print *,-best
347			end
348			EOFA
349
350			f77 tmp_cmpnum.f
351			if [ -x ./a.out ]; then
352			echo "OK"
353			return 0
354			else
355			echo
356	edhill	1.3	echo "createcodelet: failed to compile codelet"
357	edhill	1.1	exit 1
358			fi
359			}
360
361			formatresults()
362			{
363			# formatresults expt genmake depend make run results*
364
365			nm=$1
366			printf '%s %s %s %s' $2 $3 $4 $5
367			shift; shift; shift; shift; shift;
368			printf '%3s' $@
369
370			if [ $1 = '--' ]; then
371			printf ' N/O '
372			else
373			if [ $1 -gt 12 ]; then
374			printf ' pass'
375			else
376			printf ' FAIL'
377			fi
378			fi
379			printf ' %s' $nm
380			printf '\n'
381
382			}
383
384			show_help()
385			{
386			cat - << EOF
387			$0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
388	edhill	1.9.2.1
389			-help\|-h Show this help message
390	edhill	1.1	-quiet Reduce the amount of output
391			-verbose Produce copious amounts of output
392			-debug Produce even more output which will mean nothing to most
393			-force Do "make CLEAN" before compiling. This forces a complete rebuild.
394			-clean Do "make CLEAN" after compiling and testing.
395			-cleanup Aggresively removes all model output, executables and object files
396			and then exits. Use with care.
397
398			Normal usage:
399			$0 * Configure, compile, run and analyze in all experiment directories
400			EOF
401			}
402
403			scandirs()
404			{
405			if [ $# -eq 0 ]; then
406			for arg in * ; do
407			test -d $arg/input && echo $arg
408			done
409			else
410			echo $*
411			fi
412			}
413
414
415			###############################################################################
416			###############################################################################
417			###############################################################################
418
419
420			# Default properties
421			debug=0
422			verbose=1
423			clean=0
424			expts=''
425	edhill	1.6	# ieee=1
426	edhill	1.9.2.1
427	edhill	1.9.2.9	IEEE=
428			if test "x$MITGCM_IEEE" != x ; then
429			IEEE=$MITGCM_IEEE
430			fi
431
432
433	edhill	1.9.2.1	CLEANUP=f
434	edhill	1.9.2.2	QUICK=f
435	edhill	1.9.2.1	NOGENMAKE=f
436			NOCLEAN=f
437			NODEPEND=f
438	edhill	1.1
439	edhill	1.4	BASH=
440	edhill	1.9.2.6	OPTFILE=NONE
441	edhill	1.9.2.1	ADDRESSES=
442	edhill	1.1	TESTDIRS=
443			MPACKDIR="../tools/mpack-1.6"
444			MPACK="$MPACKDIR/mpack"
445	edhill	1.9.2.2	COMMAND="make output.txt"
446	edhill	1.8	MAKE=make
447	edhill	1.7	MPI=f
448	edhill	1.1
449			echo -n "parsing options... "
450
451			ac_prev=
452			for ac_option ; do
453
454			# If the previous option needs an argument, assign it.
455			if test -n "$ac_prev"; then
456			eval "$ac_prev=\$ac_option"
457			ac_prev=
458			continue
459			fi
460
461			ac_optarg=`expr "x$ac_option" : 'x[^=]=\(.\)'`
462
463			case $ac_option in
464
465			-help \| --help \| -h \| --h)
466			usage ;;
467
468	edhill	1.2	-optfile \| --optfile \| -of \| --of)
469	edhill	1.9.2.6	ac_prev=OPTFILE ;;
470	edhill	1.2	-optfile=* \| --optfile=* \| -of=* \| --of=*)
471	edhill	1.9.2.6	OPTFILE=$ac_optarg ;;
472	edhill	1.1
473			-addr \| --addr \| -a \| --a)
474			ac_prev=ADDRESSES ;;
475			-addr=* \| --addr=*)
476			ADDRESSES=$ac_optarg ;;
477
478			-tdir \| --tdir \| -t \| --t)
479			ac_prev=TESTDIRS ;;
480			-tdir=* \| --tdir=*)
481			TESTDIRS=$ac_optarg ;;
482	edhill	1.4
483			-bash \| --bash \| -b \| --b)
484			ac_prev=BASH ;;
485			-bash=* \| --bash=*)
486			BASH=$ac_optarg ;;
487	edhill	1.5
488	edhill	1.6	-command \| --command \| -c \| --c)
489			ac_prev=COMMAND ;;
490			-command=* \| --command=*)
491			COMMAND=$ac_optarg ;;
492	edhill	1.8
493			-make \| --make \| -m \| --m)
494			ac_prev=MAKE ;;
495			-make=* \| --make=*)
496			MAKE=$ac_optarg ;;
497	edhill	1.1
498	edhill	1.9.2.3	-clean \| --clean)
499	edhill	1.9.2.2	CLEANUP=t ;;
500
501	edhill	1.9.2.1	-quick \| --quick \| -q \| --q)
502			QUICK=t ;;
503			-nogenmake \| --nogenmake \| -ng \| --ng)
504			NOGENMAKE=t ;;
505			-noclean \| --noclean \| -nc \| --nc)
506			NOCLEAN=t ;;
507			-nodepend \| --nodepend \| -nd \| --nd)
508			NODEPEND=t ;;
509
510	edhill	1.9.2.9	-mpi) MPI=true ;;
511
512			-ieee) IEEE=true ;;
513			-noieee) IEEE= ;;
514
515	edhill	1.1	-verbose) verbose=2 ;;
516			-debug) debug=1 ;;
517			-quiet) verbose=0 ;;
518
519			-*)
520			echo "Error: unrecognized option: "$ac_option
521			usage
522			;;
523
524			*)
525			echo "Error: unrecognized argument: "$ac_option
526			usage
527			;;
528
529			esac
530
531			done
532
533	edhill	1.9.2.1	if test "x$QUICK" = xt ; then
534			NOGENMAKE=t
535			NOCLEAN=t
536			NODEPEND=t
537			fi
538
539	edhill	1.1	if test "x$TESTDIRS" = x ; then
540			TESTDIRS=`scandirs`
541			fi
542
543	edhill	1.9.2.6	if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
544			OPTFILE=$MITGCM_OF
545	edhill	1.9.2.1	fi
546
547	edhill	1.1	echo "OK"
548
549			# create the FORTRAN comparison code
550			createcodelet
551
552			# build the mpack utility
553			build_mpack
554
555			# Create a uniquely named directory to store results
556			MACH=`hostname`
557	edhill	1.2	UNAMEA=`uname -a`
558	edhill	1.1	DATE=`date +%Y%m%d`
559			BASE=$MACH"_"$DATE"_"
560			DNUM=0
561			DRESULTS="$BASE$DNUM"
562			while test -e $DRESULTS ; do
563			DNUM=$(( $DNUM + 1 ))
564			DRESULTS="$BASE$DNUM"
565			done
566			mkdir $DRESULTS
567			RETVAL=$?
568			if test "x$RETVAL" != x0 ; then
569			echo "Error: can't create results directory \"./$DRESULTS\""
570			exit 1
571			fi
572			SUMMARY="$DRESULTS/summary.txt"
573			date > $SUMMARY
574			cat << EOF >> $SUMMARY
575			T S U V
576			G D M c m s m s m s m s
577			E p a R g m m e . m m e . m m e . m m e .
578			N n k u 2 i a a d i a a d i a a d i a a d
579			2 d e n d n x n . n x n . n x n . n x n .
580
581			EOF
582
583			NDIR=0
584
585	edhill	1.9.2.6	if test "x$OPTFILE" != xNONE ; then
586			if test -r $OPTFILE ; then
587			OPTFILE=`pwd`"/$OPTFILE"
588	edhill	1.1	fi
589	edhill	1.9.2.6	fi
590			echo
591			echo "OPTFILE=$OPTFILE" >> $SUMMARY
592			echo >> $SUMMARY
593	edhill	1.3
594	edhill	1.9.2.6	# ...and each test directory...
595			for dir in $TESTDIRS ; do
596
597			# Cleanup only!
598			if test "x$CLEANUP" = xt ; then
599			if test -r $dir/build/Makefile ; then
600			( cd $dir/build ; make CLEAN )
601	edhill	1.9.2.2	fi
602	edhill	1.9.2.6	if test -r $dir/input/Makefile ; then
603			( cd $dir/input ; make CLEAN )
604	edhill	1.3	fi
605	edhill	1.9.2.6	continue
606			fi
607	edhill	1.7
608	edhill	1.9.2.6	# Verify that the testdir exists and contains previous
609			# results in the correct location--or skip this directory!
610			if test ! -r $dir"/results/output.txt" ; then
611			echo "can't read \"$dir/results/output.txt\" -- skipping $dir"
612			continue
613			fi
614	edhill	1.9.2.1
615	edhill	1.9.2.6	echo "-------------------------------------------------------------------------------"
616			echo
617			echo "Experiment: $dir"
618			echo
619			unset genmake makedepend make run
620			results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
621	edhill	1.9.2.5
622	edhill	1.9.2.6	if [ -r $dir/build ]; then
623			seperatebuilddir=1
624			builddir=build
625			rundir=build
626			( cd $dir/build; ln -sf ../input/* . )
627			else
628			seperatebuilddir=0
629			builddir=input
630			rundir=input
631			fi
632
633			CODE_DIR=$dir/code
634			BUILD_DIR=$dir/$builddir
635			MPI_FILES="CPP_EEOPTIONS.h_mpi SIZE.h_mpi"
636			NOMPI_FILES="CPP_EEOPTIONS.h_nompi SIZE.h_nompi"
637
638			# Is this an MPI run?
639			if test "x$MPI" = xt ; then
640			FILES=$MPI_FILES
641			else
642			FILES=$NOMPI_FILES
643			fi
644
645			# Check to see that we have the files
646			have_files=t
647			for i in $FILES ; do
648			if test ! -r $CODE_DIR/$i ; then
649			echo "Warning: can't read file $CODE_DIR/$i"
650			have_files=f
651	edhill	1.9.2.5	fi
652	edhill	1.9.2.6	done
653			if test "x$have_files" != xt -a "x$MPI" = xt ; then
654			echo "Skipping $dir due to lack of input files (see above warning)"
655			continue
656			fi
657
658			# If we have the $FILES and they differ, copy the $FILES to $BUILD_DIR
659			if test "x$have_files" = xt ; then
660	edhill	1.9.2.5	for i in $FILES ; do
661	edhill	1.9.2.6	cmp $CODE_DIR/$i $BUILD_DIR/$i > /dev/null 2>&1
662	edhill	1.9.2.1	RETVAL=$?
663			if test "x$RETVAL" != x0 ; then
664	edhill	1.9.2.6	cp $CODE_DIR/$i $BUILD_DIR/$i
665	edhill	1.9.2.1	fi
666	edhill	1.9.2.6	done
667			fi
668
669			# Create an output dir for each OPTFILE/tdir combination
670			CDIR=$DRESULTS"/"$DRESULTS"_"$NDIR
671			mkdir $CDIR
672			CDIR=`pwd`"/$CDIR"
673
674			if test "x$CLEANUP" = xt ; then
675			makeclean $dir/$builddir
676			else
677			genmakemodel $dir/$builddir && genmake=Y \
678			&& makeclean $dir/$builddir \
679			&& makedependmodel $dir/$builddir && makedepend=Y \
680			&& makemodel $dir/$builddir && make=Y \
681			&& linkdata $seperatebuilddir $dir/$rundir \
682			&& runmodel $dir/$builddir && run=Y \
683			&& results=`testoutput $dir $rundir`
684			fi
685
686			echo
687			formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
688			${run:-N} $results
689			echo
690			formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
691			${run:-N} $results >> $SUMMARY
692			echo "fresults='" > $CDIR"/summary.txt"
693			formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \
694			${run:-N} $results >> $CDIR"/summary.txt"
695			echo "'" >> $CDIR"/summary.txt"
696			echo "MACH='$MACH'" >> $CDIR"/summary.txt"
697			echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
698			echo "DATE='$DATE'" >> $CDIR"/summary.txt"
699			echo "tdir='$dir'" >> $CDIR"/summary.txt"
700
701			(
702			cd $DRESULTS
703			tar -cf $NDIR".tar" $DRESULTS"_"$NDIR > /dev/null 2>&1
704			gzip $NDIR".tar"
705			)
706
707			if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
708			echo "No mail sent"
709			else
710			$MPACK -s MITgcm-test -m 1000000 $DRESULTS"/"$NDIR".tar.gz" $ADDRESSES
711			RETVAL=$?
712			if test "x$RETVAL" != x0 ; then
713			echo "Warning: \"$MPACK\" failed -- please contact <edhill@mitgcm.org>"
714			else
715			rm -f $DRESULTS"/"$NDIR".tar*"
716	edhill	1.2	fi
717	edhill	1.9.2.6	fi
718	edhill	1.9.2.7
719			echo "-------------------------------------------------------------------------------"
720	edhill	1.9.2.6
721			NDIR=$(( $NDIR + 1 ))
722
723	edhill	1.1	done
724
725	edhill	1.2	rm tmp_cmpnum.f a.out
726	edhill	1.1
727			cat $SUMMARY
728