Annotation of /MITgcm/verification/testreport

#! /usr/bin/env bash
#
#  $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.152 2010/12/17 17:32:24 jmc Exp $
#  $Name:  $
#

usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-help|-h)               print usage"
    echo "  (-mth)                   run multi-threaded (using eedata.mth)"
    echo "  (-mpi)                   compile and run using MPI"
    echo "  (-ieee/-noieee)          if possible, use IEEE compiler flags"
    echo "                             (DEF=\"-ieee\")"
    echo "  (-gsl)                   compile with \"-gsl\" flag"
    echo "  (-use_r4|-ur4)           if allowed, use real*4 type for '_RS' variable"
    echo "  (-optfile|-of) STRING    list of optfiles to use"
    echo "  (-addr|-a) STRING        list of email recipients"
    echo "                             (DEF=\"\" no email is sent)"
    echo "  (-mpackdir|-mpd) DIR     location of the mpack utility"
    echo "                             (DEF=\"../tools/mpack-1.6\")"
    echo "  (-tdir|-t) STRING        list of group and/or exp. dirs to test"
    echo "                             (recognized groups: basic, tutorials)"
    echo "                             (DEF=\"\" which test all)"
    echo "                             (if list= 'start_from THIS_EXP' then"
    echo "                              test THIS_EXP + all the following)"
    echo "  (-skipdir|-skd) STRING   list of exp. dirs to skip"
    echo "                             (DEF=\"\" which test all)"
    echo "  (-bash|-b) STRING        preferred location of a \"bash\" or"
    echo "                             Bourne-compatible \"sh\" shell"
    echo "                             (DEF=\"\" for \"bash\")"
    echo "  (-adm|-ad)               perform an adjoint run"
    echo "  (-oad)                   perform an OpenAD adjoint run"
    echo "  (-command|-c) STRING     command to run"
    echo "  (-makedepend|-md) STRING command to use for \"makedepend\""
    echo "  (-make|-m) STRING        command to use for \"make\""
    echo "                             (DEF=\"make\")"
    echo "  (-odir) STRING           used to build output directory name"
    echo "                             (DEF=\"hostname\")"
    echo "  (-ptracers|-ptr) STRING  specify which ptracers to test"
    echo "                             (DEF=\"1 2 3 4 5\")"
    echo "  (-match) NUMBER          Matching Criteria (number of digits)"
    echo "                             (DEF=\"$MATCH_CRIT\")"
    echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
    echo "  (-clean)                 *ONLY* run \"make CLEAN\""
    echo "  (-norun|-nr)             skip the \"runmodel\" stage (stop after make)"
    echo "  (-runonly|-ro)           *ONLY* run stage (=\"-quick\" without make)"
    echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
    echo "  (-nogenmake|-ng)         skip the genmake stage"
    echo "  (-noclean|-nc)           skip the \"make clean\" stage"
    echo "  (-nodepend|-nd)          skip the \"make depend\" stage"
    echo "  (-deldir|-dd)            on success, delete the output directory"
    echo "  (-ts)                    provide timing information per timestep"
    echo "  (-papis)                 provide MFlop/s per timestep using PAPI"
    echo "  (-pcls)                  provide MFlop/s per timestep using PCL"
    echo
    echo "and where STRING can be a whitespace-delimited list"
    echo "such as:"
    echo
    echo "  -t 'exp0 exp2 exp3' "
    echo "  -addr='abc@123.com testing@home.org'"
    echo
    echo "provided that the expression is properly quoted within the current"
    echo "shell (note the use of single quotes to protect white space)."
    echo
    exit 1
}

#  build the mpack utility
build_mpack()
{
    printf "building the mpack utility...  "
    MPACK="$MPACKDIR/mpack"
    if test ! -x $MPACK ; then
	if test ! -d $MPACKDIR ; then
	    echo
	    echo "Error: can't find \"$MPACKDIR\""
	    echo "  are you sure this program is being run in the correct "
	    echo "  (that is, \"MITGCM_ROOT\verification\") directory?"
	    echo
	    HAVE_MPACK=f
	fi
	if test "x$CC" = x ; then
	    export CC=cc
	fi
	printf "building mpack (using CC=$CC)...  "
	( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
	RETVAL=$?
	if test "x$RETVAL" != x0 ; then
	    echo
	    echo "Error building the mpack tools at: $MPACK_DIR"
	    echo
	    HAVE_MPACK=f
	else
	    rm -f tr_build_mpack.out
	    HAVE_MPACK=t
	    echo "done"
	fi
    else
	HAVE_MPACK=t
	echo "already exist"
    fi
}

testoutput_var()
{
    # testoutput_var dir s1 label subdir reference_output
    #
    #  compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
    #     with same output from reference file $dir/results/$reference_output
    #  using search strings s1 and text label

    if [ $debug -gt 0 ]; then
	echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
    fi
    if [ -r $1/$4/$OUTPUTFILE ]; then
	grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
	lncntA=`wc -l tmp1.txt | awk '{print $1}' `
	if [ $lncntA -lt 2 ]; then
	    if [ $verbose -gt 0 ]; then
		echo Not enough lines of output when searching for "$2" 1>&2
	    fi
	    return 99
	fi
    else
	echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
	return 99
    fi
    if [ $debug -gt 0 ]; then
	echo testoutput_var: grep "$2" $1/results/$5 1>&2
    fi
    grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
    lncntB=`wc -l tmp2.txt | awk '{print $1}' `
    if [ $lncntB -lt 2 ]; then
	if [ $verbose -gt 0 ]; then
	    echo Not enough lines of output when searching for "$2" 1>&2
	fi
	return 99
    fi
    if [ $lncntA -ne $lncntB ]; then
        if [ $verbose -gt 0 ]; then
            echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
        fi
        return 99
    fi
    has_nan=`cat tmp1.txt | grep -i nan | wc -l`
    if [ $has_nan -gt 0  ] ; then
	echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values  1>&2
	return 99
    fi
    has_inf=`cat tmp1.txt | grep -i inf | wc -l`
    if [ $has_inf -gt 0  ] ; then
	echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values  1>&2
	return 99
    fi
    if [ $debug -gt 0 ]; then
	echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
    fi
    join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
    if [ $debug -gt 0 ]; then
	echo testoutput_var: compare_lines 1>&2
    fi
    if [ $verbose -gt 1 ]; then
	cat tmp3.txt 1>&2
    fi
    echo "-1" >> tmp3.txt
    # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
    cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
    digits_of_similarity=`./tr_cmpnum < tmp4.txt`
    if [ $digits_of_similarity -eq 99 ]; then
	if [ $verbose -gt 0 ]; then
	    echo testoutput_var: No comparison was available for \"$3\" 1>&2
	fi
	digits_of_similarity=99
    else
	if [ $verbose -gt 0 ]; then
	    echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
	fi
    fi
    rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt

    return $digits_of_similarity
}

testoutput_run()
{
    # testoutput_run directory subdir reference_output
    #
    #  test output from 1 run in "directory"
# --> same processing for adjoint & forward test
        # default list of output variables to be checked:
        #  1rst : main variable used to decide if it pass or FAIL
        #  others : number of matching digits to be printed in summary.txt
        listChk=$DEF_CHECK_LIST
	#  load experiment-specific list from file "tr_checklist" (if it exist)
        if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
        sVar=`echo $listChk | awk '{print $1}'`
        # remove 1rst var and expand the list: + => min max mean s.d
        listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
                               | sed 's/+//g' | sed "s/^$sVar//"`
	if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
        # check for ptracer output in reference_output file :
        outpref=$1/results/$3
	ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
	ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
	for ii in $PTRACERS_NUM ; do
	    ptrfound=0
	    for jj in $ptr_mon ; do
		name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
		tst=`grep $name $outpref | wc -l | awk '{print $1}'`
		if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
	    done
	    if test $ptrfound = '1' ; then
		eval "HAVE_PTR0"$ii"=t"
	    else
		eval "HAVE_PTR0"$ii"=f"
	      if test "x$ADM" = x -a "x$OADM" = x; then
	      # remove this ptr from the list of output variable to check
	      # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
		listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
	      fi
	    fi
	#   eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
	done
        tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
        if test $tst != 1 ; then
          if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
                 else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
          echo "==> WARNING: in checked list:" $listVar 1>&2
        #- put it back once:
          listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
        fi
	if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
	echo "listVar='$listVar'" > $locDIR"/summary.txt"
        allargs=""
        for xx in $listVar
        do
          case $xx in
	   'PS')  if [ $debug -gt 0 ]
		  then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
		  testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
		  if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
 	  'Cost') testoutput_var $1 "ADM  precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
 	  'Grad') testoutput_var $1 "ADM  precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
 	   'Tmn') testoutput_var $1 "dynstat_theta_min"  "Theta minimum"  $2 $3; yy=$? ;;
 	   'Tmx') testoutput_var $1 "dynstat_theta_max"  "Theta maximum"  $2 $3; yy=$? ;;
 	   'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean"	  $2 $3; yy=$? ;;
	   'Tsd') testoutput_var $1 "dynstat_theta_sd"   "Theta Std.Dev"  $2 $3; yy=$? ;;
	   'Smn') testoutput_var $1 "dynstat_salt_min"  "Salt minimum"	  $2 $3; yy=$? ;;
	   'Smx') testoutput_var $1 "dynstat_salt_max"  "Salt maximum"	  $2 $3; yy=$? ;;
	   'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean"	  $2 $3; yy=$? ;;
	   'Ssd') testoutput_var $1 "dynstat_salt_sd"   "Salt Std.Dev"	  $2 $3; yy=$? ;;
	   'Umn') testoutput_var $1 "dynstat_uvel_min"  "U minimum"	  $2 $3; yy=$? ;;
	   'Umx') testoutput_var $1 "dynstat_uvel_max"  "U maximum"	  $2 $3; yy=$? ;;
	   'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean"	  $2 $3; yy=$? ;;
	   'Usd') testoutput_var $1 "dynstat_uvel_sd"   "U Std.Dev"	  $2 $3; yy=$? ;;
	   'Vmn') testoutput_var $1 "dynstat_vvel_min"  "V minimum"	  $2 $3; yy=$? ;;
	   'Vmx') testoutput_var $1 "dynstat_vvel_max"  "V maximum"	  $2 $3; yy=$? ;;
	   'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean"	  $2 $3; yy=$? ;;
	   'Vsd') testoutput_var $1 "dynstat_vvel_sd"   "V Std.Dev"	  $2 $3; yy=$? ;;
	'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
	   testoutput_var $1 "trcstat_ptracer0"$ii"_min"  "p0"$ii"_min"	  $2 $3; yy=$? ;;
	'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
	   testoutput_var $1 "trcstat_ptracer0"$ii"_max"  "p0"$ii"_max"	  $2 $3; yy=$? ;;
	'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
	   testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
	'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
	   testoutput_var $1 "trcstat_ptracer0"$ii"_sd"   "p0"$ii"_StDv"  $2 $3; yy=$? ;;
	 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum"  $2 $3; yy=$? ;;
	 'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum"  $2 $3; yy=$? ;;
	 'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean"	  $2 $3; yy=$? ;;
	 'Qntsd') testoutput_var $1 "forcing_qnet_sd"  "Qnet Std.Dev"  $2 $3; yy=$? ;;
	 'aSImn') testoutput_var $1 "seaice_area_min"   "SIce Area min"	  $2 $3; yy=$? ;;
	 'aSImx') testoutput_var $1 "seaice_area_max"   "SIce Area max"	  $2 $3; yy=$? ;;
	 'aSIav') testoutput_var $1 "seaice_area_mean"  "SIce Area mean"  $2 $3; yy=$? ;;
	 'aSIsd') testoutput_var $1 "seaice_area_sd"    "SIce Area StDv"  $2 $3; yy=$? ;;
	 'hSImn') testoutput_var $1 "seaice_heff_min"   "SIce Heff min"	  $2 $3; yy=$? ;;
	 'hSImx') testoutput_var $1 "seaice_heff_max"   "SIce Heff max"	  $2 $3; yy=$? ;;
	 'hSIav') testoutput_var $1 "seaice_heff_mean"  "SIce Heff mean"  $2 $3; yy=$? ;;
	 'hSIsd') testoutput_var $1 "seaice_heff_sd"    "SIce Heff StDv"  $2 $3; yy=$? ;;
	'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
	'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South"  $2 $3; yy=$? ;;
	'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North"  $2 $3; yy=$? ;;
	'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global"	  $2 $3; yy=$? ;;
	'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South"	  $2 $3; yy=$? ;;
	'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North"	  $2 $3; yy=$? ;;
              *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
          esac
          if test $xx = $sVar
          then allargs="$allargs > $yy <"
          else allargs="$allargs $yy"
          fi
        done

	nbVar=`echo $listVar | awk '{print NF}'`
        if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
        #-- fill line (up to standard length) with dot:
          adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
	  echo $allargs $adNul
        else
	  echo $allargs
        fi
# <-- same processing for adjoint & forward test
}

genmakemodel()
{
    # genmakemodel directory
    if test "x$NOGENMAKE" = xt ; then
	echo "genmake skipped!"
    else
	if test "x$BASH" = x ; then
	    GENMAKE2="../../../tools/genmake2"
	else
	    GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
	fi
	(
	    cd $1;
	    command="$GENMAKE2  -ds -m $MAKE"
	    if test "x$MKDEPEND" != x ; then
		command="$command -makedepend=$MKDEPEND"
	    fi
	    if test "x$ADM" = xt ; then
		command="$command --mods=../code_ad"
	    elif test "x$OADM" = xt ; then
		command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
	    else
		command="$command -mods=../code"
	    fi
	    if test "x$OPTFILE" != xNONE ; then
		command="$command -optfile=$OPTFILE"
	    fi
	    if test "x$IEEE" != x ; then
		command="$command -ieee"
	    fi
	    if test "x$GSL" = xt ; then
		command="$command -gsl"
	    fi
	    if test "x$MPI" = xt ; then
		command="$command -mpi"
	    fi
	    if test "x$MULTI_THREAD" = xt ; then
	    #- run multi-threaded using OpenMP:
		command="$command -omp"
	    fi
	    if test "x$USE_R4" = xt ; then
		command="$command -use_r4"
	    fi
	    if test "x$TS" = xt ; then
		command="$command -ts"
	    fi
	    if test "x$PAPIS" = xt ; then
		command="$command -papis"
	    else
	    if test "x$PCLS" = xt ; then
		command="$command -pcls"
	    fi
	    fi
	    printf 'genmake ... '
	    eval $command > genmake.tr_log 2>&1
	    RETVAL=$?
	    #  Reduce the size of the testing emails!
	    head -100 Makefile > $CDIR/Makefile_head
	    if test "x$RETVAL" != x0 ; then
		tail genmake.tr_log
		echo "genmakemodel: genmake failed"
		cp genmake.log genmake_* genmake.tr_log $CDIR
		return 1
	    else
		echo "successful"
	    fi
	)
    fi
}

makeclean()
{
    # makeclean directory
    if test "x$NODEPEND" = xf ; then rm -f $1/make.tr_log ; fi
    if test "x$NOCLEAN" = xt ; then
	echo "make Clean skipped!"
    else
	(
	    cd $1;
	    #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
	    if test -r Makefile ; then
		printf 'clean build-dir: make Clean ... '
		rm -f Makefile.old
		$MAKE Clean >> make.tr_log 2>&1
		RETVAL=$?
		if test "x$RETVAL" != x0 ; then
		    tail make.tr_log
		    echo "makeclean: \"make Clean\" failed"
		    cp make.tr_log genmake.log genmake.tr_log $CDIR
		    return 1
		fi
		echo successful
	    else
		echo ''
	    fi
	    exit 0
	)
    fi
}

run_clean()
{
    # run_clean directory
    if test "x$NOCLEAN" = xt ; then
	echo "run_clean skipped!"
    else
	(
	    cd $1;
	    printf 'clean run-dir ... '
	    # part of what is done after "make clean" when doing "make CLEAN"
	    find . -name "*.meta" -exec rm {} \;
	    find . -name "*.data" -exec rm {} \;
	    find . -name "fort.*" -exec rm {} \;
	    find . -type l -exec rm {} \;
	    rm -f $EXECUTABLE $RUNLOG *.txt STD* *diagnostics.log datetime
	    rm -rf mnc_test_*
 	    rm -f *_MIT_CE_000.opt0000 costfunction*0000
	    echo successful
	    exit 0
	)
    fi
}

makedependmodel()
{
    # makedependmodel directory
    if test "x$NODEPEND" = xt ; then
	echo "make depend skipped!"
    else
	(
	    cd $1;
	    printf 'make depend ... '
	    $MAKE depend >> make.tr_log 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		tail make.tr_log
		echo "makedependmodel: make depend failed"
		cp make.tr_log genmake.log genmake.tr_log $CDIR
		return 1
	    else
		echo successful
	    fi
	)
    fi
}

makemodel()
{
    # makemodel directory
    (
    if test "x$NOMAKE" = xt ; then
	cd $1;
	if test -x $EXECUTABLE ; then
	    echo "make skipped!"
	else
	    echo "no executable!"
	    return 2
	fi
    else
	cd $1;
	if test -r Makefile ; then
	    printf 'make ... '
	    if test "x$ADM" = xt ; then
		$MAKE adall >> make.tr_log 2>&1
	    elif test "x$OADM" = xt ; then
		$MAKE adAll >> make.tr_log 2>&1
            else
		if test "x$JOBS" = x ; then
		    $MAKE >> make.tr_log 2>&1
		else
		    $MAKE -j $JOBS >> make.tr_log 2>&1
		fi
	    fi
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		tail make.tr_log
		echo failed
		cp make.tr_log genmake.log genmake.tr_log $CDIR
		rm -f $EXECUTABLE
		return 1
	    else
		echo successful
	    fi
	fi
    fi
    )
}

symlink_mpifiles()
{
    # Put special links so that MPI specific files are used
    # This MUST be invoked between makeclean and makelinks because
    # the Makefile will link to non-mpi files by default

    dir=$1
    code_dir=$2
    BUILD_DIR=$dir/$3
    CODE_DIR=$dir/$code_dir

    # These are files that should replace their counter-part when using -mpi
    MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`

    #  Is this an MPI run?
    if test "x$MPI" = xt ; then
	# YES: We symbolically link these files to the build
	# dir so long as there is no real file in place
	for ii in $MPI_FILES ; do
	    i=`echo $ii | sed 's:^\./::'`
	    name=`echo $i | sed 's:_mpi::' `
	    cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" != x0 ; then
		if ! test -f $BUILD_DIR/$i ; then
	            #echo Linking $name to $i
		    (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
		fi
	    fi
	done
    else
        # NO: We undo any _mpi symbolically linked files
	for ii in $MPI_FILES ; do
	    i=`echo $ii | sed 's:^\./::'`
	    name=`echo $i | sed 's:_mpi::' `
	    if test -L $BUILD_DIR/$name ; then
		cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
		RETVAL=$?
		if test "x$RETVAL" = x0 ; then
		    #echo Un-linking $name from $linktarg
		    rm -f $BUILD_DIR/$name
		fi
	    fi
	done
    fi

}

linkdata()
{
    # linkdata run_dir input_dir_1 input_dir_2 ...
    #
    # symbolically link data files to run directory
    if test -d $1 ; then
	(
	    cd $1 ; shift
	    echo 'linkdata from dirs:' $*
            inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
	    for xx in $inpMPI ; do
	      if test -r "../"$1"/"$xx ; then
	        # found 1 _mpi sfx file in 1rst input dir and it is readable
		yy=`echo $xx | sed 's:\.mpi$::'`
		if test "x$MPI" = "xt" ; then
		# mpi test: remove symbolic link & link _mpi sfx file
		    if test -h $yy ; then rm -f $yy ; fi
		    if test ! -r $yy ; then
			ln -sf "../"$1"/"$xx $yy ;
			printf " $xx" 1>&2
		    fi
		else
		# not mpi test: remove symbolic link
		    if test -h $yy ; then rm -f $yy ; fi
		fi
	      fi
	    done
	    if test -r "../"$1"/eedata.mth" ; then
	    # found eedata.mth in 1rst input dir and it is readable
		if test "x$MULTI_THREAD" = "xt" ; then
		# multi-threaded test: remove symbolic link & link eedata.mth
		    if test -h eedata ; then rm -f eedata ; fi
		    if test ! -r eedata ; then
			ln -sf "../"$1"/eedata.mth" eedata ;
			printf ' eedata.mth' 1>&2
		    fi
		else
		# not multi-threaded test: remove eedata symbolic link
		    if test -h eedata ; then rm -f eedata ; fi
		fi
	    fi
	    prevDir='NONE'
	    for ldir in $* ; do
		if test -d "../"$ldir -a $ldir != $prevDir ; then
                    printf " ldir=${ldir}:" 1>&2
		    files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
		    for i in $files ; do
			if test ! -d "../"$ldir/$i ; then
			    if test ! -r $i  ; then
				printf ' '$i 1>&2
				ln -sf "../"$ldir"/"$i $i
			    fi
			fi
		    done
		    printf ' ;' 1>&2
		    if test -x "../"$ldir"/"prepare_run ; then
			"../"$ldir"/"prepare_run 1>&2
		    else
			echo '' 1>&2
		    fi
		fi
		prevDir=$ldir
	    done
	)
    fi
}

runmodel()
{
    # runmodel directory
    #
    #  runs "$COMMAND in "directory"
    #  (where "$COMMAND" is relative to "directory")
    (
	cd $1
	printf 'runmodel in %s ... ' $1
	if test -L $EXECUTABLE ; then
	  if test -x "../"$builddir"/"$EXECUTABLE ; then
	    diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
	    outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi
	  else rm -f $EXECUTABLE
	  fi
	fi
        if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
	    echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
	    ln -sf "../"$builddir"/"$EXECUTABLE .
	fi
        if test ! -x $EXECUTABLE ; then
		rm -f $RUNLOG ; touch $RUNLOG
		if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
		echo " no executable:" $EXECUTABLE >> $RUNLOG
		RETVAL=8
		ENDVAL=-1
	else
            if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
              # output do not exist or is older than executable:
		rm -f $RUNLOG ; touch $RUNLOG
		if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
		( eval $COMMAND ) >> $RUNLOG 2>&1
		RETVAL=$?
	    else
		RETVAL=0
		if test -f $RUNLOG ; then
		    if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
     		    echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
		else
		    touch $RUNLOG
		    if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
     		    echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
     		    echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1
		fi
	    fi
	    #ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
	    ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'`
	fi
	rm -f run.log_tmp
	if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
	    echo successful
	    printf '=> output from running in %s :\n' $1 1>&2
	    tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
	    # === Reduce the size of the testing emails!
	    #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
	    return 0
	elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then
	    #-- for some weird cases (run is finihed but with error code)
	    echo 'finished with error (run:' $RETVAL ' end:' $ENDVAL ')'
	    printf '=> output from running in %s :\n' $1 1>&2
	    tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
	    return 0
	else
	    echo 'failed (run:' $RETVAL ' end:' $ENDVAL ')'
	    printf '=> output from running in %s :\n' $1 1>&2
	    tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
	    cp $RUNLOG $CDIR"/"$RUNLOG
            if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
	    return 1
	fi
    )
}

createcodelet()
{
    # create codelet for comparing model output

    printf "creating the comparison code (using CC=$CC)...  "
    cat > tr_cmpnum.c <<EOF
#include <stdio.h>
#include <math.h>
int main( int argc, char** argv )  {
  int linnum,cmplin,best,lncnt;
  double a,b,abave,relerr;
  best = -22;
  lncnt = 0;
  while( 1 & ( (lncnt+=1) < 999 ) )  {
    scanf("%d", &linnum);
    if (linnum == -1)  break;
    scanf("%lf", &a);  scanf("%lf", &b);
    abave = 0.5*(fabs(a)+fabs(b));
    if ( abave == abave ) {
      if (abave > 0.0) {
        relerr=fabs(a-b)/abave;
        if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
        else { cmplin = -16 ; }
        best = (best > cmplin) ? best : cmplin; }
      else { cmplin = -22 ; }
   /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
      }
   else {
   /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
      break; }
  }
  if (lncnt == 999) best=-29;
  if (linnum != -1) best=-99;
  printf("%d\n", -best);
  return 0;
}
EOF
    $CC -o tr_cmpnum tr_cmpnum.c -lm

    if [ -x ./tr_cmpnum ]; then
	echo "OK"
	return 0
    else
	echo
	echo "ERROR: failed to compile comparison code -- please specify"
	echo "  a C compiler using the CC environment variable."
	exit 1
    fi
}

formatresults()
{
    # formatresults expt genmake depend make run results*

    nm=$1
    printf '%s %s %s %s' $2 $3 $4 $5
    shift; shift; shift; shift; shift;
    listPrt=$@
    listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
    xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
    printf '%3s' $listPrt
#   line below does not work on hp-ux_ia64 : do those substitutions later on
#   printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</'

    if [ $xx = '..' ]; then
	printf ' N/O '
    elif [ $xx = '--' ]; then
	printf ' N/O '
    elif [ $xx = 99 ]; then
	printf ' N/O '
    else
	if [ $xx -ge $MATCH_CRIT ]; then
	    printf ' pass'
	else
	    printf ' FAIL'
	fi
    fi
    printf '  %s' $nm
    printf '\n'

}

scandirs()
{
    if [ $# -eq 1 ]; then
 	for arg in * ; do
 	    test -f $arg/$1 && echo $arg
 	done
    else
 	echo $*
    fi
}


check_eedata()
{
    # check_eedata size.h eedata
    if [ $# -eq 2 ] ; then
      if test -f $1 -a -f $2 ; then
      sx=`grep "^     & *nSx *=" $1 | sed "s/^     & *nSx *=//" | sed 's/, *$//'`
      nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
      if test "x$nx" = x ; then
	rx=10
      else
	rx=`expr $sx % $nx`
      fi
      sy=`grep "^     & *nSy *=" $1 | sed "s/^     & *nSy *=//" | sed 's/, *$//'`
      ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
      if test "x$ny" = x ; then
	ry=20
      else
	ry=`expr $sy % $ny`
      fi
      echo `expr $rx + $ry`
    else
      echo '-1'
      fi
    else
      echo '-2'
    fi

}

###############################################################################


#  Default properties
debug=0
verbose=1
clean=0

IEEE=true
if test "x$MITGCM_IEEE" != x ; then
    IEEE=$MITGCM_IEEE
fi
GSL=f

CLEANUP=f
NORUN=f
QUICK=f
NOMAKE=f
NOGENMAKE=f
NOCLEAN=f
NODEPEND=f
POSTCLEAN=f

BASH=
OPTFILE=NONE
ADDRESSES=
TESTDIRS=
SKIPDIRS=
MPACKDIR="../tools/mpack-1.6"
HAVE_MPACK=
MPACK=
COMMAND=
MKDEPEND=
if test "x$MAKE" = x ; then
    MAKE=make
fi
if test "x$CC" = x ; then
    CC=cc
fi
JOBS=
MPI=f
MULTI_THREAD=f
OUTDIR=
DELDIR=
USE_R4=

ADM=
OADM=

# list of pTracers to check for monitor output
PTRACERS_NUM="1 2 3 4 5"

MATCH_CRIT=10

printf "parsing options...  "

ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`

    case $ac_option in
	
	-help | --help | -h | --h)
	    usage ;;
	
	-optfile | --optfile | -of | --of)
            ac_prev=OPTFILE ;;
        -optfile=* | --optfile=* | -of=* | --of=*)
            OPTFILE=$ac_optarg ;;
	
	-addr | --addr | -a | --a)
            ac_prev=ADDRESSES ;;
        -addr=* | --addr=* | -a=* | --a=*)
            ADDRESSES=$ac_optarg ;;
        -mpackdir | --mpackdir | -mpd | --mpd)
            ac_prev=MPACKDIR ;;
        -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
            MPACKDIR=$ac_optarg ;;

	-tdir | --tdir | -t | --t)
            ac_prev=TESTDIRS ;;
        -tdir=* | --tdir=* | -t=* | --t=*)
            TESTDIRS=$ac_optarg ;;

	-skipdir | --skipdir | -skd | --skd)
            ac_prev=SKIPDIRS ;;
        -skipdir=* | --skipdir=* | -skd=* | --skd=*)
            SKIPDIRS=$ac_optarg ;;

	-bash | --bash | -b | --b)
            ac_prev=BASH ;;
        -bash=* | --bash=* | -b=* | --b=*)
            BASH=$ac_optarg ;;

	-command | --command | -c | --c)
	    ac_prev=COMMAND ;;
	-command=* | --command=* | -c=* | --c=*)
	    COMMAND=$ac_optarg ;;

	-makedepend | --makedepend | -md | --md)
	    ac_prev=MKDEPEND ;;
	-makedepend=* | --makedepend=* | -md=* | --md=*)
	    MKDEPEND=$ac_optarg ;;

	-make | --make | -m | --m)
	    ac_prev=MAKE ;;
	-make=* | --make=* | -m=* | --m=*)
	    MAKE=$ac_optarg ;;

        -odir | --odir)
            ac_prev=OUTDIR ;;
        -odir=* | --odir=*)
            OUTDIR=$ac_optarg ;;

	-ptracers | --ptracers | -ptr | --ptr)
	    ac_prev=PTRACERS_NUM ;;
	-ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
	    PTRACERS_NUM=$ac_optarg ;;

	-match | --match ) ac_prev=MATCH_CRIT ;;
	-match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;

	-j | --j) ac_prev=JOBS ;;
	-j=* | --j=*) JOBS=$ac_optarg ;;

	-clean | --clean)
	    CLEANUP=t ; DELDIR=t ;;

	-norun | --norun | -nr | --nr)
	    NORUN=t ;;
	-runonly | --runonly | -ro | --ro)
	    QUICK=t ; NOMAKE=t ;;
	-quick | --quick | -q | --q)
	    QUICK=t ;;
	-nogenmake | --nogenmake | -ng | --ng)
	    NOGENMAKE=t ;;
	-noclean | --noclean | -nc | --nc)
	    NOCLEAN=t ;;
	-nodepend | --nodepend | -nd | --nd)
	    NODEPEND=t ;;

	-postclean | --postclean | -pc | --pc)
	    POSTCLEAN=t ;;

	-mpi) MPI=t ;;

	-mth) MULTI_THREAD=t ;;

	-adm | -ad) ADM=t ;;

	-oad) OADM=t; NODEPEND=t ;;

	-ieee) IEEE=true ;;
	-noieee) IEEE= ;;
        -gsl) GSL=t ;;

	-verbose) verbose=2 ;;
	-debug) debug=1 ;;
	-quiet) verbose=0 ;;

	-deldir | -dd) DELDIR=t ;;

	-use_r4|-ur4) USE_R4=t ;;

	-ts) TS=t;;
	-papis) PAPIS=t;;
	-pcls) PCL=t;;

        -*)
            echo "Error: unrecognized option: "$ac_option
            usage
            ;;

        *)
            echo "Error: unrecognized argument: "$ac_option
            usage
            ;;

     esac

done

if test "x$QUICK" = xt ; then
    NOGENMAKE=t
    NOCLEAN=t
    NODEPEND=t
fi

#- setting for forward or ADM testing
if test "x$ADM" = xt ; then
    code_dir=code_ad
    inputdir=input_ad
    ref_outp="output_adm.txt"
    EXECUTABLE="mitgcmuv_ad"
elif test "x$OADM" = xt ; then
    code_dir=code_oad
    inputdir=input_oad
    ref_outp="output_oadm.txt"
    EXECUTABLE="mitgcmuv_ad"
else
    code_dir=code
    inputdir=input
    ref_outp="output.txt"
    EXECUTABLE="mitgcmuv"
fi

xx=`echo $TESTDIRS | awk '{print $1}'`
if test "x$TESTDIRS" = x ; then
    LIST=`scandirs results/$ref_outp`
elif test $xx = 'start_from' ; then
    xx=`echo $TESTDIRS | awk '{print $2}'`
    LIST=`scandirs results/$ref_outp | sed -n "/$xx/,$ p"`
else
    #- expand group of experiments:
    LIST=" "
    for xx in $TESTDIRS
    do
      case $xx in
	'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
		 LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
		 LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
		;;
	'tutorials')
		 LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
	*)	 LIST=${LIST}" "$xx ;;
      esac
    done
fi
#echo 'LIST='${LIST}'<'
#- skip dirs, remove duplicate and non-directory:
TESTDIRS=" "
count=0
for xx in $LIST
do
    yy=`echo $SKIPDIRS | grep -c $xx`
    if test $yy = 0 ; then
	if test -d $xx ; then
	    yy=`echo $TESTDIRS | grep -c $xx`
	    if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
	else count=1 ;
	    echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
	fi
    else
        if test $count = 1 ; then echo -n ", \"$xx\""
	else count=1 ; echo "" ;  echo -n " skip: \"$xx\""
	fi
    fi
done
if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
#echo 'TESTDIRS='${TESTDIRS}'<'

if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
    OPTFILE=$MITGCM_OF
fi

RUNLOG="run.tr_log"
OUTPUTFILE=$ref_outp
if test "x$COMMAND" = x ; then
    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
fi
if test "x$MPI" = xt ; then
    OUTPUTFILE="STDOUT.0000"
fi

echo "OK (COMMAND= $COMMAND )"

# set the Default List of output variables to be checked:
#  (use default or load experiment-specific list from file "tr_checklist")
# content : 1rst = main variable used to decide if it pass or FAIL
#         others = number of matching digits to be printed in summary.txt
if test "x$ADM" = x -a "x$OADM" = x; then
    DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
    EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
    LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
    ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
    EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf "  ."}}'`
else
    DEF_CHECK_LIST='Grad Cost Grad'
    EMPTY_RESULTS='.. ..'
    LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
fi

#  create the FORTRAN comparison code
if test -x tr_cmpnum ; then
    echo "skipping comparison code build"
else
    createcodelet
fi

#  build the mpack utility (if ADDRESSES = NONE, do it to test the build)
if test "x$ADDRESSES" = x ; then
    echo "skipping mpack build"
else
    build_mpack
fi

#  Create a uniquely named directory to store results
CMDLINE=$0
for xx in "$@" ; do nw=`echo $xx | wc -w`
    if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
                        else CMDLINE="$CMDLINE '$xx'" ; fi
done
#for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
MACH=`hostname`
UNAMEA=`uname -a`
DATE=`date +%Y%m%d`
BASE="tr_"$MACH"_"$DATE"_"
if test "x$OUTDIR" != x ; then
    BASE="tr_"$OUTDIR"_"$DATE"_"
else
    short_name=`hostname | sed 's/\..*$//'`
    BASE="tr_"$short_name"_"$DATE"_"
fi
DNUM=0
DRESULTS="$BASE$DNUM"
while test -e $DRESULTS ; do
    DNUM=$(( $DNUM + 1 ))
    DRESULTS="$BASE$DNUM"
done
mkdir $DRESULTS
RETVAL=$?
if test "x$RETVAL" != x0 ; then
    echo "ERROR: Can't create results directory \"./$DRESULTS\""
    exit 1
fi
SUMMARY="$DRESULTS/summary.txt"
start_date=`date`
echo $start_date > $SUMMARY
echo 'run:' $CMDLINE >> $SUMMARY
echo 'on :' $UNAMEA  >> $SUMMARY

of_path=
if test "x$OPTFILE" != xNONE ; then
    if test -r $OPTFILE ; then
	# get the path
	path=${OPTFILE%/*}
	if test "x$path" = x ; then
	    of_path=`pwd`
	else
	    of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
	fi
	file=${OPTFILE##*/}
	OPTFILE=$of_path/$file
	cp $OPTFILE $DRESULTS
	echo >> $SUMMARY
	echo "  OPTFILE=$OPTFILE" >> $SUMMARY
    else
	echo | tee $SUMMARY
	echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
	exit 1
    fi
else
    echo >> $SUMMARY
    echo "No \"OPTFILE\" was specified ; genmake2 found and uses:" >> $SUMMARY
    #-note: to be filled later after 1rst run
fi
echo
echo >> $SUMMARY
if test "x$ADM" = x -a "x$OADM" = x; then
    if [ $MATCH_CRIT -lt 10 ] ;
    then line_0="default  "$MATCH_CRIT ;
    else line_0="default "$MATCH_CRIT ; fi
       line_0="$line_0  ----T-----  ----S-----  ----U-----  ----V-----"
    line_1="G D M    c        m  s        m  s        m  s        m  s"
    line_2="E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
    line_3="N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
    line_4="2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  ."
    for ii in $PTRACERS_NUM ; do
	line_0="$line_0  --PTR 0"$ii"--"
	line_1="$line_1        m  s"
	line_2="$line_2  m  m  e  ."
	line_3="$line_3  i  a  a  d"
	line_4="$line_4  n  x  n  ."
    done
    echo "$line_0" | tee -a $SUMMARY
    echo "$line_1" | tee -a $SUMMARY
    echo "$line_2" | tee -a $SUMMARY
    echo "$line_3" | tee -a $SUMMARY
    echo "$line_4" | tee -a $SUMMARY
    echo " "       | tee -a $SUMMARY
else
    echo "ADJOINT=true" >> $SUMMARY
    echo >> $SUMMARY
    if [ $MATCH_CRIT -lt 10 ] ;
    then line_0="default     "$MATCH_CRIT ;
    else line_0="default    "$MATCH_CRIT ; fi
    echo "$line_0" | tee -a $SUMMARY
    cat << EOF | tee -a $SUMMARY
G D M    C  G
E p a R  o  r
N n k u  s  a
2 d e n  t  d

EOF
fi
echo "-------------------------------------------------------------------------------"

#  ...and each test directory...
for dir in $TESTDIRS ; do

    # set builddir & rundir:
    builddir="build"
    if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
    rundir="run"
    pfxdir="tr_$rundir"
    if test ! -d $dir/$rundir ; then
	rundir=$builddir
    fi
    CODE_DIR=$dir/$code_dir
    BUILD_DIR=$dir/$builddir

    #  Cleanup only!
    if test "x$CLEANUP" = xt ; then
	echo -n '  --- dir:' $BUILD_DIR ': '
	makeclean $BUILD_DIR
	(   cd $BUILD_DIR
	    rm -f $EXECUTABLE *.bak
	    rm -f genmake_state genmake_*optfile genmake.log
	    rm -f genmake.tr_log make.tr_log
	)
	if test -d $dir/$rundir/CVS ; then
            echo -n '  --- dir:' $dir/$rundir ': '
	    run_clean $dir/$rundir
	fi
	trdir=`( cd $dir ; find . -type d -name "$pfxdir.*" -print | sed 's/^.\///')`
	ttd=`echo $trdir | wc -w`
	if test $ttd != 0 ; then
	    echo '  --- rm dir:' $trdir
	    ( cd $dir ; rm -rf $trdir )
	fi
	continue
    fi

    #  Verify that the testdir exists and contains previous
    #  results in the correct location--or skip this directory!
    fout=$dir"/results/"$ref_outp
    if test ! -r $fout ; then
	echo "can't read \"$fout\" -- skipping $dir"
	continue
    fi

    # Check for specific files for particular type of run

    if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
	echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
	continue
    fi
    if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
	echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
	continue
    fi
    if test "x$MPI" = "xt" -a "x$MULTI_THREAD" = "xt" ; then
      retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/input/eedata.mth"`
      if test $retv != 0 ; then
	echo "input/eedata.mth tiling misfit -- skipping $dir"
	continue
      fi
    fi

    #  Check whether there are "extra runs" for this testdir
    extra_runs=
    if test "x$NORUN" = xf ; then
	ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
    fi
    #echo "ex_run_dirs='$ex_run_dirs'"
    for exd in $ex_run_dirs ; do
	name=`echo $exd | sed -e "s/$inputdir\.//"`
	refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
	outf="$dir/results/$refExOut"
        if test -f $outf -a -r $outf ; then
	  if test "x$MULTI_THREAD" = "xt" ; then
	    if test -r $dir"/"$exd"/eedata.mth" ; then
	      if test "x$MPI" = "xt" ; then
		retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/"$exd"/eedata.mth"`
		if test $retv = 0 ; then
		    extra_runs="$extra_runs $name"
		else
		    echo $exd"/eedata.mth tiling misfit -- skipping $dir"
		fi
	      else
		extra_runs="$extra_runs $name"
	      fi
	    #else echo $dir"/"$exd"/eedata.mth: not found"
	    fi
	  else
	    extra_runs="$extra_runs $name"
	  fi
	fi
    done

    echo
    if test "x$extra_runs" = "x" ; then
       echo "Experiment:  $dir"
    else
       echo "Experiment:  $dir ; extra_runs=$extra_runs"
    fi
    echo
    unset genmake makedepend make run
    results=$EMPTY_RESULTS

    #  Create an output dir for each OPTFILE/tdir combination
    locDIR=$DRESULTS"/"$dir
    mkdir $locDIR
    CDIR=`pwd`"/$locDIR"

    if test "x$NORUN" = xt ; then
	    run=N
	genmakemodel $dir/$builddir && genmake=Y \
	    && makeclean $dir/$builddir \
            && symlink_mpifiles $dir $code_dir $builddir \
	    && makedependmodel $dir/$builddir && makedepend=Y \
	    && makemodel $dir/$builddir && make=Y
    else
	genmakemodel $dir/$builddir && genmake=Y \
	    && makeclean $dir/$builddir \
            && symlink_mpifiles $dir $code_dir $builddir \
	    && makedependmodel $dir/$builddir && makedepend=Y \
	    && makemodel $dir/$builddir && make=Y \
	    && run_clean $dir/$rundir \
	    && linkdata $dir/$rundir $inputdir \
	    && runmodel $dir/$rundir && run=Y \
	    && results=`testoutput_run $dir $rundir $ref_outp`
    fi
    #echo "results='$results'"

	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
	echo 1>&2
 	echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
	touch $locDIR"/summary.txt"
	echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
	echo "MACH='$MACH'" >> $locDIR"/summary.txt"
	echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
	echo "DATE='$DATE'" >> $locDIR"/summary.txt"
	echo "tdir='$dir'" >> $locDIR"/summary.txt"
	if test "x$ADM" = xt ; then
	    head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt"
	    grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
				>> $locDIR"/summary.txt"
	fi

	for ex in $extra_runs ; do
	    unset run
            results=$EMPTY_RESULTS
	    #  reference output file
	    refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
	    #  Create an output dir for each OPTFILE/tdir.ex combination
	    locDIR=$DRESULTS"/"$dir"."$ex
	    mkdir $locDIR
	    CDIR=`pwd`"/$locDIR"
	    test ! -e "$dir/$pfxdir.$ex" && mkdir "$dir/$pfxdir.$ex"
	    run_clean $dir/$pfxdir.$ex
	    linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir
	    runmodel $dir/$pfxdir.$ex && run=Y \
	    && results=`testoutput_run $dir $pfxdir.$ex $refExOut`
	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
	    fres="$fres.$ex"
	    echo 1>&2
	    echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
	    touch $locDIR"/summary.txt"
	    echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
	    echo "MACH='$MACH'" >> $locDIR"/summary.txt"
	    echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
	    echo "DATE='$DATE'" >> $locDIR"/summary.txt"
	    echo "tdir='$dir.$ex'" >> $locDIR"/summary.txt"
            if test "x$ADM" = xt ; then
	        head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt"
	        grep -A3 'Seconds in section "ALL' $dir/$pfxdir.$ex/$OUTPUTFILE \
			           >> $locDIR"/summary.txt"
            fi
	    if test "x$POSTCLEAN" = xt ; then
		run_clean $dir/$pfxdir.$ex
	    fi
	done

    if test -f $DRESULTS"/"genmake_state ; then : ; else
	if test -f $dir/$builddir/Makefile ; then
	    mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
	    echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state
	    sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
	    echo " $mkOpt" >> $DRESULTS/genmake_state
	    if test "x$OPTFILE" = xNONE ; then
		eval $mkOpt
		sed "/^No \"OPTFILE\" was specified ; genmake2/a\  OPTFILE=${OPTFILE}"\
			$SUMMARY > tr_0.tmp_log
		RETVAL=$?
		if test "x$RETVAL" = x0 ; then rm -f $SUMMARY
			cp tr_0.tmp_log $SUMMARY
		else rm -f tr_0.tmp_log
		fi
	    fi
	    gmkLog=$dir/$builddir/genmake.log
	    grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1
	    RETVAL=$?
	    if test "x$RETVAL" = x0 ; then
		echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state
		sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \
		       | grep -v '^... compiler version ' > tr_1.tmp_log
		sed -n '1,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \
			>> $DRESULTS/genmake_state
		rm -f tr_1.tmp_log
	    fi
	fi
    fi
    #postclean $dir/$builddir
    if test "x$POSTCLEAN" = xt ; then
 	makeclean $dir/$builddir \
 	    && run_clean $dir/$rundir
    fi

    echo "-------------------------------------------------------------------------------"

done

printf "Start time:  " >> $SUMMARY
echo "$start_date" >> $SUMMARY
printf "End time:    " >> $SUMMARY
date >> $SUMMARY

#  If addresses were supplied and mpack built successfully, then try
#  to send email using mpack.
if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
    echo "No results email was sent."
else
    if test "x$HAVE_MPACK" = xt ; then
	tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
	    && gzip $DRESULTS".tar" \
	    && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
	RETVAL=$?
	if test "x$RETVAL" != x0 ; then
	    echo
	    echo "Warning: The tar, gzip, & mpack step failed.  Please send email"
	    echo "  to <MITgcm-support@mitgcm.org> for help.  You may copy the "
	    echo "  summary of results from the directory \"$DRESULTS\"."
	    echo
	else
	    echo
	    echo "An email containing results was sent to the following addresses:"
	    echo "  \"$ADDRESSES\""
	    echo
	    test -f $DRESULTS".tar"  &&  rm -f $DRESULTS".tar"
	    test -f $DRESULTS".tar.gz"  &&  rm -f $DRESULTS".tar.gz"
	fi
    fi
fi

if test "x$QUICK" = xf -a "x$NORUN" = xf ; then
    rm -f tr_cmpnum.c tr_cmpnum
fi

if test "x$CLEANUP" != xt ; then
    cat $SUMMARY | sed 's/ \.  \.  \.  \.  \.  \.  \.  \.  \.  \.  \.  \. //'
    if test -e tr_out.txt ; then
        mv tr_out.txt tr_out.txt.old
    fi
    cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
fi

if test "x$DELDIR" = xt ; then
    rm -rf $DRESULTS
fi
