--- MITgcm/verification/testreport	2007/09/20 03:17:32	1.102
+++ MITgcm/verification/testreport	2008/06/26 14:41:33	1.116
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.102 2007/09/20 03:17:32 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.116 2008/06/26 14:41:33 mlosch Exp $
 #  $Name:  $
 #
 
@@ -15,9 +15,12 @@
     echo "  (-mpi)                   compile and run using MPI"
     echo "  (-ieee|-noieee)          if possible, use IEEE compiler flags"
     echo "                             (DEF=\"-ieee\")"
+    echo "  (-gsl)                   compile with \"-gsl\" flag"
     echo "  (-of=|-optfile=)STRING   list of optfiles to use"
     echo "  (-a|-addr) STRING        list of email recipients"
-    echo "                             (DEF=\"edhill@mitgcm.org\")"
+    echo "                             (DEF=\"\" no email is sent)"
+    echo "  (-mpd|-mpackdir) DIR     location of the mpack utility"
+    echo "                             (DEF=\"../tools/mpack-1.6\")"
     echo "  (-t|-tdir) STRING        list of group and/or exp. dirs to test"
     echo "                             (recognized groups: basic, tutorials)"
     echo "                             (DEF=\"\" which test all)"
@@ -64,7 +67,8 @@
 build_mpack()
 {
     printf "building the mpack utility...  "
-    if test ! -x "$MPACKDIR/mpack" ; then
+    MPACK="$MPACKDIR/mpack"
+    if test ! -x $MPACK ; then
 	if test ! -d $MPACKDIR ; then
 	    echo
 	    echo "Error: can't find \"$MPACKDIR\""
@@ -73,10 +77,10 @@
 	    echo
 	    HAVE_MPACK=f
 	fi
-	printf "building mpack...  "
 	if test "x$CC" = x ; then
 	    export CC=cc
 	fi
+	printf "building mpack (using CC=$CC)...  "
 	( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
 	RETVAL=$?
 	if test "x$RETVAL" != x0 ; then
@@ -87,11 +91,12 @@
 	else
 	    rm -f tr_build_mpack.out
 	    HAVE_MPACK=t
+	    echo "done"
 	fi
     else
 	HAVE_MPACK=t
+	echo "already exist"
     fi
-    echo "OK"
 }
 
 testoutput_var()
@@ -174,37 +179,6 @@
     return $digits_of_similarity
 }
 
-dashnum()
-{
-    # dashnum n1 n2 n3 ...
-    #
-    #  print numbers using %3i format or "--" if number = 99
-
-    for num in $@ ; do
-	if [ $num = 99 ]; then
-	    printf ' --'
-	else
-	    printf '%3i' $num
-	fi
-    done
-}
-
-testoutput_ad()
-{
-    grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt
-    grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $5}' > t15.txt
-    grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt
-    grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $6}' > t16.txt
-    join t05.txt t15.txt > t5.txt
-    join t06.txt t16.txt > t6.txt
-    echo "-1" >> t5.txt
-    echo "-1" >> t6.txt
-    digits_5=`./tr_cmpnum < t5.txt`
-    digits_6=`./tr_cmpnum < t6.txt`
-    dashnum $digits_5 $digits_6
-    rm -f t[01][56].txt t[56].txt
-}
-
 check_for_add_mon_output()
 {
     # Check for additional types of monitor output
@@ -235,7 +209,7 @@
     # testoutput_run directory subdir reference_output
     #
     #  test output from 1 run in "directory"
-#   if test "x$ADM" = x ; then
+# --> same processing for adjoint & forward test
         # default list of output variables to be checked:
         #  1rst : main variable used to decide if it pass or FAIL
         #  others : number of matching digits to be printed in summary.txt
@@ -252,7 +226,7 @@
 	 #echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
  	  if test "x$tst" != xt ; then listVar=`echo "$listVar" | sed "s/ pt$ii..//g"` ; fi
         done
-        tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){t+=($i==$1)}; print t }'`
+        tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
         if test $tst != 1 ; then
           if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
                  else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
@@ -330,10 +304,7 @@
         else
 	  echo $allargs
         fi 
-
-#   else
-#	testoutput_ad $1 $2 "precision_grdchk_result"
-#   fi
+# <-- same processing for adjoint & forward test
 }
 
 genmakemodel()
@@ -361,6 +332,9 @@
 	    if test "x$IEEE" != x ; then
 		command="$command -ieee"
 	    fi
+	    if test "x$GSL" = xt ; then
+		command="$command -gsl"
+	    fi
 	    if test "x$MPI" = xt ; then
 		command="$command -mpi"
 	    fi
@@ -433,6 +407,7 @@
 	    find . -type l -exec rm {} \;
 	    rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
 	    rm -rf mnc_test_*
+ 	    rm -f *_MIT_CE_000.opt0000 costfunction*0000
 	    echo successful 1>&2
 	    exit 0
 	)
@@ -503,7 +478,7 @@
     CODE_DIR=$dir/$code_dir
     
     # These are files that should replace their counter-part when using -mpi
-    MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
+    MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`
 
     #  Is this an MPI run?
     if test "x$MPI" = xt ; then
@@ -561,8 +536,9 @@
 		    if test -h eedata ; then rm -f eedata ; fi
 		fi
 	    fi
+	    prevDir='NONE'
 	    for ldir in $* ; do
-		if test -d "../"$ldir ; then
+		if test -d "../"$ldir -a $ldir != $prevDir ; then
                     printf 'ldir='${ldir} 1>&2
 		    files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
 		    for i in $files ; do
@@ -578,6 +554,7 @@
 		    fi
 		    printf ' ; ' 1>&2
 		fi
+		prevDir=$ldir
 	    done
 	)
     fi
@@ -594,6 +571,10 @@
 	printf 'runmodel in %s ...' $1 1>&2
 	# make output.txt
 	echo
+	if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
+	    diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
+	    outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; rm -f run.log ; fi
+	fi
         if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
 	    echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_00
 	    ln -sf "../"$builddir"/"$EXECUTABLE .
@@ -648,7 +629,7 @@
 {
     # create codelet for comparing model output
 
-    printf "creating the comparison code...  "
+    printf "creating the comparison code (using CC=$CC)...  "
     cat > tr_cmpnum.c <<EOF
 #include <stdio.h>
 #include <math.h>
@@ -745,14 +726,12 @@
 debug=0
 verbose=1
 clean=0
-expts=''
-# ieee=1
 
 IEEE=true
 if test "x$MITGCM_IEEE" != x ; then
     IEEE=$MITGCM_IEEE
 fi
-
+GSL=f
 
 CLEANUP=f
 QUICK=f
@@ -768,7 +747,7 @@
 SKIPDIRS=
 MPACKDIR="../tools/mpack-1.6"
 HAVE_MPACK=
-MPACK="$MPACKDIR/mpack"
+MPACK=
 COMMAND=
 if test "x$MAKE" = x ; then
     MAKE=make
@@ -817,6 +796,10 @@
             ac_prev=ADDRESSES ;;
         -addr=* | --addr=*)
             ADDRESSES=$ac_optarg ;;
+        -mpackdir | --mpackdir | -mpd | --mpd)
+            ac_prev=MPACKDIR ;;
+        -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
+            MPACKDIR=$ac_optarg ;;
 
 	-tdir | --tdir | -t | --t)
             ac_prev=TESTDIRS ;;
@@ -882,6 +865,7 @@
 
 	-ieee) IEEE=true ;;
 	-noieee) IEEE= ;;
+        -gsl) GSL=t ;;
 
 	-verbose) verbose=2 ;;
 	-debug) debug=1 ;;
@@ -918,22 +902,18 @@
 #- setting for forward or ADM testing
 if test "x$ADM" = xt ; then
     code_dir=code_ad
-    input_dirs='input_ad input'
+    inputdir=input_ad
     ref_outp="output_adm.txt"
     EXECUTABLE="mitgcmuv_ad"
 else
     code_dir=code
-    input_dirs='input'
+    inputdir=input
     ref_outp="output.txt"
     EXECUTABLE="mitgcmuv"
 fi
 
 if test "x$TESTDIRS" = x ; then
-#   if test "x$ADM" = xt ; then
-#	LIST=`scandirs results_ad/$ref_outp`
-#   else
-	LIST=`scandirs results/$ref_outp`
-#   fi
+    LIST=`scandirs results/$ref_outp`
 else
     #- expand group of experiments:
     LIST=" "
@@ -985,7 +965,6 @@
     OUTPUTFILE="STDOUT.0000"
 fi
 
-#echo "OK"
 echo "OK (COMMAND= $COMMAND )"
 
 # set the Default List of output variables to be checked:
@@ -999,7 +978,7 @@
     ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
     EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf "  ."}}'`
 else
-    DEF_CHECK_LIST='Cost Cost Grad'
+    DEF_CHECK_LIST='Grad Cost Grad'
     EMPTY_RESULTS='.. ..'
     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
 fi
@@ -1007,14 +986,16 @@
 #  create the FORTRAN comparison code
 createcodelet
 
-#  build the mpack utility
-if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
+#  build the mpack utility (if ADDRESSES = NONE, do it to test the build)
+if test "x$ADDRESSES" = x ; then
     echo "skipping mpack build"
 else
     build_mpack
 fi
 
 #  Create a uniquely named directory to store results
+CMDLINE=$0
+for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
 MACH=`hostname`
 UNAMEA=`uname -a`
 DATE=`date +%Y%m%d`
@@ -1035,9 +1016,10 @@
     exit 1
 fi
 SUMMARY="$DRESULTS/summary.txt"
-printf "Start time:  " >> $SUMMARY
 start_date=`date`
 echo $start_date > $SUMMARY
+echo 'run:' $CMDLINE >> $SUMMARY
+echo 'on :' $UNAMEA  >> $SUMMARY
 
 of_path=
 if test "x$OPTFILE" != xNONE ; then
@@ -1096,8 +1078,8 @@
     echo "ADJOINT=true" >> $SUMMARY
     echo >> $SUMMARY
     if [ $MATCH_CRIT -lt 10 ] ; 
-    then line_0="default  "$MATCH_CRIT ; 
-    else line_0="default "$MATCH_CRIT ; fi
+    then line_0="default     "$MATCH_CRIT ; 
+    else line_0="default    "$MATCH_CRIT ; fi
     echo "$line_0" | tee -a $SUMMARY
     cat << EOF | tee -a $SUMMARY
 G D M    C  G
@@ -1111,15 +1093,25 @@
 #  ...and each test directory...
 for dir in $TESTDIRS ; do
     
+    # set builddir & rundir:
+    builddir="build"
+    if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
+    rundir="run"
+    if test ! -d $dir/$rundir ; then
+	rundir=$builddir
+    fi
+    CODE_DIR=$dir/$code_dir
+    BUILD_DIR=$dir/$builddir
+
     #  Cleanup only!
     if test "x$CLEANUP" = xt ; then
-	if test -r $dir/build/Makefile ; then
-            echo '  ------  clean dir:' $dir/build
-	    ( cd $dir/build ; make CLEAN )
-	fi
-	if test -d $dir/run/CVS ; then
-            echo '  ------  clean dir:' $dir/run
-	    run_clean $dir/run
+	if test -r $BUILD_DIR/Makefile ; then
+            echo '  ------  clean dir:' $dir/$builddir
+	    ( cd $BUILD_DIR ; make CLEAN )
+	fi
+	if test -d $dir/$rundir/CVS ; then
+            echo '  ------  clean dir:' $dir/$rundir
+	    run_clean $dir/$rundir
 	fi
 	(
 	    cd $dir
@@ -1130,12 +1122,7 @@
 
     #  Verify that the testdir exists and contains previous 
     #  results in the correct location--or skip this directory!
-    fout=
-    if test "x$ADM" = x ; then
-	fout=$dir"/results/"$ref_outp
-    else
-	fout=$dir"/results_ad/"$ref_outp
-    fi
+    fout=$dir"/results/"$ref_outp
     if test ! -r $fout ; then
 	echo "can't read \"$fout\" -- skipping $dir"
 	continue
@@ -1146,15 +1133,6 @@
 
     # Check for additional types of monitor output
 
-    builddir="build"
-    if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
-    rundir="run"
-    if test ! -d $dir/$rundir ; then
-	rundir=$builddir
-    fi
-    CODE_DIR=$dir/$code_dir
-    BUILD_DIR=$dir/$builddir
-
     if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
 	echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
 	continue
@@ -1166,24 +1144,23 @@
 
     #  Check whether there are "extra runs" for this testdir
     extra_runs=
-    if test "x$ADM" = x ; then
-	ex_run_dirs=`( cd $dir ; echo input.* )`
-	#echo "ex_run_dirs='$ex_run_dirs'"
-	for exd in $ex_run_dirs ; do
-	    name=`echo $exd | sed -e 's/input.//g'`
-	    outf="$dir/results/output.$name.txt"
-            if test -f $outf -a -r $outf ; then
-		if test "x$MULTI_THREAD" = "xt" ; then 
-		    if test -r $dir"/"$exd"/eedata.mth" ; then
-			extra_runs="$extra_runs $name"
-		    #else echo $dir"/"$exd"/eedata.mth: not found"
-		    fi
-		else
+    ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
+    #echo "ex_run_dirs='$ex_run_dirs'"
+    for exd in $ex_run_dirs ; do
+	name=`echo $exd | sed -e "s/$inputdir\.//"`
+	refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
+	outf="$dir/results/$refExOut"
+        if test -f $outf -a -r $outf ; then
+	    if test "x$MULTI_THREAD" = "xt" ; then 
+		if test -r $dir"/"$exd"/eedata.mth" ; then
 		    extra_runs="$extra_runs $name"
+		#else echo $dir"/"$exd"/eedata.mth: not found"
 		fi
+	    else
+		extra_runs="$extra_runs $name"
 	    fi
-	done
-    fi
+	fi
+    done
 
     echo "-------------------------------------------------------------------------------"
     echo
@@ -1212,13 +1189,12 @@
             && symlink_mpifiles $dir $code_dir $builddir \
 	    && makedependmodel $dir/$builddir && makedepend=Y \
 	    && makemodel $dir/$builddir && make=Y \
-	    && linkdata $dir/$rundir $input_dirs \
+	    && linkdata $dir/$rundir $inputdir input \
 	    && runmodel $dir/$rundir && run=Y \
 	    && results=`testoutput_run $dir $rundir $ref_outp`
     fi
     
     echo
-#   if test "x$ADM" = x ; then
 	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	echo
  	echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
@@ -1228,19 +1204,26 @@
 	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
 	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
 	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+	if test "x$ADM" = xt ; then
+	    head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
+	    grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
+				>> $CDIR"/summary.txt"
+	fi
 
 	for ex in $extra_runs ; do
 	    unset run
             results=$EMPTY_RESULTS
+	    #  reference output file
+	    refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
 	    #  Create an output dir for each OPTFILE/tdir.ex combination
 	    rel_CDIR=$DRESULTS"/"$dir"."$ex
 	    mkdir $rel_CDIR
 	    CDIR=`pwd`"/$rel_CDIR"
 	    test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
- 	    run_clean $dir/tr_run.$ex
-            linkdata $dir/tr_run.$ex input.$ex input
+	    run_clean $dir/tr_run.$ex
+	    linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
 	    runmodel $dir/tr_run.$ex && run=Y \
-		&& results=`testoutput_run $dir tr_run.$ex "output.${ex}.txt"`
+	    && results=`testoutput_run $dir tr_run.$ex $refExOut`
 	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	    fres="$fres.$ex"
 	    echo
@@ -1251,26 +1234,15 @@
 	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
 	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
 	    echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
+            if test "x$ADM" = xt ; then
+	        head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
+	        grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
+			           >> $CDIR"/summary.txt"
+            fi
 	    if test "x$POSTCLEAN" = xt ; then
 		run_clean $dir/tr_run.$ex
 	    fi
 	done
-#   else
-#	fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
-#	fres=$fres"$results   $dir"
-#	echo
-#	echo "$fres" >> $SUMMARY
-#	touch $CDIR"/summary.txt"
-#	echo "fresults='$fres'" >> $CDIR"/summary.txt"
-#	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
-#	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
-#	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-#	echo "tdir='$dir'" >> $CDIR"/summary.txt"
-#   fi
-    if test "x$ADM" = xt ; then
-	grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
-			   >> $CDIR"/summary.txt"
-    fi
 
     #postclean $dir/$builddir
     if test "x$POSTCLEAN" = xt ; then
@@ -1283,7 +1255,7 @@
 done
 
 printf "Start time:  " >> $SUMMARY
-echo $start_date >> $SUMMARY
+echo "$start_date" >> $SUMMARY
 printf "End time:    " >> $SUMMARY
 date >> $SUMMARY
 
@@ -1321,7 +1293,7 @@
     if test -e tr_out.txt ; then
         mv tr_out.txt tr_out.txt.old
     fi
-    cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/  \.//g' > tr_out.txt
+    cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
 fi
 
 if test "x$DELDIR" = xt ; then