--- MITgcm/verification/testreport	2005/06/07 22:21:47	1.68
+++ MITgcm/verification/testreport	2006/06/07 01:20:26	1.79
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.68 2005/06/07 22:21:47 edhill Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.79 2006/06/07 01:20:26 heimbach Exp $
 #  $Name:  $
 #
 
@@ -27,6 +27,8 @@
     echo "                             (DEF=\"make output.txt\")"
     echo "  (-m|-make) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
+    echo "  (-odir) STRING           used to build output directory name"
+    echo "                             (DEF=\"hostname\")"
     echo "  (-ptracers|-ptr) STRING  specify which ptracers to test"
     echo "                             (DEF=\"1 2 3 4 5\")"
     echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
@@ -123,6 +125,16 @@
         fi
         return 99
     fi
+    has_nan=`cat tmp1.txt | grep -i nan | wc -l`
+    if [ $has_nan -gt 0  ] ; then
+	echo testoutput_for_prop: output.txt contains $has_nan NaN values  1>&2
+	return 99
+    fi
+    has_inf=`cat tmp1.txt | grep -i inf | wc -l`
+    if [ $has_inf -gt 0  ] ; then
+	echo testoutput_for_prop: output.txt contains $has_inf Inf values  1>&2
+	return 99
+    fi
     if [ $debug -gt 0 ]; then
 	echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
     fi
@@ -169,10 +181,10 @@
 
 testoutput_ad()
 {
-    grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $5}' > t05.txt
-    grep $3 $1/$2/output.txt_adm | awk '{print NR " " $5}' > t15.txt
-    grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $6}' > t06.txt
-    grep $3 $1/$2/output.txt_adm | awk '{print NR " " $6}' > t16.txt
+    grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt
+    grep $3 $1/$2/output_adm.txt | awk '{print NR " " $5}' > t15.txt
+    grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt
+    grep $3 $1/$2/output_adm.txt | awk '{print NR " " $6}' > t16.txt
     join t05.txt t15.txt > t5.txt
     join t06.txt t16.txt > t6.txt
     echo "-1" >> t5.txt
@@ -292,7 +304,6 @@
 		command="$command --mods=../code"
 	    else
 		command="$command --mods=../code_ad"
-		command="$command -adof=../../../tools/adjoint_options/adjoint_staf"
 	    fi
 	    if test "x$OPTFILE" != xNONE ; then
 		command="$command --optfile=$OPTFILE"
@@ -503,14 +514,19 @@
     #  (where "$COMMAND" is relative to "directory")
     (
 	cd $1
-	printf 'runmodel ... ' 1>&2
+	printf 'runmodel in %s ...' $1 1>&2
 	# make output.txt
 	echo
 	rm -f run.log
+        if test $OUTPUTFILE -ot $EXECUTABLE ; then
+	   ( eval $COMMAND ) > run.log 2>&1
+	   RETVAL=$?
+	else
+           echo " $OUTPUTFILE is up to date " > run.log 2>&1
+	   RETVAL=0
+	fi
 	# echo "COMMAND='$COMMAND'"
 	# echo "pwd='"`pwd`"'"
-	( eval $COMMAND ) > run.log 2>&1
-	RETVAL=$?
 	if test "x$RETVAL" = x0 ; then
 	    tail run.log
 	    echo successful 1>&2
@@ -518,7 +534,7 @@
 	    # if test "x$ADM" = x ; then
 	    #   cp output.txt $CDIR"/output.txt"
 	    # else
-	    #   cp output.txt_adm $CDIR"/output.txt_adm"
+	    #   cp output_adm.txt $CDIR"/output_adm.txt"
 	    # fi
             if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 0
@@ -668,6 +684,7 @@
 fi
 JOBS=
 MPI=f
+OUTDIR=
 DELDIR=
 
 ADM=
@@ -724,6 +741,11 @@
 	-make=* | --make=*)
 	    MAKE=$ac_optarg ;;
 
+        -odir | --odir)
+            ac_prev=OUTDIR ;;
+        -odir=* | --odir=*)
+            OUTDIR=$ac_optarg ;;
+
 	-ptracers | --ptracers | -ptr | --ptr)
 	    ac_prev=PTRACERS_NUM ;;
 	-ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
@@ -733,7 +755,7 @@
 	-j=*) JOBS=$ac_optarg ;;
 
 	-clean | --clean)
-	    CLEANUP=t ;;
+	    CLEANUP=t ; DELDIR=t ;;
 
 	-quick | --quick | -q | --q)
 	    QUICK=t ;;
@@ -788,15 +810,24 @@
     OPTFILE=$MITGCM_OF
 fi
 
+if test "x$ADM" = xt ; then
+    EXECUTABLE="mitgcmuv_ad"
+    OUTPUTFILE="output_adm.txt"
+else
+    EXECUTABLE="mitgcmuv"
+    OUTPUTFILE="output.txt"
+fi
+
 if test "x$ADM" = xt -a "x$COMMAND" = x ; then
-    COMMAND="./mitgcmuv_ad > output.txt_adm 2>&1"
+    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
 fi
 
 if test "x$COMMAND" = x ; then
-    COMMAND="$MAKE output.txt"
+    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
 fi
 
-echo "OK"
+#echo "OK"
+echo "OK (COMMAND= $COMMAND )"
 
 #  create the FORTRAN comparison code
 createcodelet
@@ -813,6 +844,9 @@
 UNAMEA=`uname -a`
 DATE=`date +%Y%m%d`
 BASE="tr_"$MACH"_"$DATE"_"
+if test "x$OUTDIR" != x ; then
+    BASE="tr_"$OUTDIR"_"$DATE"_"
+fi
 DNUM=0
 DRESULTS="$BASE$DNUM"
 while test -e $DRESULTS ; do
@@ -915,7 +949,7 @@
     if test "x$ADM" = x ; then
 	fout=$dir"/results/output.txt"
     else
-	fout=$dir"/results_ad/output.txt_adm"
+	fout=$dir"/results_ad/output_adm.txt"
     fi
     if test ! -r $fout ; then
 	echo "can't read \"$fout\" -- skipping $dir"
@@ -970,7 +1004,7 @@
     echo "Experiment:  $dir"
     echo
     unset genmake makedepend make run
-    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
 
     #  Create an output dir for each OPTFILE/tdir combination
     rel_CDIR=$DRESULTS"/"$dir
@@ -1002,6 +1036,12 @@
 	echo "tdir='$dir'" >> $CDIR"/summary.txt"
 
 	for ex in $extra_runs ; do
+	    unset run
+	    results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+	    #  Create an output dir for each OPTFILE/tdir.ex combination
+	    rel_CDIR=$DRESULTS"/"$dir"."$ex
+	    mkdir $rel_CDIR
+	    CDIR=`pwd`"/$rel_CDIR"
 	    test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
 	    for ldir in input.$ex input ; do
 		(
@@ -1021,19 +1061,9 @@
 	    done
 	    ldir=build
 	    (
-		cd "$dir/$ldir" > /dev/null 2>&1
-		ls -1 Makefile *.[fFhco] mitgcmuv  2>/dev/null \
-		    | sed -e 's|^CVS$||g' | sed -e 's|^output.txt$||g'
-	    ) > tr_exrun_links
-	    (
-		cd "$dir/tr_run.$ex"
-		    cat ../../tr_exrun_links | while read i ; do
-			if test ! "x$i" = x ; then
-			    test ! -r $i  &&  ln -s "../"$ldir"/"$i $i
-			fi
-		    done
+		cd $dir/tr_run.$ex
+		test ! -e mitgcmuv  &&  ln -s "../"$ldir"/"mitgcmuv .
 	    )
-	    test -e tr_exrun_links  &&  rm -f tr_exrun_links
 	    runmodel $dir/tr_run.$ex && run=Y \
 		&& results=`testoutput $dir tr_run.$ex "."$ex`
 	    fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
@@ -1045,7 +1075,7 @@
 	    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
 	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
 	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-	    echo "tdir='$dir'" >> $CDIR"/summary.txt"
+	    echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
 	done
     else
 	fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
@@ -1057,6 +1087,8 @@
 	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
 	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
 	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+        grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
+			   >> $CDIR"/summary.txt"
     fi
 
     postclean $dir/$builddir