--- MITgcm/verification/testreport	2010/03/08 16:36:20	1.136
+++ MITgcm/verification/testreport	2010/11/05 16:45:02	1.148
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.136 2010/03/08 16:36:20 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.148 2010/11/05 16:45:02 jmc Exp $
 #  $Name:  $
 #
 
@@ -33,6 +33,7 @@
     echo "  (-adm|-ad)               perform an adjoint run"
     echo "  (-oad)                   perform an OpenAD adjoint run"
     echo "  (-command|-c) STRING     command to run"
+    echo "  (-makedepend|-md) STRING command to use for \"makedepend\""
     echo "  (-make|-m) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
     echo "  (-odir) STRING           used to build output directory name"
@@ -40,7 +41,7 @@
     echo "  (-ptracers|-ptr) STRING  specify which ptracers to test"
     echo "                             (DEF=\"1 2 3 4 5\")"
     echo "  (-match) NUMBER          Matching Criteria (number of digits)"
-    echo "                             (DEF=\"12\")"
+    echo "                             (DEF=\"$MATCH_CRIT\")"
     echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\""
     echo "  (-norun|-nr)             skip the \"runmodel\" stage (stop after make)"
@@ -231,7 +232,7 @@
           listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
         fi
 	if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
-	echo "listVar='$listVar'" > $CDIR"/summary.txt"
+	echo "listVar='$listVar'" > $locDIR"/summary.txt"
         allargs=""
         for xx in $listVar
         do
@@ -317,6 +318,9 @@
 	(
 	    cd $1;
 	    command="$GENMAKE2  -ds -m $MAKE"
+	    if test "x$MKDEPEND" != x ; then
+		command="$command -makedepend=$MKDEPEND"
+	    fi
 	    if test "x$ADM" = xt ; then
 		command="$command --mods=../code_ad"
 	    elif test "x$OADM" = xt ; then
@@ -353,19 +357,18 @@
 		command="$command -pcls"
 	    fi
 	    fi
-	    printf 'genmake ... ' 1>&2
-	    # $command > make.log 2>&1
-	    eval $command > make.log 2>&1
+	    printf 'genmake ... '
+	    eval $command > genmake.tr_log 2>&1
 	    RETVAL=$?
 	    #  Reduce the size of the testing emails!
 	    head -100 Makefile > $CDIR/Makefile_head
 	    if test "x$RETVAL" != x0 ; then
-		tail make.log
-		echo "genmakemodel: genmake failed" 1>&2
-		cp genmake_* make.log $CDIR
+		tail genmake.tr_log
+		echo "genmakemodel: genmake failed"
+		cp genmake.log genmake_* genmake.tr_log $CDIR
 		return 1
 	    else
-		echo "successful" 1>&2
+		echo "successful"
 	    fi
 	)
     fi
@@ -374,6 +377,7 @@
 makeclean()
 {
     # makeclean directory
+    if test "x$NODEPEND" = xf ; then rm -f $1/make.tr_log ; fi
     if test "x$NOCLEAN" = xt ; then
 	echo "make Clean skipped!"
     else
@@ -381,16 +385,17 @@
 	    cd $1;
 	    #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
 	    if test -r Makefile ; then
-		printf 'clean build-dir: make Clean ... ' 2>&1
-		$MAKE Clean >> make.log 2>&1
+		printf 'clean build-dir: make Clean ... '
+		rm -f Makefile.old
+		$MAKE Clean >> make.tr_log 2>&1
 		RETVAL=$?
 		if test "x$RETVAL" != x0 ; then
-		    tail make.log
-		    echo "makeclean: \"make Clean\" failed" 1>&2
-		    cp make.log $CDIR"/make.log"
+		    tail make.tr_log
+		    echo "makeclean: \"make Clean\" failed"
+		    cp make.tr_log genmake.log genmake.tr_log $CDIR
 		    return 1
 		fi
-		echo successful 1>&2
+		echo successful
 	    else
 		echo ''
 	    fi
@@ -407,16 +412,16 @@
     else
 	(
 	    cd $1;
-	    printf 'clean run-dir ... ' 2>&1
+	    printf 'clean run-dir ... '
 	    # part of what is done after "make clean" when doing "make CLEAN"
 	    find . -name "*.meta" -exec rm {} \;
 	    find . -name "*.data" -exec rm {} \;
 	    find . -name "fort.*" -exec rm {} \;
 	    find . -type l -exec rm {} \;
-	    rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
+	    rm -f $EXECUTABLE $RUNLOG *.txt STD* *diagnostics.log datetime
 	    rm -rf mnc_test_*
  	    rm -f *_MIT_CE_000.opt0000 costfunction*0000
-	    echo successful 1>&2
+	    echo successful
 	    exit 0
 	)
     fi
@@ -430,16 +435,16 @@
     else
 	(
 	    cd $1;
-	    printf 'make depend ... ' 1>&2
-	    $MAKE depend >> make.log 2>&1
+	    printf 'make depend ... '
+	    $MAKE depend >> make.tr_log 2>&1
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
-		tail make.log
-		echo "makedependmodel: make depend failed" 1>&2
-		cp make.log $CDIR"/make.log"
+		tail make.tr_log
+		echo "makedependmodel: make depend failed"
+		cp make.tr_log genmake.log genmake.tr_log $CDIR
 		return 1
 	    else
-		echo successful 1>&2
+		echo successful
 	    fi
 	)
     fi
@@ -460,27 +465,27 @@
     else
 	cd $1;
 	if test -r Makefile ; then
-	    printf 'make ... ' 1>&2
+	    printf 'make ... '
 	    if test "x$ADM" = xt ; then
-		$MAKE adall >> make.log 2>&1
+		$MAKE adall >> make.tr_log 2>&1
 	    elif test "x$OADM" = xt ; then
-		$MAKE adAll >> make.log 2>&1
+		$MAKE adAll >> make.tr_log 2>&1
             else
 		if test "x$JOBS" = x ; then
-		    $MAKE >> make.log 2>&1
+		    $MAKE >> make.tr_log 2>&1
 		else
-		    $MAKE -j $JOBS >> make.log 2>&1
+		    $MAKE -j $JOBS >> make.tr_log 2>&1
 		fi
 	    fi
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
-		tail make.log
-		echo failed 1>&2
-		cp make.log $CDIR"/make.log"
+		tail make.tr_log
+		echo failed
+		cp make.tr_log genmake.log genmake.tr_log $CDIR
 		rm -f $EXECUTABLE
 		return 1
 	    else
-		echo successful 1>&2
+		echo successful
 	    fi
 	fi
     fi
@@ -543,6 +548,7 @@
     if test -d $1 ; then
 	(
 	    cd $1 ; shift
+	    echo 'linkdata from dirs:' $*
             inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
 	    for xx in $inpMPI ; do
 	      if test -r "../"$1"/"$xx ; then
@@ -553,7 +559,7 @@
 		    if test -h $yy ; then rm -f $yy ; fi
 		    if test ! -r $yy ; then
 			ln -sf "../"$1"/"$xx $yy ;
-			printf "$xx " 1>&2
+			printf " $xx" 1>&2
 		    fi
 		else
 		# not mpi test: remove symbolic link
@@ -568,7 +574,7 @@
 		    if test -h eedata ; then rm -f eedata ; fi
 		    if test ! -r eedata ; then
 			ln -sf "../"$1"/eedata.mth" eedata ;
-			printf 'eedata.mth ' 1>&2
+			printf ' eedata.mth' 1>&2
 		    fi
 		else
 		# not multi-threaded test: remove eedata symbolic link
@@ -578,7 +584,7 @@
 	    prevDir='NONE'
 	    for ldir in $* ; do
 		if test -d "../"$ldir -a $ldir != $prevDir ; then
-                    printf 'ldir='${ldir} 1>&2
+                    printf " ldir=${ldir}:" 1>&2
 		    files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
 		    for i in $files ; do
 			if test ! -d "../"$ldir/$i ; then
@@ -588,10 +594,12 @@
 			    fi
 			fi
 		    done
+		    printf ' ;' 1>&2
 		    if test -x "../"$ldir"/"prepare_run ; then
-			"../"$ldir"/"prepare_run
+			"../"$ldir"/"prepare_run 1>&2
+		    else
+			echo '' 1>&2
 		    fi
-		    printf ' ; ' 1>&2
 		fi
 		prevDir=$ldir
 	    done
@@ -607,9 +615,7 @@
     #  (where "$COMMAND" is relative to "directory")
     (
 	cd $1
-	printf 'runmodel in %s ...' $1 1>&2
-	# make output.txt
-	echo
+	printf 'runmodel in %s ... ' $1
 	if test -L $EXECUTABLE ; then
 	  if test -x "../"$builddir"/"$EXECUTABLE ; then
 	    diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
@@ -650,20 +656,23 @@
 	fi
 	rm -f run.log_tmp
 	if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
-	    tail $RUNLOG
-	    echo successful 1>&2
+	    echo successful
+	    printf '=> output from running in %s :\n' $1 1>&2
+	    tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
 	    # === Reduce the size of the testing emails!
 	    #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
             if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 0
 	elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then
 	    #-- for some weird cases (run is finihed but with error code)
-	    tail $RUNLOG
-	    echo finished with error '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
+	    echo 'finished with error (run:' $RETVAL ' end:' $ENDVAL ')'
+	    printf '=> output from running in %s :\n' $1 1>&2
+	    tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
 	    return 0
 	else
-	    tail $RUNLOG
-	    echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
+	    echo 'failed (run:' $RETVAL ' end:' $ENDVAL ')'
+	    printf '=> output from running in %s :\n' $1 1>&2
+	    tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
 	    cp $RUNLOG $CDIR"/"$RUNLOG
             if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 1
@@ -826,6 +835,7 @@
 HAVE_MPACK=
 MPACK=
 COMMAND=
+MKDEPEND=
 if test "x$MAKE" = x ; then
     MAKE=make
 fi
@@ -845,7 +855,7 @@
 # list of pTracers to check for monitor output
 PTRACERS_NUM="1 2 3 4 5"
 
-MATCH_CRIT=13
+MATCH_CRIT=10
 
 printf "parsing options...  "
 
@@ -900,6 +910,11 @@
 	-command=* | --command=* | -c=* | --c=*)
 	    COMMAND=$ac_optarg ;;
 
+	-makedepend | --makedepend | -md | --md)
+	    ac_prev=MKDEPEND ;;
+	-makedepend=* | --makedepend=* | -md=* | --md=*)
+	    MKDEPEND=$ac_optarg ;;
+
 	-make | --make | -m | --m)
 	    ac_prev=MAKE ;;
 	-make=* | --make=* | -m=* | --m=*)
@@ -1002,8 +1017,12 @@
     EXECUTABLE="mitgcmuv"
 fi
 
+xx=`echo $TESTDIRS | awk '{print $1}'`
 if test "x$TESTDIRS" = x ; then
     LIST=`scandirs results/$ref_outp`
+elif test $xx = 'start_from' ; then
+    xx=`echo $TESTDIRS | awk '{print $2}'`
+    LIST=`scandirs results/$ref_outp | sed -n "/$xx/,$ p"`
 else
     #- expand group of experiments:
     LIST=" "
@@ -1047,7 +1066,7 @@
     OPTFILE=$MITGCM_OF
 fi
 
-RUNLOG="run.log"
+RUNLOG="run.tr_log"
 OUTPUTFILE=$ref_outp
 if test "x$COMMAND" = x ; then
     COMMAND="./$EXECUTABLE > $OUTPUTFILE"
@@ -1155,7 +1174,6 @@
     then line_0="default  "$MATCH_CRIT ;
     else line_0="default "$MATCH_CRIT ; fi
        line_0="$line_0  ----T-----  ----S-----  ----U-----  ----V-----"
-#   line_0="            ----T-----  ----S-----  ----U-----  ----V-----"
     line_1="G D M    c        m  s        m  s        m  s        m  s"
     line_2="E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
     line_3="N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
@@ -1188,6 +1206,7 @@
 
 EOF
 fi
+echo "-------------------------------------------------------------------------------"
 
 #  ...and each test directory...
 for dir in $TESTDIRS ; do
@@ -1196,6 +1215,7 @@
     builddir="build"
     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
     rundir="run"
+    pfxdir="tr_$rundir"
     if test ! -d $dir/$rundir ; then
 	rundir=$builddir
     fi
@@ -1206,12 +1226,16 @@
     if test "x$CLEANUP" = xt ; then
 	echo -n '  --- dir:' $BUILD_DIR ': '
 	makeclean $BUILD_DIR
-        rm -f $BUILD_DIR/$EXECUTABLE
+	(   cd $BUILD_DIR
+	    rm -f $EXECUTABLE *.bak
+	    rm -f genmake_state genmake_*optfile genmake.log
+	    rm -f genmake.tr_log make.tr_log
+	)
 	if test -d $dir/$rundir/CVS ; then
             echo -n '  --- dir:' $dir/$rundir ': '
 	    run_clean $dir/$rundir
 	fi
-	trdir=`( cd $dir ; find . -type d -name "tr_run.*" -print | sed 's/^.\///')`
+	trdir=`( cd $dir ; find . -type d -name "$pfxdir.*" -print | sed 's/^.\///')`
 	ttd=`echo $trdir | wc -w`
 	if test $ttd != 0 ; then
 	    echo '  --- rm dir:' $trdir
@@ -1277,7 +1301,6 @@
 	fi
     done
 
-    echo "-------------------------------------------------------------------------------"
     echo
     if test "x$extra_runs" = "x" ; then
        echo "Experiment:  $dir"
@@ -1289,45 +1312,43 @@
     results=$EMPTY_RESULTS
 
     #  Create an output dir for each OPTFILE/tdir combination
-    rel_CDIR=$DRESULTS"/"$dir
-    mkdir $rel_CDIR
-    CDIR=`pwd`"/$rel_CDIR"
+    locDIR=$DRESULTS"/"$dir
+    mkdir $locDIR
+    CDIR=`pwd`"/$locDIR"
 
     if test "x$NORUN" = xt ; then
 	    run=N
 	genmakemodel $dir/$builddir && genmake=Y \
 	    && makeclean $dir/$builddir \
-	    && run_clean $dir/$rundir \
             && symlink_mpifiles $dir $code_dir $builddir \
 	    && makedependmodel $dir/$builddir && makedepend=Y \
 	    && makemodel $dir/$builddir && make=Y
     else
 	genmakemodel $dir/$builddir && genmake=Y \
 	    && makeclean $dir/$builddir \
-	    && run_clean $dir/$rundir \
             && symlink_mpifiles $dir $code_dir $builddir \
 	    && makedependmodel $dir/$builddir && makedepend=Y \
 	    && makemodel $dir/$builddir && make=Y \
-	    && linkdata $dir/$rundir $inputdir input \
+	    && run_clean $dir/$rundir \
+	    && linkdata $dir/$rundir $inputdir \
 	    && runmodel $dir/$rundir && run=Y \
 	    && results=`testoutput_run $dir $rundir $ref_outp`
     fi
-
     #echo "results='$results'"
-    echo
+
 	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
-	echo
+	echo 1>&2
  	echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
-	touch $CDIR"/summary.txt"
-	echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
-	echo "MACH='$MACH'" >> $CDIR"/summary.txt"
-	echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
-	echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-	echo "tdir='$dir'" >> $CDIR"/summary.txt"
+	touch $locDIR"/summary.txt"
+	echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
+	echo "MACH='$MACH'" >> $locDIR"/summary.txt"
+	echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
+	echo "DATE='$DATE'" >> $locDIR"/summary.txt"
+	echo "tdir='$dir'" >> $locDIR"/summary.txt"
 	if test "x$ADM" = xt ; then
-	    head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
+	    head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt"
 	    grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
-				>> $CDIR"/summary.txt"
+				>> $locDIR"/summary.txt"
 	fi
 
 	for ex in $extra_runs ; do
@@ -1336,34 +1357,42 @@
 	    #  reference output file
 	    refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
 	    #  Create an output dir for each OPTFILE/tdir.ex combination
-	    rel_CDIR=$DRESULTS"/"$dir"."$ex
-	    mkdir $rel_CDIR
-	    CDIR=`pwd`"/$rel_CDIR"
-	    test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
-	    run_clean $dir/tr_run.$ex
-	    linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
-	    runmodel $dir/tr_run.$ex && run=Y \
-	    && results=`testoutput_run $dir tr_run.$ex $refExOut`
+	    locDIR=$DRESULTS"/"$dir"."$ex
+	    mkdir $locDIR
+	    CDIR=`pwd`"/$locDIR"
+	    test ! -e "$dir/$pfxdir.$ex" && mkdir "$dir/$pfxdir.$ex"
+	    run_clean $dir/$pfxdir.$ex
+	    linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir
+	    runmodel $dir/$pfxdir.$ex && run=Y \
+	    && results=`testoutput_run $dir $pfxdir.$ex $refExOut`
 	    fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	    fres="$fres.$ex"
-	    echo
+	    echo 1>&2
 	    echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
-	    touch $CDIR"/summary.txt"
-	    echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
-	    echo "MACH='$MACH'" >> $CDIR"/summary.txt"
-	    echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
-	    echo "DATE='$DATE'" >> $CDIR"/summary.txt"
-	    echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
+	    touch $locDIR"/summary.txt"
+	    echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
+	    echo "MACH='$MACH'" >> $locDIR"/summary.txt"
+	    echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
+	    echo "DATE='$DATE'" >> $locDIR"/summary.txt"
+	    echo "tdir='$dir.$ex'" >> $locDIR"/summary.txt"
             if test "x$ADM" = xt ; then
-	        head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
-	        grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
-			           >> $CDIR"/summary.txt"
+	        head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt"
+	        grep -A3 'Seconds in section "ALL' $dir/$pfxdir.$ex/$OUTPUTFILE \
+			           >> $locDIR"/summary.txt"
             fi
 	    if test "x$POSTCLEAN" = xt ; then
-		run_clean $dir/tr_run.$ex
+		run_clean $dir/$pfxdir.$ex
 	    fi
 	done
 
+    if test -f $DRESULTS"/"genmake_state ; then : ; else
+	if test -f $dir/$builddir/Makefile ; then
+	    mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
+	    echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state
+	    sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
+	    echo $mkOpt >> $DRESULTS/genmake_state
+	fi
+    fi
     #postclean $dir/$builddir
     if test "x$POSTCLEAN" = xt ; then
  	makeclean $dir/$builddir \