--- MITgcm/verification/testreport	2009/04/22 21:07:00	1.124
+++ MITgcm/verification/testreport	2010/01/16 22:08:27	1.133
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.124 2009/04/22 21:07:00 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.133 2010/01/16 22:08:27 jmc Exp $
 #  $Name:  $
 #
 
@@ -43,6 +43,7 @@
     echo "                             (DEF=\"12\")"
     echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\""
+    echo "  (-norun|-nr)             skip the \"runmodel\" stage (stop after make)"
     echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
     echo "  (-nogenmake|-ng)         skip the genmake stage"
     echo "  (-noclean|-nc)           skip the \"make clean\" stage"
@@ -264,18 +265,18 @@
 	   testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
 	'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
 	   testoutput_var $1 "trcstat_ptracer0"$ii"_sd"   "p0"$ii"_StDv"  $2 $3; yy=$? ;;
-	 'Qntmn') testoutput_var $1 "extforcing_qnet_min" "Qnet minimum"  $2 $3; yy=$? ;;
-	 'Qntmx') testoutput_var $1 "extforcing_qnet_max" "Qnet maximum"  $2 $3; yy=$? ;;
-	 'Qntav') testoutput_var $1 "extforcing_qnet_mean" "Qnet mean"	  $2 $3; yy=$? ;;
-	 'Qntsd') testoutput_var $1 "extforcing_qnet_sd"  "Qnet Std.Dev"  $2 $3; yy=$? ;;
+	 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum"  $2 $3; yy=$? ;;
+	 'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum"  $2 $3; yy=$? ;;
+	 'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean"	  $2 $3; yy=$? ;;
+	 'Qntsd') testoutput_var $1 "forcing_qnet_sd"  "Qnet Std.Dev"  $2 $3; yy=$? ;;
 	 'aSImn') testoutput_var $1 "seaice_area_min"   "SIce Area min"	  $2 $3; yy=$? ;;
 	 'aSImx') testoutput_var $1 "seaice_area_max"   "SIce Area max"	  $2 $3; yy=$? ;;
 	 'aSIav') testoutput_var $1 "seaice_area_mean"  "SIce Area mean"  $2 $3; yy=$? ;;
 	 'aSIsd') testoutput_var $1 "seaice_area_sd"    "SIce Area StDv"  $2 $3; yy=$? ;;
 	 'hSImn') testoutput_var $1 "seaice_heff_min"   "SIce Heff min"	  $2 $3; yy=$? ;;
-	 'hSImx') testoutput_var $1 "seaice_heff_max"   "SIce Hell max"	  $2 $3; yy=$? ;;
-	 'hSIav') testoutput_var $1 "seaice_heff_mean"  "SIce Hell mean"  $2 $3; yy=$? ;;
-	 'hSIsd') testoutput_var $1 "seaice_heff_sd"    "SIce Hell StDv"  $2 $3; yy=$? ;;
+	 'hSImx') testoutput_var $1 "seaice_heff_max"   "SIce Heff max"	  $2 $3; yy=$? ;;
+	 'hSIav') testoutput_var $1 "seaice_heff_mean"  "SIce Heff mean"  $2 $3; yy=$? ;;
+	 'hSIsd') testoutput_var $1 "seaice_heff_sd"    "SIce Heff StDv"  $2 $3; yy=$? ;;
 	'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
 	'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South"  $2 $3; yy=$? ;;
 	'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North"  $2 $3; yy=$? ;;
@@ -385,8 +386,10 @@
 		    cp make.log $CDIR"/make.log"
 		    return 1
 		fi
+		echo successful 1>&2
+	    else
+		echo ''
 	    fi
-	    echo successful 1>&2
 	    exit 0
 	)
     fi
@@ -461,6 +464,7 @@
 		tail make.log
 		echo failed 1>&2
 		cp make.log $CDIR"/make.log"
+		rm -f $EXECUTABLE
 		return 1
 	    else
 		echo successful 1>&2
@@ -525,6 +529,24 @@
     if test -d $1 ; then
 	(
 	    cd $1 ; shift
+            inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
+	    for xx in $inpMPI ; do
+	      if test -r "../"$1"/"$xx ; then
+	        # found 1 _mpi sfx file in 1rst input dir and it is readable
+		yy=`echo $xx | sed 's:\.mpi$::'`
+		if test "x$MPI" = "xt" ; then
+		# mpi test: remove symbolic link & link _mpi sfx file
+		    if test -h $yy ; then rm -f $yy ; fi
+		    if test ! -r $yy ; then 
+			ln -sf "../"$1"/"$xx $yy ;
+			printf "$xx " 1>&2
+		    fi
+		else
+		# not mpi test: remove symbolic link
+		    if test -h $yy ; then rm -f $yy ; fi
+		fi
+	      fi
+	    done
 	    if test -r "../"$1"/eedata.mth" ; then
 	    # found eedata.mth in 1rst input dir and it is readable
 		if test "x$MULTI_THREAD" = "xt" ; then
@@ -610,7 +632,6 @@
 	    ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
 	fi
 	rm -f run.log_tmp
-	#if test "x$RETVAL" = x0 ; then
 	if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
 	    tail $RUNLOG
 	    echo successful 1>&2
@@ -618,6 +639,11 @@
 	    #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
             if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	    return 0
+	elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then
+	    #-- for some weird cases (run is finihed but with error code)
+	    tail $RUNLOG
+	    echo finished with error '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
+	    return 0
 	else
 	    tail $RUNLOG
 	    echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
@@ -766,6 +792,7 @@
 GSL=f
 
 CLEANUP=f
+NORUN=f
 QUICK=f
 NOGENMAKE=f
 NOCLEAN=f
@@ -878,6 +905,8 @@
 	-clean | --clean)
 	    CLEANUP=t ; DELDIR=t ;;
 
+	-norun | --norun | -nr | --nr)
+	    NORUN=t ;;
 	-quick | --quick | -q | --q)
 	    QUICK=t ;;
 	-nogenmake | --nogenmake | -ng | --ng)
@@ -1025,7 +1054,11 @@
 fi
 
 #  create the FORTRAN comparison code
-createcodelet
+if test -x tr_cmpnum ; then
+    echo "skipping comparison code build"
+else
+    createcodelet
+fi
 
 #  build the mpack utility (if ADDRESSES = NONE, do it to test the build)
 if test "x$ADDRESSES" = x ; then
@@ -1150,18 +1183,19 @@
 
     #  Cleanup only!
     if test "x$CLEANUP" = xt ; then
-	if test -r $BUILD_DIR/Makefile ; then
-            echo '  ------  clean dir:' $dir/$builddir
-	    ( cd $BUILD_DIR ; make CLEAN )
-	fi
+	echo -n '  --- dir:' $BUILD_DIR ': '
+	makeclean $BUILD_DIR
+        rm -f $BUILD_DIR/$EXECUTABLE
 	if test -d $dir/$rundir/CVS ; then
-            echo '  ------  clean dir:' $dir/$rundir
+            echo -n '  --- dir:' $dir/$rundir ': '
 	    run_clean $dir/$rundir
 	fi
-	(
-	    cd $dir
-	    rm -rf tr_run.*
-	)
+	trdir=`( cd $dir ; find . -type d -name "tr_run.*" -print | sed 's/^.\///')`
+	ttd=`echo $trdir | wc -w`
+	if test $ttd != 0 ; then
+	    echo '  --- rm dir:' $trdir
+	    ( cd $dir ; rm -rf $trdir )
+	fi
 	continue
     fi
 
@@ -1193,7 +1227,9 @@
 
     #  Check whether there are "extra runs" for this testdir
     extra_runs=
-    ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
+    if test "x$NORUN" = xf ; then
+	ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
+    fi
     #echo "ex_run_dirs='$ex_run_dirs'"
     for exd in $ex_run_dirs ; do
 	name=`echo $exd | sed -e "s/$inputdir\.//"`
@@ -1236,10 +1272,14 @@
     mkdir $rel_CDIR
     CDIR=`pwd`"/$rel_CDIR"
     
-    if test "x$CLEANUP" = xt ; then
-	echo '====>>> this is to check that we never go through this part <<< ==='
-	makeclean $dir/$builddir \
-	    && run_clean $dir/$rundir
+    if test "x$NORUN" = xt ; then
+	    run=N
+	genmakemodel $dir/$builddir && genmake=Y \
+	    && makeclean $dir/$builddir \
+	    && run_clean $dir/$rundir \
+            && symlink_mpifiles $dir $code_dir $builddir \
+	    && makedependmodel $dir/$builddir && makedepend=Y \
+	    && makemodel $dir/$builddir && make=Y
     else
 	genmakemodel $dir/$builddir && genmake=Y \
 	    && makeclean $dir/$builddir \
@@ -1252,6 +1292,7 @@
 	    && results=`testoutput_run $dir $rundir $ref_outp`
     fi
     
+    #echo "results='$results'"
     echo
 	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	echo
@@ -1338,13 +1379,15 @@
 	    echo "An email containing results was sent to the following addresses:"
 	    echo "  \"$ADDRESSES\""
 	    echo
+	    test -f $DRESULTS".tar"  &&  rm -f $DRESULTS".tar"
+	    test -f $DRESULTS".tar.gz"  &&  rm -f $DRESULTS".tar.gz"
 	fi
-	test -f $DRESULTS".tar"  &&  rm -f $DRESULTS".tar"
-	test -f $DRESULTS".tar.gz"  &&  rm -f $DRESULTS".tar.gz"
     fi
 fi
 
-rm -f tr_cmpnum.c tr_cmpnum
+if test "x$QUICK" = xf -a "x$NORUN" = xf ; then
+    rm -f tr_cmpnum.c tr_cmpnum
+fi
 
 if test "x$CLEANUP" != xt ; then
     cat $SUMMARY | sed 's/ \.  \.  \.  \.  \.  \.  \.  \.  \.  \.  \.  \. //'