--- MITgcm/verification/testreport	2013/08/18 19:53:47	1.192
+++ MITgcm/verification/testreport	2015/11/20 19:47:46	1.204
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.192 2013/08/18 19:53:47 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.204 2015/11/20 19:47:46 jmc Exp $
 #  $Name:  $
 #
 
@@ -19,7 +19,8 @@
     echo "  (-mpi)                   use MPI to compile and run on 2 processors"
     echo "  (-MPI)  NUMBER           use MPI to compile and run on max NUMBER procs"
     echo "  (-mfile|-mf) STRING      MPI: file with list of possible machines to run on"
-    echo "  (-command|-c) STRING     command to run (needed for MPI run)"
+    echo "  (-command|-c) STRING     command to run (e.g., if non-standard MPI setting)"
+    echo "                            DEF='mitgcmuv' or ='mpirun -np TR_NPROC mitgcmuv'"
     echo " ---- testing options : ----"
     echo "  (-optfile|-of) STRING    list of optfiles to use"
     echo "  (-fast)                  use optfile default for compiler flags (no '-ieee')"
@@ -41,6 +42,8 @@
     echo "  (-bash|-b) STRING        preferred location of a \"bash\" or \"sh\" shell"
     echo "                             (DEF=\"\" for \"bash\")"
     echo "  (-ef) STRING             used as genmake2 \"-extra_flag\" argument"
+    echo "  (-ncad)                  use genmake2 option \"-nocat4ad\" (-ncad)"
+    echo "  (-small_f)               make target small_f before making target all"
     echo "  (-makedepend|-md) STRING command to use for \"makedepend\""
     echo "  (-make|-m) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
@@ -129,7 +132,7 @@
 	echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
     fi
     if [ -r $1/$4/$OUTPUTFILE ]; then
-	grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > ${TMP}a.txt
+	grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | nl > ${TMP}a.txt
 	lncntA=`wc -l ${TMP}a.txt | awk '{print $1}' `
 	if [ $lncntA -lt 2 ]; then
 	    if [ $verbose -gt 0 ]; then
@@ -144,7 +147,7 @@
     if [ $debug -gt 0 ]; then
 	echo testoutput_var: grep "$2" $1/$5 1>&2
     fi
-    grep "$2" $1/$5 | sed 's/.*=//' | cat -n > ${TMP}b.txt
+    grep "$2" $1/$5 | sed 's/.*=//' | nl > ${TMP}b.txt
     lncntB=`wc -l ${TMP}b.txt | awk '{print $1}' `
     if [ $lncntB -lt 2 ]; then
 	if [ $verbose -gt 0 ]; then
@@ -330,6 +333,10 @@
 	'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North-av"  $2 $refoutp ; yy=$? ;;
 	'HthMxS') testoutput_var $1 "thSI_IceH_max_S" "thSIc H South-max" $2 $refoutp ; yy=$? ;;
 	'HthMxN') testoutput_var $1 "thSI_IceH_max_N" "thSIc H North-max" $2 $refoutp ; yy=$? ;;
+	 'sbo_M') testoutput_var $1 "sbo_mass"          "SBO mass"	  $2 $refoutp ; yy=$? ;;
+	 'sboFW') testoutput_var $1 "sbo_mass_fw"       "SBO m-FW"	  $2 $refoutp ; yy=$? ;;
+	 'sboAc') testoutput_var $1 "sbo_zoamc"         "SBO AM-C"	  $2 $refoutp ; yy=$? ;;
+	 'sboAp') testoutput_var $1 "sbo_zoamp"         "SBO AM-P"	  $2 $refoutp ; yy=$? ;;
 	'StrmIc') testoutput_var $1 "STREAMICE_FP_ERR" "StreamIce Solver" $2 $refoutp ; yy=$? ;;
 	      *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
 	  esac
@@ -367,12 +374,10 @@
 	    if test "x$MKDEPEND" != x ; then
 		command="$command -makedepend=$MKDEPEND"
 	    fi
-	    if test $KIND = 1 -o $KIND = 2 ; then
-		command="$command --mods=../code_ad"
-	    elif test $KIND = 4 ; then
-		command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
+	    if test $KIND = 4 ; then
+		command="$command -oad -mods=../$code_dir"
 	    else
-		command="$command -mods=../code"
+		command="$command -mods=../$code_dir"
 	    fi
 	    if test "x$OPTFILE" != xNONE ; then
 		command="$command -optfile=$OPTFILE"
@@ -399,6 +404,9 @@
 	    if test "x$EXTRFLG" != x ; then
 		command="$command -extra_flag $EXTRFLG"
 	    fi
+	    if test "x$NOCATAD" = xt ; then
+		command="$command -nocat4ad"
+	    fi
 	    if test "x$TS" = xt ; then
 		command="$command -ts"
 	    fi
@@ -429,7 +437,7 @@
 makeclean()
 {
     # makeclean directory
-    if test "x$NODEPEND" = xf ; then rm -f $1/make.tr_log ; fi
+    if test "x$NOGENMAKE" = xf ; then rm -f $1/make.tr_log ; fi
     if test "x$NOCLEAN" = xt ; then
 	echo "make Clean skipped!"
     else
@@ -520,12 +528,23 @@
     else
 	cd $1;
 	if test -r Makefile ; then
-	    printf 'make ... '
-	    if test "x$REPLMAKE" = x ; then
-		$MAKE $TARG >> make.tr_log 2>&1
-	    else
-		$REPLMAKE $TARG >> make.tr_log 2>&1
+	    if test "x$MKSMALLF" = xt ; then
+		printf 'make small_f ... '
+		$MAKE_CMD small_f >> make.tr_log 2>&1
+		RETVAL=$?
+		if test "x$RETVAL" != x0 ; then
+		    tail make.tr_log
+		    echo failed
+		    cp genmake.log genmake.tr_log $CDIR
+		    tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail"
+		    rm -f $EXECUTABLE
+		    mk_fail=1
+		else
+		    echo successful
+		fi
 	    fi
+	    printf 'make ... '
+	    $MAKE_CMD $TARG >> make.tr_log 2>&1
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
 		tail make.tr_log
@@ -773,7 +792,7 @@
 	if test "x$MPI" != x0 ; then
 	    #- adjust the MPI run command with the right number of Procs
 	    #echo '' ; echo "  COMMAND='$COMMAND'"
-	    COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"`
+	    COMMAND=`echo $COMMAND | sed "s/ TR_NPROC/ $LOC_NPROC/"`
 	    if test "x$MPI_MFILE" != x ; then
 	      COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"`
 	    fi
@@ -839,8 +858,15 @@
 	if test -f $OUTPUTFILE ; then
 	  grep '(PID\.TID 0000\.0001)      n.. =' $OUTPUTFILE \
 		| sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
-	  grep -A3 'Seconds in section "ALL' $OUTPUTFILE \
-		| sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
+#	  grep -A3 'Seconds in section "ALL' $OUTPUTFILE \
+#		| sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
+#         some implementations of grep cannot do contextual searches so we
+#         replace the above with a sed command
+	  cat $OUTPUTFILE | sed -n '/Seconds in section "ALL/{N
+	  N
+	  N
+	  p
+	  }' | sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
 	fi
 	if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
@@ -1041,7 +1067,7 @@
 if test "x$MAKE" = x ; then
     MAKE=make
 fi
-REPLMAKE=
+MAKE_CMD=
 if test "x$CC" = x ; then
     CC=cc
 fi
@@ -1054,6 +1080,8 @@
 DELDIR=
 USE_R4=
 EXTRFLG=
+NOCATAD=
+MKSMALLF=
 
 #- type of testing (KIND):
 #   KIND=0 : forward (= default) ;  KIND=1 : Tangent Linear with TAF ;
@@ -1113,8 +1141,8 @@
 	-make | --make | -m | --m) ac_prev=MAKE ;;
 	-make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;;
 
-	-repl_mk | --repl_mk ) ac_prev=REPLMAKE ;;
-	-repl_mk=* | --repl_mk=*) REPLMAKE=$ac_optarg ;;
+	-repl_mk | --repl_mk ) ac_prev=MAKE_CMD ;;
+	-repl_mk=* | --repl_mk=*) MAKE_CMD=$ac_optarg ;;
 
 	-odir | --odir) ac_prev=OUTDIR ;;
 	-odir=* | --odir=*) OUTDIR=$ac_optarg ;;
@@ -1162,6 +1190,8 @@
 	-oad) if test $KIND = 0 ; then KIND=4 ; NODEPEND=t ; else
 		echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage
 	      fi ;;
+	-ncad) NOCATAD=t ;;
+	-small_f) MKSMALLF=t ;;
 
 	-ieee)   echo "Warning: ignore option '-ieee' (already the default)"
 		 printf " ... " ;;
@@ -1181,7 +1211,7 @@
 
 	-ts) TS=t;;
 	-papis) PAPIS=t;;
-	-pcls) PCL=t;;
+	-pcls) PCLS=t;;
 
 	-*) echo "Error: unrecognized option: "$ac_option
 	    usage ;;
@@ -1197,6 +1227,7 @@
     NOCLEAN=t
     NODEPEND=t
 fi
+if test "x$MAKE_CMD" = x ; then MAKE_CMD=$MAKE ; fi
 
 #- check length of MPI machine file:
 if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then
@@ -1235,12 +1266,12 @@
     ref_outp="output_oadm.txt"
     EXECUTABLE="mitgcmuv_ad"
 else
-    if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi
     code_dir=code
     inputdir=input
     ref_outp="output.txt"
     EXECUTABLE="mitgcmuv"
 fi
+if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi
 
 xx=`echo $TESTDIRS | awk '{print $1}'`
 if test "x$TESTDIRS" = x ; then
@@ -1293,27 +1324,27 @@
 
 LOC_MFILE='tr_mpi_mfile'
 RUNLOG="run.tr_log"
-OUTPUTFILE=$ref_outp
-if test "x$COMMAND" = x ; then
-    COMMAND="./$EXECUTABLE > $OUTPUTFILE"
-fi
-if test "x$MPI" != x0 ; then
-    OUTPUTFILE="STDOUT.0000"
+if test "x$MPI" = x0 ; then
+  OUTPUTFILE=$ref_outp
+  if test "x$COMMAND" = x ; then COMMAND="./$EXECUTABLE > $OUTPUTFILE" ; fi
+else
+  OUTPUTFILE="STDOUT.0000"
+  if test "x$COMMAND" = x ; then COMMAND="mpirun -v -np TR_NPROC ./$EXECUTABLE" ; fi
 fi
 
-echo "OK (COMMAND= $COMMAND )"
+echo "OK (COMMAND='$COMMAND')"
 
-TMP=./tr_$$
+#TMP=./tr_$$
 #- try to put temporary files in system-local /tmp dir
-#TMP=/tmp/tr_$$
-#touch $TMP ; retVal=$?
-#if [ $retVal -eq 0 ] ; then
-#  if test ! -r $TMP ; then TMP=./tr_$$ ; fi
-#else
-#  TMP=./tr_$$
-#fi
-#rm -f $TMP
-#if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi
+TMP=/tmp/tr_${USER}_$$
+touch $TMP ; retVal=$?
+if [ $retVal -eq 0 ] ; then
+  if test ! -r $TMP ; then TMP=./tr_$$ ; fi
+else
+  TMP=./tr_$$
+fi
+rm -f $TMP
+if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi
 
 # set the Default List of output variables to be checked:
 #  (use default or load experiment-specific list from file "tr_checklist")
@@ -1360,7 +1391,6 @@
 			     CMDLINE="$CMDLINE $yy" ; fi
     fi
 done
-#for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
 MACH=`hostname`
 UNAMEA=`uname -a`
 DATE=`date +%Y%m%d`
@@ -1368,7 +1398,9 @@
 if test "x$OUTDIR" != x ; then
     BASE="tr_"$OUTDIR"_"$DATE"_"
 else
-    short_name=`hostname | sed 's/\..*$//'`
+   #short_name=`hostname -s | tr '[:upper:]' '[:lower:]'`
+   # hostname -s is not universal (does work on AIX system)
+    short_name=`hostname | sed 's/\..*$//' | tr '[:upper:]' '[:lower:]'`
     BASE="tr_"$short_name"_"$DATE"_"
 fi
 DNUM=0
@@ -1674,12 +1706,20 @@
 	if test -f $dir/$builddir/Makefile ; then
 	    mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
 	    echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state
-	    sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
+#	    sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
+# bsd-sed cannot do the above code
+	    cat $dir/$builddir/Makefile | \
+		sed -n '/^# executed by:/{N
+                                          p
+                                          }' >> $DRESULTS/genmake_state
 	    echo " $mkOpt" >> $DRESULTS/genmake_state
 	    if test "x$OPTFILE" = xNONE ; then
 		eval $mkOpt
-		sed "/^No \"OPTFILE\" was specified ; genmake2/a\  OPTFILE=${OPTFILE}"\
-			$SUMMARY > tmp.tr_log
+#		sed "/^No \"OPTFILE\" was specified ; genmake2/a\  OPTFILE=${OPTFILE}"\
+#			$SUMMARY > tmp.tr_log
+# bsd-sed requires a newline after "a\":
+		sed "/^No \"OPTFILE\" was specified ; genmake2/a\\
+                         OPTFILE=${OPTFILE}" $SUMMARY > tmp.tr_log
 		RETVAL=$?
 		if test "x$RETVAL" = x0 ; then
 		    cp -f tmp.tr_log $SUMMARY
@@ -1700,10 +1740,13 @@
 					 >> $DRESULTS/genmake_state
 		  rm -f tmp.tr_log
 		fi
-		echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state
-		echo "get NETCDF & LAPACK settings:" >> $DRESULTS/genmake_state
-		sed -n '/set HAVE_NETCDF=/p' $gmkLog >> $DRESULTS/genmake_state
-		sed -n '/set HAVE_LAPACK=/p' $gmkLog >> $DRESULTS/genmake_state
+	    fi
+	    gmkLog=$dir/$builddir/genmake_state
+	    if test -r $gmkLog ; then
+		echo -n "from '$gmkLog', "      >> $DRESULTS/genmake_state
+		echo "get genmake settings:"    >> $DRESULTS/genmake_state
+		sed -n '/^HAVE_/p' $gmkLog | sed 's/^./ &/' \
+						>> $DRESULTS/genmake_state
 	    fi
 	fi
     fi