--- MITgcm/verification/testreport	2014/11/11 15:45:57	1.198
+++ MITgcm/verification/testreport	2016/08/04 17:59:15	1.210
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.198 2014/11/11 15:45:57 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.210 2016/08/04 17:59:15 jmc Exp $
 #  $Name:  $
 #
 
@@ -22,7 +22,7 @@
     echo "  (-command|-c) STRING     command to run (e.g., if non-standard MPI setting)"
     echo "                            DEF='mitgcmuv' or ='mpirun -np TR_NPROC mitgcmuv'"
     echo " ---- testing options : ----"
-    echo "  (-optfile|-of) STRING    list of optfiles to use"
+    echo "  (-optfile|-of) STRING    optfile to use"
     echo "  (-fast)                  use optfile default for compiler flags (no '-ieee')"
     echo "                            DEF=off => use IEEE numerics option (if available)"
     echo "  (-devel)                 use optfile developement flags (if available)"
@@ -42,6 +42,8 @@
     echo "  (-bash|-b) STRING        preferred location of a \"bash\" or \"sh\" shell"
     echo "                             (DEF=\"\" for \"bash\")"
     echo "  (-ef) STRING             used as genmake2 \"-extra_flag\" argument"
+    echo "  (-ncad)                  use genmake2 option \"-nocat4ad\" (-ncad)"
+    echo "  (-small_f)               make target small_f before making target all"
     echo "  (-makedepend|-md) STRING command to use for \"makedepend\""
     echo "  (-make|-m) STRING        command to use for \"make\""
     echo "                             (DEF=\"make\")"
@@ -130,7 +132,7 @@
 	echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
     fi
     if [ -r $1/$4/$OUTPUTFILE ]; then
-	grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > ${TMP}a.txt
+	grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | nl > ${TMP}a.txt
 	lncntA=`wc -l ${TMP}a.txt | awk '{print $1}' `
 	if [ $lncntA -lt 2 ]; then
 	    if [ $verbose -gt 0 ]; then
@@ -145,7 +147,7 @@
     if [ $debug -gt 0 ]; then
 	echo testoutput_var: grep "$2" $1/$5 1>&2
     fi
-    grep "$2" $1/$5 | sed 's/.*=//' | cat -n > ${TMP}b.txt
+    grep "$2" $1/$5 | sed 's/.*=//' | nl > ${TMP}b.txt
     lncntB=`wc -l ${TMP}b.txt | awk '{print $1}' `
     if [ $lncntB -lt 2 ]; then
 	if [ $verbose -gt 0 ]; then
@@ -268,7 +270,11 @@
 	allargs=""
 	for xx in $listVar
 	do
-	  case $xx in
+         #ii=`echo $xx | sed 's/^pt[0-9][0-9]*..$/XXX/'`	#- any ptr number
+          ii=`echo $xx | sed 's/^pt[1-9]..$/XXX/'`		#- 1 to 9 ptr number
+          if test $ii != 'XXX' ; then
+          #- all except ptracer stats:
+	    case $xx in
 	   'PS')  if [ $debug -gt 0 ]
 		  then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
 		  testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $refoutp ; yy=$?
@@ -295,14 +301,6 @@
 	   'Vmx') testoutput_var $1 "dynstat_vvel_max"  "V maximum"	  $2 $refoutp ; yy=$? ;;
 	   'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean"	  $2 $refoutp ; yy=$? ;;
 	   'Vsd') testoutput_var $1 "dynstat_vvel_sd"   "V Std.Dev"	  $2 $refoutp ; yy=$? ;;
-	'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
-	   testoutput_var $1 "trcstat_ptracer0"$ii"_min"  "p0"$ii"_min"	  $2 $refoutp ; yy=$? ;;
-	'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
-	   testoutput_var $1 "trcstat_ptracer0"$ii"_max"  "p0"$ii"_max"	  $2 $refoutp ; yy=$? ;;
-	'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
-	   testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $refoutp ; yy=$? ;;
-	'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
-	   testoutput_var $1 "trcstat_ptracer0"$ii"_sd"   "p0"$ii"_StDv"  $2 $refoutp ; yy=$? ;;
 	 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum"  $2 $refoutp ; yy=$? ;;
 	 'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum"  $2 $refoutp ; yy=$? ;;
 	 'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean"	  $2 $refoutp ; yy=$? ;;
@@ -337,7 +335,18 @@
 	 'sboAp') testoutput_var $1 "sbo_zoamp"         "SBO AM-P"	  $2 $refoutp ; yy=$? ;;
 	'StrmIc') testoutput_var $1 "STREAMICE_FP_ERR" "StreamIce Solver" $2 $refoutp ; yy=$? ;;
 	      *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
-	  esac
+	    esac
+          else
+          #- ptracers stats:
+            nn=`echo $xx | sed 's/pt//' | sed 's/..$//'`
+            ii=`echo $xx | sed 's/^pt[0-9]*//'`
+	    case $ii in
+	    'mn') testoutput_var $1 "trcstat_ptracer0"$nn"_min"  "p0"$nn"_min"  $2 $refoutp ; yy=$? ;;
+	    'mx') testoutput_var $1 "trcstat_ptracer0"$nn"_max"  "p0"$nn"_max"  $2 $refoutp ; yy=$? ;;
+	    'av') testoutput_var $1 "trcstat_ptracer0"$nn"_mean" "p0"$nn"_mean" $2 $refoutp ; yy=$? ;;
+	    'sd') testoutput_var $1 "trcstat_ptracer0"$nn"_sd"   "p0"$nn"_StDv" $2 $refoutp ; yy=$? ;;
+	    esac
+          fi
 	  if test $xx = $sVar
 	  then allargs="$allargs > $yy <"
 	  else allargs="$allargs $yy"
@@ -402,6 +411,9 @@
 	    if test "x$EXTRFLG" != x ; then
 		command="$command -extra_flag $EXTRFLG"
 	    fi
+	    if test "x$NOCATAD" = xt ; then
+		command="$command -nocat4ad"
+	    fi
 	    if test "x$TS" = xt ; then
 		command="$command -ts"
 	    fi
@@ -523,12 +535,23 @@
     else
 	cd $1;
 	if test -r Makefile ; then
-	    printf 'make ... '
-	    if test "x$REPLMAKE" = x ; then
-		$MAKE $TARG >> make.tr_log 2>&1
-	    else
-		$REPLMAKE $TARG >> make.tr_log 2>&1
+	    if test "x$MKSMALLF" = xt ; then
+		printf 'make small_f ... '
+		$MAKE_CMD small_f >> make.tr_log 2>&1
+		RETVAL=$?
+		if test "x$RETVAL" != x0 ; then
+		    tail make.tr_log
+		    echo failed
+		    cp genmake.log genmake.tr_log $CDIR
+		    tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail"
+		    rm -f $EXECUTABLE
+		    mk_fail=1
+		else
+		    echo successful
+		fi
 	    fi
+	    printf "make $TARG ... "
+	    $MAKE_CMD $TARG >> make.tr_log 2>&1
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
 		tail make.tr_log
@@ -776,7 +799,7 @@
 	if test "x$MPI" != x0 ; then
 	    #- adjust the MPI run command with the right number of Procs
 	    #echo '' ; echo "  COMMAND='$COMMAND'"
-	    COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"`
+	    COMMAND=`echo $COMMAND | sed "s/ TR_NPROC/ $LOC_NPROC/"`
 	    if test "x$MPI_MFILE" != x ; then
 	      COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"`
 	    fi
@@ -842,8 +865,15 @@
 	if test -f $OUTPUTFILE ; then
 	  grep '(PID\.TID 0000\.0001)      n.. =' $OUTPUTFILE \
 		| sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
-	  grep -A3 'Seconds in section "ALL' $OUTPUTFILE \
-		| sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
+#	  grep -A3 'Seconds in section "ALL' $OUTPUTFILE \
+#		| sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
+#         some implementations of grep cannot do contextual searches so we
+#         replace the above with a sed command
+	  cat $OUTPUTFILE | sed -n '/Seconds in section "ALL/{N
+	  N
+	  N
+	  p
+	  }' | sed 's/(PID.TID 0000.0001)   //' >> $CDIR"/summary.txt"
 	fi
 	if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
 	if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
@@ -1044,7 +1074,7 @@
 if test "x$MAKE" = x ; then
     MAKE=make
 fi
-REPLMAKE=
+MAKE_CMD=
 if test "x$CC" = x ; then
     CC=cc
 fi
@@ -1057,6 +1087,8 @@
 DELDIR=
 USE_R4=
 EXTRFLG=
+NOCATAD=
+MKSMALLF=
 
 #- type of testing (KIND):
 #   KIND=0 : forward (= default) ;  KIND=1 : Tangent Linear with TAF ;
@@ -1116,8 +1148,8 @@
 	-make | --make | -m | --m) ac_prev=MAKE ;;
 	-make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;;
 
-	-repl_mk | --repl_mk ) ac_prev=REPLMAKE ;;
-	-repl_mk=* | --repl_mk=*) REPLMAKE=$ac_optarg ;;
+	-repl_mk | --repl_mk ) ac_prev=MAKE_CMD ;;
+	-repl_mk=* | --repl_mk=*) MAKE_CMD=$ac_optarg ;;
 
 	-odir | --odir) ac_prev=OUTDIR ;;
 	-odir=* | --odir=*) OUTDIR=$ac_optarg ;;
@@ -1165,6 +1197,8 @@
 	-oad) if test $KIND = 0 ; then KIND=4 ; NODEPEND=t ; else
 		echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage
 	      fi ;;
+	-ncad) NOCATAD=t ;;
+	-small_f) MKSMALLF=t ;;
 
 	-ieee)   echo "Warning: ignore option '-ieee' (already the default)"
 		 printf " ... " ;;
@@ -1184,7 +1218,7 @@
 
 	-ts) TS=t;;
 	-papis) PAPIS=t;;
-	-pcls) PCL=t;;
+	-pcls) PCLS=t;;
 
 	-*) echo "Error: unrecognized option: "$ac_option
 	    usage ;;
@@ -1200,6 +1234,7 @@
     NOCLEAN=t
     NODEPEND=t
 fi
+if test "x$MAKE_CMD" = x ; then MAKE_CMD=$MAKE ; fi
 
 #- check length of MPI machine file:
 if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then
@@ -1238,12 +1273,12 @@
     ref_outp="output_oadm.txt"
     EXECUTABLE="mitgcmuv_ad"
 else
-    if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi
     code_dir=code
     inputdir=input
     ref_outp="output.txt"
     EXECUTABLE="mitgcmuv"
 fi
+if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi
 
 xx=`echo $TESTDIRS | awk '{print $1}'`
 if test "x$TESTDIRS" = x ; then
@@ -1290,10 +1325,6 @@
 if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
 #echo 'TESTDIRS='${TESTDIRS}'<'
 
-if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
-    OPTFILE=$MITGCM_OF
-fi
-
 LOC_MFILE='tr_mpi_mfile'
 RUNLOG="run.tr_log"
 if test "x$MPI" = x0 ; then
@@ -1301,22 +1332,22 @@
   if test "x$COMMAND" = x ; then COMMAND="./$EXECUTABLE > $OUTPUTFILE" ; fi
 else
   OUTPUTFILE="STDOUT.0000"
-  if test "x$COMMAND" = x ; then COMMAND="mpirun -v -np TR_NPROC ./$EXECUTABLE" ; fi
+  if test "x$COMMAND" = x ; then COMMAND="mpirun -np TR_NPROC ./$EXECUTABLE" ; fi
 fi
 
 echo "OK (COMMAND='$COMMAND')"
 
-TMP=./tr_$$
+#TMP=./tr_$$
 #- try to put temporary files in system-local /tmp dir
-#TMP=/tmp/tr_$$
-#touch $TMP ; retVal=$?
-#if [ $retVal -eq 0 ] ; then
-#  if test ! -r $TMP ; then TMP=./tr_$$ ; fi
-#else
-#  TMP=./tr_$$
-#fi
-#rm -f $TMP
-#if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi
+TMP=/tmp/tr_${USER}_$$
+touch $TMP ; retVal=$?
+if [ $retVal -eq 0 ] ; then
+  if test ! -r $TMP ; then TMP=./tr_$$ ; fi
+else
+  TMP=./tr_$$
+fi
+rm -f $TMP
+if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi
 
 # set the Default List of output variables to be checked:
 #  (use default or load experiment-specific list from file "tr_checklist")
@@ -1370,8 +1401,9 @@
 if test "x$OUTDIR" != x ; then
     BASE="tr_"$OUTDIR"_"$DATE"_"
 else
-   #short_name=`hostname | sed 's/\..*$//'`
-    short_name=`hostname -s | tr '[:upper:]' '[:lower:]'`
+   #short_name=`hostname -s | tr '[:upper:]' '[:lower:]'`
+   # hostname -s is not universal (does work on AIX system)
+    short_name=`hostname | sed 's/\..*$//' | tr '[:upper:]' '[:lower:]'`
     BASE="tr_"$short_name"_"$DATE"_"
 fi
 DNUM=0
@@ -1628,6 +1660,7 @@
 	    && symlink_mpifiles $dir $code_dir $builddir \
 	    && makedependmodel $dir/$builddir && makedepend=Y \
 	    && makemodel $dir/$builddir && make=Y
+	    echo
     else
 	genmakemodel $dir/$builddir && genmake=Y \
 	    && makeclean $dir/$builddir \
@@ -1638,11 +1671,11 @@
 	    && linkdata $dir/$rundir $inputdir \
 	    && runmodel $dir/$rundir && run=Y \
 	    && results=`testoutput_run $dir $rundir $ref_outp`
+	    echo 1>&2
     fi
     #echo "results='$results'"
 
 	fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
-	echo 1>&2
 	echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
 	echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
 
@@ -1664,8 +1697,9 @@
 	    linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir
 	    runmodel $dir/$pfxdir.$ex && run=Y \
 	    && results=`testoutput_run $dir $pfxdir.$ex $refExOut`
-	    fres=`formatresults $dir.$ex ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	    echo 1>&2
+
+	    fres=`formatresults $dir.$ex ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
 	    echo "$fres" | sed 's/ 99/ --/g' | sed 's/  > />/' | sed 's/  < /</' >> $SUMMARY
 	    echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
 	    if test "x$POSTCLEAN" = x2 ; then
@@ -1677,12 +1711,20 @@
 	if test -f $dir/$builddir/Makefile ; then
 	    mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
 	    echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state
-	    sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
+#	    sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
+# bsd-sed cannot do the above code
+	    cat $dir/$builddir/Makefile | \
+		sed -n '/^# executed by:/{N
+                                          p
+                                          }' >> $DRESULTS/genmake_state
 	    echo " $mkOpt" >> $DRESULTS/genmake_state
 	    if test "x$OPTFILE" = xNONE ; then
 		eval $mkOpt
-		sed "/^No \"OPTFILE\" was specified ; genmake2/a\  OPTFILE=${OPTFILE}"\
-			$SUMMARY > tmp.tr_log
+#		sed "/^No \"OPTFILE\" was specified ; genmake2/a\  OPTFILE=${OPTFILE}"\
+#			$SUMMARY > tmp.tr_log
+# bsd-sed requires a newline after "a\":
+		sed "/^No \"OPTFILE\" was specified ; genmake2/a\\
+                         OPTFILE=${OPTFILE}" $SUMMARY > tmp.tr_log
 		RETVAL=$?
 		if test "x$RETVAL" = x0 ; then
 		    cp -f tmp.tr_log $SUMMARY
@@ -1703,10 +1745,13 @@
 					 >> $DRESULTS/genmake_state
 		  rm -f tmp.tr_log
 		fi
-		echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state
-		echo "get NETCDF & LAPACK settings:" >> $DRESULTS/genmake_state
-		sed -n '/set HAVE_NETCDF=/p' $gmkLog >> $DRESULTS/genmake_state
-		sed -n '/set HAVE_LAPACK=/p' $gmkLog >> $DRESULTS/genmake_state
+	    fi
+	    gmkLog=$dir/$builddir/genmake_state
+	    if test -r $gmkLog ; then
+		echo -n "from '$gmkLog', "      >> $DRESULTS/genmake_state
+		echo "get genmake settings:"    >> $DRESULTS/genmake_state
+		sed -n '/^HAVE_/p' $gmkLog | sed 's/^./ &/' \
+						>> $DRESULTS/genmake_state
 	    fi
 	fi
     fi