--- MITgcm/verification/testreport	2011/01/27 18:20:38	1.158
+++ MITgcm/verification/testreport	2011/09/26 12:58:48	1.167
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.158 2011/01/27 18:20:38 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.167 2011/09/26 12:58:48 jmc Exp $
 #  $Name:  $
 #
 
@@ -15,8 +15,9 @@
     echo "  (-mpi)                   use MPI to compile and run on 2 processors"
     echo "  (-MPI)  NUMBER           use MPI to compile and run on max NUMBER procs"
     echo "  (-mfile|-mf) STRING      MPI: file with list of possible machines to run on"
-    echo "  (-ieee/-noieee)          if possible, use IEEE compiler flags"
-    echo "                             (DEF=\"-ieee\")"
+    echo "  (-fast)                  use optfile default for compiler flags (no '-ieee')"
+    echo "                            DEF=off => use IEEE numerics option (if available)"
+    echo "  (-devel)                 use optfile developement flags (if available)"
     echo "  (-gsl)                   compile with \"-gsl\" flag"
     echo "  (-use_r4|-ur4)           if allowed, use real*4 type for '_RS' variable"
     echo "  (-optfile|-of) STRING    list of optfiles to use"
@@ -47,8 +48,10 @@
     echo "  (-match) NUMBER          Matching Criteria (number of digits)"
     echo "                             (DEF=\"$MATCH_CRIT\")"
     echo "  (-j) JOBS                use \"make -j JOBS\" for parallel builds"
+    echo "  (-ef) STRING             used as genmake2 \"-extra_flag\" argument"
     echo "  (-clean)                 *ONLY* run \"make CLEAN\" & clean run-dir"
     echo "  (-norun|-nr)             skip the \"runmodel\" stage (stop after make)"
+    echo "  (-obj)                   only produces objects (=norun & no executable)"
     echo "  (-runonly|-ro)           *ONLY* run stage (=\"-quick\" without make)"
     echo "  (-quick|-q)              same as \"-nogenmake -noclean -nodepend\""
     echo "  (-nogenmake|-ng)         skip the genmake stage"
@@ -248,8 +251,9 @@
 		  then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
 		  testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
 		  if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
- 	  'Cost') testoutput_var $1 "ADM  precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
- 	  'Grad') testoutput_var $1 "ADM  precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
+ 	  'Cost') testoutput_var $1 "ADM  ref_cost_function" "ADM CostFct" $2 $3; yy=$? ;;
+ 	 'AdGrd') testoutput_var $1 "ADM  adjoint_gradient"  "ADM Ad Grad" $2 $3; yy=$? ;;
+ 	 'FDGrd') testoutput_var $1 "ADM  finite-diff_grad"  "ADM FD Grad" $2 $3; yy=$? ;;
  	   'Tmn') testoutput_var $1 "dynstat_theta_min"  "Theta minimum"  $2 $3; yy=$? ;;
  	   'Tmx') testoutput_var $1 "dynstat_theta_max"  "Theta maximum"  $2 $3; yy=$? ;;
  	   'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean"	  $2 $3; yy=$? ;;
@@ -338,8 +342,11 @@
 	    if test "x$OPTFILE" != xNONE ; then
 		command="$command -optfile=$OPTFILE"
 	    fi
-	    if test "x$IEEE" != x ; then
-		command="$command -ieee"
+	    if test $OptLev = 1 ; then
+ 		command="$command -ieee"
+ 	    fi
+	    if test $OptLev = 0 ; then
+		command="$command -devel"
 	    fi
 	    if test "x$GSL" = xt ; then
 		command="$command -gsl"
@@ -354,6 +361,9 @@
 	    if test "x$USE_R4" = xt ; then
 		command="$command -use_r4"
 	    fi
+	    if test "x$EXTRFLG" != x ; then
+		command="$command -extra_flag $EXTRFLG"
+	    fi
 	    if test "x$TS" = xt ; then
 		command="$command -ts"
 	    fi
@@ -424,7 +434,8 @@
 	    find . -name "*.data" -exec rm {} \;
 	    find . -name "fort.*" -exec rm {} \;
 	    find . -type l -exec rm {} \;
-	    rm -f $EXECUTABLE $RUNLOG *.txt STD* *diagnostics.log datetime
+	    #- should remove executable only if sym-link (alredy done above)
+	    rm -f $RUNLOG *.txt STD* *diagnostics.log datetime
 	    rm -rf mnc_test_*
  	    rm -f *_MIT_CE_000.opt0000 costfunction*0000
 	    echo successful
@@ -460,13 +471,14 @@
 {
     # makemodel directory
     (
+    mk_fail=0
     if test "x$NOMAKE" = xt ; then
 	cd $1;
 	if test -x $EXECUTABLE ; then
 	    echo "make skipped!"
 	else
 	    echo "no executable!"
-	    return 2
+	    mk_fail=3
 	fi
     else
 	cd $1;
@@ -478,21 +490,25 @@
 		$MAKE adAll >> make.tr_log 2>&1
             else
 		if test "x$JOBS" = x ; then
-		    $MAKE >> make.tr_log 2>&1
+		    $MAKE $TARG >> make.tr_log 2>&1
 		else
-		    $MAKE -j $JOBS >> make.tr_log 2>&1
+		    $MAKE -j $JOBS $TARG >> make.tr_log 2>&1
 		fi
 	    fi
 	    RETVAL=$?
 	    if test "x$RETVAL" != x0 ; then
 		tail make.tr_log
 		echo failed
-		cp make.tr_log genmake.log genmake.tr_log $CDIR
+		cp genmake.log genmake.tr_log $CDIR
+		tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail"
 		rm -f $EXECUTABLE
-		return 1
+		mk_fail=1
 	    else
 		echo successful
 	    fi
+	else
+	    echo "no Makefile !"
+	    mk_fail=2
 	fi
     fi
     if test "x$ADM" = xt -a -f taf_ad.log ; then
@@ -502,12 +518,13 @@
 	    echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \
 				>> $CDIR"/summary.txt"
     fi
+    if test $mk_fail != 0 ; then return $mk_fail ; fi
     )
 }
 
 mk_mpi_size()
 {
-    # mk_mpi_size input_file output_file proc_Nb
+    # mk_mpi_size input_file output_file proc_Nb prefer_to_split_X
     #
     # make new SIZE.h (=output_file) from SIZE.h_mpi (=input_file)
     #     for an MPI build with no more than proc_Nb processors ;
@@ -516,6 +533,7 @@
     inp=$1
     out=$2
     np=$3
+    dirX=$4
     tmp=TTT.$$
 
     px=`grep "^     & *nPx *=" $inp | sed "s/^     & *nPx *= *//" | sed 's/, *$//'`
@@ -533,6 +551,13 @@
 	  if [ `expr $py % $j` -eq 0 ] ; then
 	    ij=`expr $i \* $j`
 	    if [ $ij -gt $pp ] ; then
+		flag=1
+	    elif [ $ij -eq $pp ] ; then
+		flag=$dirX
+	    else
+		flag=0
+	    fi
+	    if test $flag = 1 ; then
 	      if [ $ij -le $np ] ; then
 		ix=$i ; jy=$j ; pp=$ij
 		#echo "  ix,jy= $ix,$jy"
@@ -712,7 +737,7 @@
 	else
           if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
             # output do not exist or is older than executable:
-	    rm -f $RUNLOG ; touch $RUNLOG
+	    rm -f $OUTPUTFILE $RUNLOG ; touch $RUNLOG
 	    if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
 	    ( eval $COMMAND ) >> $RUNLOG 2>&1
 	    RETVAL=$?
@@ -857,27 +882,38 @@
 
 check_eedata()
 {
-    # check_eedata size.h eedata
+    # check_eedata eedata size.h
     if [ $# -eq 2 ] ; then
-      if test -f $1 -a -f $2 ; then
-      sx=`grep "^     & *nSx *=" $1 | sed "s/^     & *nSx *=//" | sed 's/, *$//'`
-      nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
+     if test -f $1 -a -f $2 ; then
+      nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
+      sx=`grep "^     & *nSx *=" $2 | sed "s/^     & *nSx *=//" | sed 's/, *$//'`
       if test "x$nx" = x ; then
 	rx=10
       else
 	rx=`expr $sx % $nx`
       fi
-      sy=`grep "^     & *nSy *=" $1 | sed "s/^     & *nSy *=//" | sed 's/, *$//'`
-      ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
+      ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
+      sy=`grep "^     & *nSy *=" $2 | sed "s/^     & *nSy *=//" | sed 's/, *$//'`
       if test "x$ny" = x ; then
 	ry=20
       else
 	ry=`expr $sy % $ny`
       fi
       echo `expr $rx + $ry`
-    else
+     else
       echo '-1'
-      fi
+     fi
+    elif [ $# -eq 1 ] ; then
+     if test -f $1 ; then
+      nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
+      if test "x$nx" = x ; then nx=1 ; fi
+      ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
+      if test "x$ny" = x ; then ny=1 ; fi
+     #echo $nx $ny
+      echo $nx
+     else
+      echo '-1'
+     fi
     else
       echo '-2'
     fi
@@ -890,11 +926,13 @@
 #  Default properties
 debug=0
 verbose=1
+NBLINES_MKLOG=16000
 
 IEEE=true
 if test "x$MITGCM_IEEE" != x ; then
     IEEE=$MITGCM_IEEE
 fi
+OptLev=1
 GSL=f
 
 CLEANUP=f
@@ -923,12 +961,14 @@
     CC=cc
 fi
 JOBS=
+TARG=
 MPI=0
 MPI_MFILE=
 MULTI_THREAD=f
 OUTDIR=
 DELDIR=
 USE_R4=
+EXTRFLG=
 
 ADM=
 OADM=
@@ -993,9 +1033,13 @@
 	-j | --j) ac_prev=JOBS ;;
 	-j=* | --j=*) JOBS=$ac_optarg ;;
 
+	-ef | --ef) ac_prev=EXTRFLG ;;
+	-ef=* | --ef=*) EXTRFLG=$ac_optarg ;;
+
 	-clean | --clean) CLEANUP=t ; DELDIR=t ;;
 
 	-norun | --norun | -nr | --nr) NORUN=t ;;
+	-obj | --obj ) TARG='obj' ; NORUN=t ;;
 	-runonly | --runonly | -ro | --ro) QUICK=t ; NOMAKE=t ;;
 	-quick | --quick | -q | --q) QUICK=t ;;
 	-nogenmake | --nogenmake | -ng | --ng) NOGENMAKE=t ;;
@@ -1017,8 +1061,12 @@
 	-adm | -ad) ADM=t ;;
 	-oad) OADM=t; NODEPEND=t ;;
 
-	-ieee) IEEE=true ;;
-	-noieee) IEEE= ;;
+	-ieee)   echo "Warning: ignore option '-ieee' (already the default)"
+                 printf " ... " ;;
+	-noieee) echo "Warning: will use option '-fast' instead of '-noieee' (obsolete)"
+                 printf " ... " ; OptLev=`expr $OptLev \* 2` ;;
+	-fast)  OptLev=`expr $OptLev \* 2` ;;
+	-devel) OptLev=0 ;;
 	-gsl) GSL=t ;;
 
 	-verbose) verbose=2 ;;
@@ -1155,7 +1203,7 @@
     ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
     EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf "  ."}}'`
 else
-    DEF_CHECK_LIST='Grad Cost Grad'
+    DEF_CHECK_LIST='AdGrd Cost AdGrd FDGrd'
     EMPTY_RESULTS='.. ..'
     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
 fi
@@ -1242,8 +1290,8 @@
     else line_0="default "$MATCH_CRIT ; fi
        line_0="$line_0  ----T-----  ----S-----  ----U-----  ----V-----"
     line_1="G D M    c        m  s        m  s        m  s        m  s"
-    line_2="E p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
-    line_3="N n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
+    line_2="e p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  ."
+    line_3="n n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d"
     line_4="2 d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  ."
     for ii in $PTRACERS_NUM ; do
 	line_0="$line_0  --PTR 0"$ii"--"
@@ -1266,10 +1314,10 @@
     else line_0="default    "$MATCH_CRIT ; fi
     echo "$line_0" | tee -a $SUMMARY
     cat << EOF | tee -a $SUMMARY
-G D M    C  G
-E p a R  o  r
-N n k u  s  a
-2 d e n  t  d
+G D M    C  A  F
+e p a R  o  d  D
+n n k u  s  G  G
+2 d e n  t  r  r
 
 EOF
 fi
@@ -1321,26 +1369,35 @@
 
     # Check for specific files for particular type of run
 
+    if test ! -r $CODE_DIR"/SIZE.h_mpi"  -a "x$MPI" != "x0" ; then
+	echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
+	continue
+    fi
+    if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
+	echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
+	continue
+    fi
+
     if test "x$MPI" != "x0" ; then
-	if test -r $CODE_DIR"/SIZE.h_mpi" ; then
-	    #- create new SIZE.h with no more than '$MPI' Procs
-	    mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI
-	    LOC_NPROC=$?
-	    (   cd $BUILD_DIR
-		if test -r SIZE.h.mpi ; then
-		    cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$?
-		else RETVAL=1
-		fi
-		if test "x$RETVAL" = x0 ; then
-		    rm -f tr_size.mpi
-		else
-		    rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi
-		fi
-	    )
-	else
-	    echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
-	    continue
-	fi
+	prefer_X=0
+	if test "x$MULTI_THREAD" = "xt" ; then
+	    retv=`check_eedata $dir"/input/eedata.mth"`
+	    if test $retv = 1 ; then prefer_X=1 ; fi
+	fi
+	#- create new SIZE.h with no more than '$MPI' Procs
+	mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI $prefer_X
+	LOC_NPROC=$?
+	(   cd $BUILD_DIR
+	    if test -r SIZE.h.mpi ; then
+		cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$?
+	    else RETVAL=1
+	    fi
+	    if test "x$RETVAL" = x0 ; then
+		rm -f tr_size.mpi
+	    else
+		rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi
+	    fi
+	)
 	if test "x$MPI_MFILE" != x ; then
 	    #- create new MPI machine-file with the right number of Procs
 	    rm -f $LOC_MFILE
@@ -1356,16 +1413,12 @@
 		echo " new LOC_MFILE=$LOC_MFILE : $nl procs for LOC_NPROC=$LOC_NPROC"
 	    fi
 	fi
-    fi
-    if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
-	echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
-	continue
-    fi
-    if test "x$MPI" != "x0" -a "x$MULTI_THREAD" = "xt" ; then
-	retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/input/eedata.mth"`
-	if test $retv != 0 ; then
-	    echo "input/eedata.mth tiling misfit -- skipping $dir"
-	    continue
+	if test "x$MULTI_THREAD" = "xt" ; then
+	    retv=`check_eedata $dir"/input/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"`
+	    if test $retv != 0 ; then
+		echo "input/eedata.mth tiling misfit -- skipping $dir"
+		continue
+	    fi
 	fi
     fi
 
@@ -1385,7 +1438,7 @@
 	      if test "x$MPI" = "x0" ; then
 		extra_runs="$extra_runs $name"
 	      else
-		retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/"$exd"/eedata.mth"`
+		retv=`check_eedata $dir"/"$exd"/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"`
 		if test $retv = 0 ; then
 		    extra_runs="$extra_runs $name"
 		else
@@ -1494,8 +1547,9 @@
 	    if test "x$RETVAL" = x0 ; then
 		echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state
 		sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \
-		       | grep -v '^... compiler version ' > tr_1.tmp_log
-		sed -n '1,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \
+			| grep -v '^... compiler version ' > tr_1.tmp_log
+		sed -n '1p' tr_1.tmp_log >> $DRESULTS/genmake_state
+		sed -n '2,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \
 			>> $DRESULTS/genmake_state
 		rm -f tr_1.tmp_log
 	    fi