--- MITgcm/tools/example_scripts/csail/test_baudelaire	2012/08/16 17:12:17	1.11
+++ MITgcm/tools/example_scripts/csail/test_baudelaire	2016/01/30 15:55:45	1.23
@@ -1,9 +1,14 @@
 #! /usr/bin/env bash
 
-# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/example_scripts/csail/test_baudelaire,v 1.11 2012/08/16 17:12:17 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/example_scripts/csail/test_baudelaire,v 1.23 2016/01/30 15:55:45 jmc Exp $
 
-#  Test script for MITgcm that should work on most of the csail.mit.edu
-#  Linux machines.
+#  Test script for MITgcm that should work on most of the csail.mit.edu Linux machines.
+
+tst_grp=0 ; if test $# = 1 ; then tst_grp=$1 ; fi
+if test $tst_grp != 'a' -a $tst_grp != 'b' ; then
+  echo "missing or invalid argument (expect: 'a' or 'b') ==> exit"
+  exit 1
+fi
 
 # defaults
 #export PATH="$PATH:/usr/local/bin"
@@ -25,39 +30,49 @@
 cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack'
 cmdCVS='cvs -d /u/gcmpack'
 
+# checkOut=2 : download new code ;
+#   =3 : skip download but, if sepDir, use a new copy
+#   =1 : update code       (if no existing code -> swith to 2)
+#   =0 : use existing code (if no existing code -> swith to 2)
 dInWeek=`date +%a`
-TESTDIR="/scratch/jmc/test_"`hostname -s`
+outDir=`hostname -s`
+TESTDIR="/scratch/jmc/test_${outDir}"
+outDir="${outDir}-${tst_grp}"
 MC=13
-checkOut=1
 sepDir=1
 option=
-tst_list='g7a adm mpa g77 gfo+rs mth mp2+rs mpi ifc'
-if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm" ; fi
-#tst_list='g77 adm gfo ifc mth pgi+rs'
-#tst_list='adm gfo+rs mth'
+#tst_list='g7a adm mpa g77 gfo+rs mth mp2+rs mpi ifc'
+#if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm oad" ; fi
+if test $tst_grp = 'a' ; then
+ checkOut=2
+ tst_list='g7a mpa mth mp2+rs mpi oad'
+else
+ checkOut=3
+ tst_list='adm g77 gfo+rs ifc'
+ if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm" ; fi
+fi
+echo " test: $outDir ; list='$tst_list'"
 
-#option="-nc" ; checkOut=0
-#option="-q"  ; checkOut=0
+#option="-nc" ; checkOut=1
+#option="-q"  ; checkOut=1
 
 TODAY=`date +%d`
 tdir=$TESTDIR
-if test $checkOut = '0' ; then
+if test $checkOut != 2 ; then
   if test -e $tdir/MITgcm/CVS ; then
     echo $tdir/MITgcm/CVS 'exist'
-    echo -n "Update the MITgcm code using: $cmdCVS ..."
-    cd $tdir/MITgcm
-    if test $sepDir = 1 ; then
-      $cmdCVS update -P -d > /dev/null
-    else
+    if test $sepDir = 0 -a $checkOut = 1 ; then
+      echo -n "Update the MITgcm code using: $cmdCVS ..."
+      cd $tdir/MITgcm
       $cmdCVS update -P -d
+      echo "  done"
     fi
-    echo "  done"
   else
     echo -n $tdir/MITgcm 'missing ; '
-    checkOut=1
+    checkOut=2
   fi
 fi
-if test $checkOut = '1' ; then
+if test $checkOut = 2 ; then
   if test -e $tdir ; then
     echo -n "Removing working copy: $tdir/MITgcm ..."
     test -e $tdir/MITgcm  &&  rm -rf $tdir/MITgcm
@@ -83,6 +98,11 @@
 
   echo "================================================================"
   typ=`echo $tt | sed 's/+rs//'`
+  #- define list of additional experiences to test:
+  addExp=''
+  if test $typ = 'mp2' -o $typ = 'ifc' ; then
+    addExp='offline_cheapaml atm_gray'
+  fi
   #- check day and time:
   curDay=`date +%d` ; curHour=`date +%H`
   if [ $curDay -ne $TODAY ] ; then
@@ -93,20 +113,46 @@
     date ; echo "too late to run test $typ"
     continue
   fi
+  if test $sepDir = 0 -a "x$option" != x -a  $tt != $last_Tst ; then
+    echo "using option='$option' prevent multi-tests => skip test $typ"
+    continue
+  fi
   #- clean-up old output files
   rm -f $tdir/output_${typ}*
+  touch $tdir/output_$tt
   if test $sepDir = 1 ; then
     new_dir="MITgcm_$typ"
-    if test -d $new_dir/CVS -a $checkOut = '0' ; then
+    reUse=1 ; if [ $checkOut -ge 2 ] ; then reUse=0 ; fi
+    if test -d $new_dir/CVS -a $reUse = 1 ; then
       pushd $new_dir
-      echo -n "Update the MITgcm code using: $cmdCVS ..."
-      $cmdCVS update -P -d
-      echo "  done"
+      if test $tt != $typ ; then
+        ( cd verification ; ../tools/do_tst_2+2 -clean )
+      fi
+      if test $checkOut = 1 ; then
+        echo -n "Update the MITgcm code using: $cmdCVS ..."
+        $cmdCVS update -P -d
+        echo "  done"
+      fi
     else
+      if test -d prev ; then
+      #-- save previous summary:
+        oldS=`ls -t ${new_dir}/verification/tr_${outDir}_*/summary.txt 2> /dev/null | head -1`
+        if test "x$oldS" != x ; then cp -p -f $oldS prev/tr_out.$typ ; fi
+       if test $tt != $typ ; then
+        oldS=`ls -t ${new_dir}/verification/rs_${outDir}_*/summary.txt 2> /dev/null | head -1`
+        if test "x$oldS" != x ; then cp -p -f $oldS prev/rs_out.$typ ; fi
+       fi
+      fi
       test -e $new_dir  &&  rm -rf $new_dir
       mkdir $new_dir
       pushd $new_dir
       cp -ra ../MITgcm/* .
+      #-- download additional experience from Contrib:
+      for exp2add in $addExp ; do
+        echo " add dir: $exp2add (from Contrib:verification_other)"
+        ( cd verification ;  $cmdCVS co -P -d $exp2add \
+             MITgcm_contrib/verification_other/$exp2add > /dev/null )
+      done
     fi
   else
     pushd MITgcm
@@ -114,25 +160,29 @@
   cd verification
 
 #-- set the testreport command:
-  nbl='-100'
   comm="./testreport"
   if test $typ = 'g7a' -o  $typ = 'adm' -o  $typ = 'mpa' ; then
-    comm="$comm -adm" ; nbl='-60'
+    comm="$comm -adm"
+  elif test $typ = 'oad' ; then
+    comm="$comm -oad"
   elif test $typ = 'tlm' ; then
-    comm="$comm -tlm" ; nbl='-60'
+    comm="$comm -tlm"
   elif test $typ = 'mth' -o  $typ = 'mp2' ; then
     export GOMP_STACKSIZE=400m
     export OMP_NUM_THREADS=2
     comm="$comm -mth"
+  else
+    comm="$comm -md cyrus-makedepend"
   fi
-  comm="$comm -a jmc@mitgcm.org"
+  comm="$comm -odir $outDir -a jmc@mitgcm.org"
 #-- set the optfile (+ mpi & match-precision)
   MPI=0
   case $typ in
    'g77'|'g7a')		OPTFILE='../tools/build_options/linux_amd64_g77' ;;
-   'gfo'|'adm'|'tlm'|'mth') comm="$comm -match $MC -devel"
+   'gfo'|'adm'|'oad'|'tlm'|'mth') comm="$comm -match $MC -devel"
 			OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
-   'ifc')		OPTFILE='../tools/build_options/linux_amd64_ifort11' ;;
+   'ifc')		comm="$comm -devel"
+			OPTFILE='../tools/build_options/linux_amd64_ifort11' ;;
    'pgi')		OPTFILE='../tools/build_options/linux_amd64_pgf77' ;;
    'mpa'|'mpi'|'mp2')	comm="$comm -match $MC -devel" ; MPI=6
 			OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
@@ -149,6 +199,9 @@
   fi
 
 #-- set specific Env Vars:
+  if test $typ = 'oad' ; then
+    source ~jmc/mitgcm/bin/setenv_OpenAD.sh
+  fi
   if test $typ = 'ifc' ; then
     source /srv/software/intel/intel-11.1.073/bin/ifortvars.sh intel64
   fi
@@ -158,53 +211,49 @@
     export PATH="$PATH:$PGI/linux86-64/10.9/bin"
     export LM_LICENSE_FILE=$PGI/license.dat
   fi
-  basename $OPTFILE | grep gfortran > /dev/null 2>&1 ; retv=$? ; 
-  GFORTRAN_OPTFILE=f ; if test $retv == 0 ; then GFORTRAN_OPTFILE='t' ; fi 
-  export GFORTRAN_OPTFILE
 
-  if test $sepDir = 0 -a $checkOut = '1' -a  $tt = $firstTst ; then
+  if test $sepDir = 0 -a "x$option" = x -a $tt = $firstTst -a $checkOut != 2 ; then
 #-- cleaning:
     echo "======================"
-    echo "Cleaning test directories:"
+    echo "Cleaning test directories:"	| tee -a $tdir/output_$tt
     cmdCLN="./testreport -clean"
-    echo " clean dir running: $cmdCLN"
-    $cmdCLN > /dev/null 2>&1
+    echo " clean dir running: $cmdCLN"	| tee -a $tdir/output_$tt
+    $cmdCLN >> $tdir/output_$tt 2>&1
     echo "======================"
-    echo
+    echo ""				| tee -a $tdir/output_$tt
   fi
 
 #-- run the testreport command:
-  echo -n "Running testreport using:"
+  echo -n "Running testreport using:"	| tee -a $tdir/output_$tt
   if test "x$OPTFILE" != x ; then
     comm="$comm -of=$OPTFILE"
   fi
-  if test $MPI = 0 ; then  echo ''
-  else echo " (EXE='$EXE')"
+  if test $MPI = 0 ; then  echo ''	| tee -a $tdir/output_$tt
+  else echo " (EXE='$EXE')"		| tee -a $tdir/output_$tt
     comm="$comm -MPI $MPI -command \"\$EXE\""
   fi
   if test "x$option" != x ; then comm="$comm $option" ; fi
  #if test $typ = 'pgi' ; then comm="$comm -t \"\$listT\"" ; fi
-  echo "  \"eval $comm\""
+  echo "  \"eval $comm\""		| tee -a $tdir/output_$tt
   echo "======================"
-  eval $comm > $tdir/output_$typ 2>&1
-  tail $nbl $tdir/output_$typ
-  echo
+  eval $comm >> $tdir/output_$tt 2>&1
+  sed -n "/^An email /,/^======== End of testreport / p" $tdir/output_$tt
+  echo ""				| tee -a $tdir/output_$tt
 
 #-- also test restart (test 2+2=4)
   if test $tt != $typ
   then
-    echo "testing restart using:"
-    comm="../tools/do_tst_2+2 -a jmc@mitgcm.org"
+    echo "testing restart using:"	| tee -a $tdir/output_$tt
+    comm="../tools/do_tst_2+2 -o $outDir -a jmc@mitgcm.org"
     if test $MPI = 0 ; then
-      echo "  \"$comm\""
+      echo "  \"$comm\""		| tee -a $tdir/output_$tt
       echo "======================"
-      $comm > $tdir/output_2+2 2>&1
+      $comm >> $tdir/output_$tt 2>&1
     else
-      echo "  \"$comm -mpi -exe $EXE\""
+      echo "  \"$comm -mpi -exe $EXE\""	| tee -a $tdir/output_$tt
       echo "======================"
-      $comm -mpi -exe "$EXE" > $tdir/output_2+2 2>&1
+      $comm -mpi -exe "$EXE" >> $tdir/output_$tt 2>&1
     fi
-   #tail $nbl $tdir/output_2+2
     echo ; cat tst_2+2_out.txt
     echo
   fi
@@ -213,16 +262,16 @@
   if test $sepDir = 0 ; then
 #-- cleaning:
     echo "======================"
-    echo "Cleaning test directories:"
+    echo "Cleaning test directories:"	| tee -a $tdir/output_$tt
     if test $tt != $typ ; then
       cmdCLN="../tools/do_tst_2+2 -clean"
-      echo " clean tst_2+2 running: $cmdCLN"
-      $cmdCLN >> $tdir/output_2+2 2>&1
+      echo " clean tst_2+2 running: $cmdCLN"	| tee -a $tdir/output_$tt
+      $cmdCLN >> $tdir/output_$tt 2>&1
     fi
     if test $tt != $last_Tst ; then
       cmdCLN="./testreport -clean"
-      echo " clean dir running: $cmdCLN"
-      $cmdCLN > /dev/null 2>&1
+      echo " clean dir running: $cmdCLN"	| tee -a $tdir/output_$tt
+      $cmdCLN >> $tdir/output_$tt 2>&1
     fi
     echo "======================"
     echo