--- MITgcm/verification/testscript	2001/03/22 16:07:56	1.2.2.3
+++ MITgcm/verification/testscript	2007/08/30 16:51:50	1.30
@@ -1,177 +1,528 @@
-#!/bin/csh
+#!/bin/sh
 
-# Run this script from the verification directory
-# It will automatically configure, compile, run and verify all experiments
-# in the verification directory for whcih there is an results/output.txt
-# file.
-
-# This is the number of least-significant digits allows to be
-# in error before the test is classified as a "fail".
-
-set on_error_die
-set passaccuracy=7
-set SKIP=( )
-printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' " " Make " " " " Exact "# of" " " " " > summary.txt
-printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' Config Depend Compile Execute Match Digits Status Experiment >> summary.txt
-
-# Pass any arguments
-foreach arg ($argv)
- switch ($arg)
-  case -clean:
-   echo Cleaning ...
-   foreach dr ([a-zA-Z01-9]*)
-    if (-d $dr/input) then
-      echo Entering $dr
-      cd $dr/input
-      echo ""
-      make CLEAN
-      \rm -f {*output,*high,*low}.txt *.log
-      \rm -f ?ake*
-      cd ../..
-    endif
-   end
-   rm -f summary.txt
-   exit 0
-   breaksw
-  case -nodie:
-   unset on_error_die
-   breaksw
-  case -skip=*:
-   set SKIP = ( $SKIP `echo $arg | sed 's/-skip=//' | sed 's/,/ /g' `)
-   breaksw
-  default:
-   echo Unknown argument given to $0
-   exit 1
- endsw
-end
-
-foreach dr ([a-zA-Z01-9]*)
- if (! -d $dr/input ) continue
- foreach skip ($SKIP)
-  if ($skip == $dr) continue;continue
- end
- set config=-
- set makedepend=-
- set compile=-
- set execute=-
- set exactmatch=-
- set accuracy=-
- set pass=FAIL
- echo ==============================================================================
- echo Entering $dr
- cd $dr
- echo ""
- if (! -r results/output.txt) then
-  echo "                ***** No results to compare with *****"
-  if (! $?untested) set untested
-  set untested=($untested $dr)
- else
-  set config=NO
-  cd input
-  echo -n "  generating Makefile ..."
-  ../../../tools/genmake -mods=../code >&! make.log
-  if ($status == 0) set config=Yes
-  set makedepend=NO
-  echo " done."
-  echo -n "  make depend ..."
-  make cleanlinks depend >>& make.log
-  if ($status == 0) set makedepend=Yes
-  echo " done."
-  echo -n "  make ..."
-  make >>& make.log
-  if ($status) then
-   echo "               ***** An error occurred during make *****"
-   cat make.log
-   echo The error during compilation occured in \"$dr\"
-   set compile=NO
-   if ($?on_error_die) exit 1
-  else
-   set compile=Yes
-   set execute=NO
-  endif
-  echo " done."
-  echo -n "  running model ..."
-  if ($compile == 'Yes') then
-# ./mitgcmuv | & grep "D iters" > output.txt
-  ./mitgcmuv > & output.txt
-  if ($status == 0) then
-    set execute=Yes
-    set exactmatch=NO
-    set accuracy=-
-    echo " done."
-    grep "D iters" output.txt | sed 's/.*D iters, err =//' \
-    | grep "   0    " \
-    > high.txt
-    grep "D iters" ../results/output.txt \
-     | sed 's/.*D iters, err =//' \
-     | grep "   0    " \
-     > oldhigh.txt
-    diff oldhigh.txt high.txt > /dev/null
-    if ($status) then
-      echo "  output differs:"
-      diff oldhigh.txt high.txt
-      echo ""
-      @ lvl=0
-      set accuracy='.'
-      set fail
-      echo -n Trying reduced accuracy
-      while ($lvl <= $passaccuracy)
-        @ lvl+=1
-        echo -n " " $lvl
-        sed s/{$accuracy}E/E/ high.txt > low.txt
-        sed s/{$accuracy}E/E/ oldhigh.txt > oldlow.txt
-        diff oldlow.txt low.txt > /dev/null
-        if ($status == 0) then
-          unset fail
-          break
+compare_lines()
+{
+# use codelet to compare lines
+ if [ $verbose -gt 1 ]; then
+  cat tmp3.txt 1>&2
+ fi
+ return `./a.out < tmp3.txt`
+}
+
+testoutput_for_prop()
+{
+# testoutput_for_prop dir s1 label subdir
+#
+#  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
+#  using search strings s1 and text label 
+ 
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
+ fi
+ if [ -r $1/$4/output.txt ]; then
+  grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
+  lncnt=`wc -l tmp1.txt | awk '{print $1}' `
+  if [ $lncnt -lt 3 ]; then
+   if [ $verbose -gt 0 ]; then
+    echo Not enough lines of output when searching for "$2" 1>&2
+   fi
+   return 99
+  fi
+ else
+  echo testoutput_for_prop: output.txt from model run was not readable 1>&2
+  return 99
+ fi
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
+ fi
+ grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
+  lncnt=`wc -l tmp2.txt | awk '{print $1}' `
+  if [ $lncnt -lt 3 ]; then
+   if [ $verbose -gt 0 ]; then
+    echo Not enough lines of output when searching for "$2" 1>&2
+   fi
+   return 99
+  fi
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
+ fi
+ join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: compare_lines 1>&2
+ fi
+ compare_lines
+ digits_of_similarity=$?
+ if [ $digits_of_similarity -eq 99 ]; then
+  if [ $verbose -gt 0 ]; then
+   echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
+  fi
+  digits_of_similarity=99
+ else
+  if [ $verbose -gt 0 ]; then
+   echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
+  fi
+ fi
+ rm tmp1.txt tmp2.txt tmp3.txt
+
+ return $digits_of_similarity
+}
+
+dashnum()
+{
+# dashnum n1 n2 n3 ...
+#
+#  print numbers using %3i format or "--" if number = 99
+for num in $@
+do
+ if [ $num = 99 ]; then
+  printf ' --'
+ else
+  printf '%3i' $num
+ fi
+done
+}
+
+testoutput()
+{
+# testoutput diretory subdir
+#
+#  test output in "directory"
+
+if [ $debug -gt 0 ]; then
+ echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
+fi
+testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
+if [ $debug -gt 0 ]; then
+ echo testoutput: cg2dres=$cg2dres 1>&2
+fi
+
+if [ $longtest -gt 0 ]; then
+testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
+testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
+testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
+testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
+testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
+testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
+testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
+testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
+testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
+testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
+testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
+testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
+testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
+testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
+testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
+testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
+else
+testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
+testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
+testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
+testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
+testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
+testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
+testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
+testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
+fi
+
+dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+        $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+#printf '%3i' $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+#       $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+}
+
+genmakemodel()
+{
+# genmakemodel directory
+ ( cd $1;
+   if [ $quick -eq 0 -o ! -r Makefile ]; then
+    printf 'genmake ... ' 1>&2
+    find . -type l -exec rm {} \;
+    if [ $ieee -eq 0 ]; then
+     ../../../tools/genmake2 -mods=../code > make.log 2>&1
+    else
+     ../../../tools/genmake2 -ieee -mods=../code > make.log 2>&1
+    fi
+    if [ $? -ne 0 ]; then
+     tail make.log
+     echo genmakemodel: $genmake failed 1>&2
+     return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makecleancompile()
+{
+# makecleancompile directory
+ ( cd $1;
+   if [ $force -gt 0 ]; then
+    rm -f output.txt
+    printf 'make clean ... ' 2>&1
+    make CLEAN >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makecleancompile: make clean failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makecleanupafter()
+{
+# makeupafter directory
+ ( cd $1;
+   if [ $clean -gt 0 ]; then
+    rm -f output.txt
+    printf 'make clean ... ' 2>&1
+    make CLEAN >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makeupafter: make clean failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makedependmodel()
+{
+# makedependmodel directory
+ ( cd $1;
+   if [ $quick -eq 0 -o ! -r Makefile ]; then
+    printf 'make depend ... ' 1>&2
+    make cleanlinks >> make.log 2>&1
+    make depend >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makemodel: make depend failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makemodel()
+{
+# makemodel directory
+ ( cd $1;
+   if [ -r Makefile ]; then
+    printf 'make ... ' 1>&2
+    make >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+     tail make.log
+     echo failed 1>&2
+     return 1
+    else
+     echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+linkdata()
+{
+    # linkdata flag
+    #
+    # symbolically link data files to run directory
+    if test "x$1" = x1 ; then
+        (
+            cd $2
+            if test "x$ADM" = x ; then
+                files=`( cd ../input ; ls -1 | grep -v CVS )`
+                for i in $files ; do
+                    if test ! -d "../input/"$i ; then
+                        ln -sf "../input/"$i $i
+                    fi
+                done
+            else
+                files=`( cd ../input ; ls -1 *.bin | grep -v CVS )`
+                for i in $files ; do
+                    if test ! -d "../input/"$i ; then
+                        ln -sf "../input/"$i $i
+                    fi
+                done
+                files=`( cd ../input_ad ; ls -1 | grep -v CVS )`
+                for i in $files ; do
+                    if test ! -d "../input_ad/"$i ; then
+                        ln -sf "../input_ad/"$i $i
+                    fi
+                done
+            fi
+        )
+    fi
+}
+
+runmodel()
+{
+# runmodel directory exe
+#
+#  runs the model "exe" in "directory" (exe is relative to directory)
+
+ ( cd $1
+   if [ -x $2 ]; then
+    if [ $quick -eq 0 ]; then
+     rm -f output.txt
+    fi
+   if [ $debug -gt 0 ]; then
+    echo -n runmodel: pwd= 1>&2
+    pwd 1>&2
+   fi
+    printf 'runmodel: ' 1>&2
+     rm -f *.nc
+     make output.txt && return 0
+     return 1
+   fi
+ )
+}
+
+copynewresults()
+{
+# copynewresults directory
+#
+#  runs the model "exe" in "directory" (exe is relative to directory)
+
+ ( cd $1
+   if [ $newresults -eq 1 ]; then
+    if [ -r output.txt ]; then
+     cp -f output.txt ../results/
+     return 0
+    fi
+   fi
+ )
+}
+
+createcodelet()
+{
+# create codelet for comparing model output
+cat > tmp_cmpnum.f <<EOFA
+      program cmpnum
+      implicit none
+      real*8 a,b,diff
+      integer linnum,best
+      best=-16
+  99  read(*,*,end=70,err=60) linnum,a,b
+      diff=0.5*(abs(a)+abs(b))
+c     print *,a,b,diff,abs(a-b)/diff
+      if (diff.gt.1.e-12) then
+        diff=abs(a-b)/diff
+        if (diff.gt.0.) then
+c         print *,int(log10(diff)),diff
+          linnum=int(log10(diff))
+          best=max(best,linnum)
         endif
-        set accuracy=`echo $accuracy | sed 's/\./../'`
-      end
-      echo ""
-      if ($?fail ) then
-        if (! $?fails) set fails
-        set fails=($fails $dr)
-        set pass=FAIL
-        echo "                        ***** FAIL *****"
       else
-        if (! $?passes) set passes
-        set passes=($passes $dr)
-        set pass=Pass
-        echo ""
-        echo "                   ***** PASS (grade $lvl) *****"
+        if (best.eq.-16.and.diff.ne.0.) best=-22
       endif
-      set accuracy=$lvl
-    else
-      set exactmatch=Yes
-      set pass=Pass
-      echo ""
-      echo Model passed at highest accuracy.
-      echo ""
-      echo "                        ***** PASS *****"
-      if (! $?passes) set passes
-      set passes=($passes $dr)
-    endif
+      goto 99
+  60  stop 'cmpnum: An error occured reading a,b'
+  70  print *,-best
+      end
+EOFA
+f77 tmp_cmpnum.f
+if [ -x ./a.out ]; then
+ return 0
+else
+ echo createcodelet: failed to compile codelet 1>&2
+ exit 1
+fi
+}
+
+formatresults()
+{
+# formatresults expt genmake depend make run results*
+
+ nm=$1
+ printf '%s %s %s %s' $2 $3 $4 $5
+ shift; shift; shift; shift; shift;
+ printf '%3s' $@
+
+ if [ $1 = '--' ]; then
+  printf ' N/O '
+ else
+  if [ $1 -gt 12 ]; then
+   printf ' pass'
   else
-    echo " error"
-    echo "               ***** An error occured running the model *****"
-    tail output.txt
-    echo The runtime error occured in \"$dr\"
-    if ($?on_error_die) exit 1
-    set pass=-
-  endif
-  rm -f high.txt oldhigh.txt low.txt oldlow.txt output.txt make.log
-  endif
-  cd ..
- endif
- echo ""
- cd ..
-# Pretty summary
- printresults:
- printf '  %s\t  %s\t  %s\t  %s\t  %s\t %s\t %s\t%s\n' $config $makedepend $compile $execute $exactmatch $accuracy $pass $dr >> summary.txt
-end
-echo ==============================================================================
-echo ""
+   printf ' FAIL'
+  fi
+ fi
+ printf '  %s' $nm
+ printf '\n'
+
+}
+
+show_help()
+{
+cat - << EOF
+$0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
+ -help      Show this help message
+ -quick     Skip "genmake" and "make depend" if the Makefile exists
+ -newresults Copy the output.txt into results/
+ -quiet     Reduce the amount of output
+ -verbose   Produce copious amounts of output
+ -debug     Produce even more output which will mean nothing to most
+ -force     Do "make CLEAN" before compiling. This forces a complete rebuild.
+ -clean     Do "make CLEAN" after compiling and testing.
+ -shorttest Don't compare numeric output for mean and s.d. of variables.
+ -noieee    By default, $0 uses the -ieee option for genmake. This turns it off.
+ -cleanup   Aggresively removes all model output, executables and object files
+            and then exits. Use with care.
+
+Normal usage:
+ $0 *       Configure, compile, run and analyze in all experiment directories
+EOF
+}
+
+scandirs()
+{
+if [ $# -eq 0 ]; then
+ for arg in *
+  do
+   test -d $arg/input && echo $arg
+ done
+else
+ echo $*
+fi
+}
+
+clean_up()
+{
+# Find all executables, object files, CPP'd source and model output
+# and DELETE it.
+for opt in '-name "mitgcmuv*"' '-name "*.o"' '-name "*.f"' '-name "*.c"' '-name "fort.*"' '-name "make.log"' '-path *results -prune -o -name "*.meta"' '-path *results -prune -o -name "*.data"' '-type l'
+ do
+   echo Cleaning: find . $opt -exec rm {} \;
+   find . $opt -exec rm {} \;
+done
+}
+
+###############################################################################
+###############################################################################
+###############################################################################
+
+# Main function
+
+# Default properties
+debug=0
+verbose=1
+quick=0
+newresults=0
+force=0
+clean=0
+ieee=1
+longtest=1
+expts=''
+genmake=genmake
+
+# Process arguments
+for arg in $@
+do
+ case $arg in
+   -cleanup) clean_up; exit 0;;
+   -quick) quick=1;;
+   -newresults) newresults=1;;
+   -verbose) verbose=2;;
+   -debug) debug=1;;
+   -force) force=1;;
+   -clean) clean=1;;
+   -noieee) ieee=0;;
+   -shorttest) longtest=0;;
+   -quiet) verbose=0;;
+   -help) show_help; exit 0;;
+   -*) echo Unrecognized option:$arg; exit 9;;
+   *) test -d $arg && expts=`echo $expts $arg`;;
+ esac
+done
+
+if [ $force -gt 0 -a $quick -gt 0 ]; then
+ echo You specified -quick and -force together which conflict.
+ echo Please specify either -quick or -force or neither but not both.
+ exit 1
+fi
+
+#if [ ${#expts} -eq 0 ]; then
+# echo Scanning all directories
+# for arg in *
+#  do
+#   test -d $arg && expts=`echo $expts $arg`
+# done
+#fi
+expts=`scandirs $expts`
+
+createcodelet
+
+date > summary.txt
+if [ $longtest -gt 0 ]; then
+cat << EOF >> summary.txt
+                T           S           U           V
+C D M    c        m  s        m  s        m  s        m  s
+n p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
+f n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
+g d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
+
+EOF
+else
+cat << EOF >> summary.txt
+              T     S     U     V
+C D M    c                        
+n p a R  g  m  m  m  m  m  m  m  m
+f n k u  2  i  a  i  a  i  a  i  a
+g d e n  d  n  x  n  x  n  x  n  x
+
+EOF
+fi
+
+# Now configue, make, run and test in each directory
+for dir in $expts
+do
+ if [ -r $dir/results/output.txt ]; then
+ echo -------------------------------------------------------------------------------
+ echo
+ echo Experiment: $dir
+ echo
+ unset genmakepass makedepend make run
+ if [ $longtest -gt 0 ]; then
+  results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+ else
+  results='-- -- -- -- -- -- -- -- --'
+ fi
+ if [ -r $dir/build ]; then
+  seperatebuilddir=1
+  builddir=build
+  rundir=build
+  (cd $dir/input; rm -f *.{o,f,c,F,h} work* output.txt Make* make.log; )
+ else
+  seperatebuilddir=0
+  builddir=input
+  rundir=input
+ fi
+ genmakemodel $dir/$builddir && genmakepass=Y \
+ && makecleancompile $dir/$builddir \
+ && makedependmodel $dir/$builddir && makedepend=Y \
+ && makemodel $dir/$builddir && make=Y \
+ && linkdata $seperatebuilddir $dir/$rundir \
+ && runmodel $dir/$builddir mitgcmuv && run=Y \
+ && results=`testoutput $dir $rundir` \
+ && copynewresults $dir/$rundir \
+ && makecleanupafter $dir/$builddir
+ echo
+ formatresults $dir ${genmakepass:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results
+ echo
+ formatresults $dir ${genmakepass:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> summary.txt
+ fi
+done
+
+rm tmp_cmpnum.f a.out
 
-echo "Summary:"
-echo ""
+echo -------------------------------------------------------------------------------
+echo 
+date >> summary.txt
 cat summary.txt