--- MITgcm/verification/testscript	2001/06/29 18:28:44	1.7
+++ MITgcm/verification/testscript	2003/06/23 22:13:48	1.24
@@ -1,180 +1,481 @@
-#!/bin/csh -f
+#!/bin/sh
 
-# Run this script from the verification directory
-# It will automatically configure, compile, run and verify all experiments
-# in the verification directory for whcih there is an results/output.txt
-# file.
-
-# This is the number of least-significant digits allows to be
-# in error before the test is classified as a "fail".
-
-set on_error_die
-set digitsOfAccuracy=8
-set SKIP=( )
-printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' " " Make " " " " Exact "# of" " " " " > summary.txt
-printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' Config Depend Compile Execute Match Digits Status Experiment >> summary.txt
-
-# Pass any arguments
-foreach arg ($argv)
- switch ($arg)
-  case -clean:
-   echo Cleaning ...
-   foreach dr ([a-zA-Z01-9]*)
-    if (-d $dr/input) then
-      echo Entering $dr
-      cd $dr/input
-      echo ""
-      make CLEAN
-      \rm -f {*output,*high,*low}.txt *.log
-      \rm -f ?ake*
-      cd ../..
-    endif
-   end
-   rm -f summary.txt
-   exit 0
-   breaksw
-  case -nodie:
-   unset on_error_die
-   breaksw
-  case -skip=*:
-   set SKIP = ( $SKIP `echo $arg | sed 's/-skip=//' | sed 's/,/ /g' `)
-   breaksw
-  default:
-   echo Unknown argument given to $0
-   exit 1
- endsw
-end
-
-# Need to make sure we don't meet non-model problems ...
-unlimit
-
-foreach dr ([a-zA-Z01-9]*)
-
- if (! -d $dr/input ) continue
- foreach skip ($SKIP)
-  if ( `echo $dr | sed s/$skip.'*'//` == '' ) continue;continue
- end
- set config=-
- set makedepend=-
- set compile=-
- set execute=-
- set exactmatch=-
- set accuracy=-
- set pass=FAIL
- echo ==============================================================================
- echo Entering $dr
- cd $dr
- echo ""
- if (! -r results/output.txt) then
-  echo "                ***** No results to compare with *****"
-  if (! $?untested) set untested
-  set untested=($untested $dr)
- else
-  set config=NO
-  cd input
-  echo -n "  generating Makefile ..."
-  ../../../tools/genmake -mods=../code >&! make.log
-  if ($status == 0) set config=Yes
-  set makedepend=NO
-  echo " done."
-  echo -n "  make depend ..."
-  make cleanlinks depend >>& make.log
-  if ($status == 0) set makedepend=Yes
-  echo " done."
-  echo -n "  make ..."
-  make >>& make.log
-  if ($status) then
-   echo "               ***** An error occurred during make *****"
-   cat make.log
-   echo The error during compilation occured in \"$dr\"
-   set compile=NO
-   if ($?on_error_die) exit 1
-  else
-   set compile=Yes
-   set execute=NO
-  endif
-  echo " done."
-  echo -n "  running model ..."
-  if ($compile == 'Yes') then
-  ./mitgcmuv > & output.txt
-  if ($status == 0) then
-    set execute=Yes
-    set exactmatch=NO
-    set accuracy=-
-    echo " done."
-
-# Compare output based on cg2d residual
-    grep "cg2d_init_res" output.txt | sed 's/.*=//' | nl > high.txt
-    grep "cg2d_init_res" ../results/output.txt | sed 's/.*=//' | nl > oldhigh.txt
-    join high.txt oldhigh.txt > low.txt
-    set nlines=`wc -l low.txt | awk '{print $1}'`
-    echo "  comparing $nlines lines of output"
-    @ line = 1
-    set bestacc=99;
-    set worstline='Exact match'
-    while ($line <= $nlines)
-     set vals=(`grep "^$line" low.txt | awk '{print $2,$3}'`)
-     @ line += 1
-     unset acc
-     foreach dp (14 13 12 11 10 9 8 7 6 5 4 3 2 1 0)
-      set cmp=(`printf '%23.'$dp'e %23.'$dp'e\n' $vals[1] $vals[2]`)
-      if ($cmp[1] =~ $cmp[2]) then
-       if (! $?acc) then
-        set acc=$dp
-        break
-       endif
+compare_lines()
+{
+# use codelet to compare lines
+ if [ $verbose -gt 1 ]; then
+  cat tmp3.txt 1>&2
+ fi
+ return `./a.out < tmp3.txt`
+}
+
+testoutput_for_prop()
+{
+# testoutput_for_prop dir s1 label subdir
+#
+#  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
+#  using search strings s1 and text label 
+ 
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
+ fi
+ if [ -r $1/$4/output.txt ]; then
+  grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
+  lncnt=`wc -l tmp1.txt | awk '{print $1}' `
+  if [ $lncnt -lt 3 ]; then
+   if [ $verbose -gt 0 ]; then
+    echo Not enough lines of output when searching for "$2" 1>&2
+   fi
+   return 99
+  fi
+ else
+  echo testoutput_for_prop: output.txt from model run was not readable 1>&2
+  return 99
+ fi
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
+ fi
+ grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
+  lncnt=`wc -l tmp2.txt | awk '{print $1}' `
+  if [ $lncnt -lt 3 ]; then
+   if [ $verbose -gt 0 ]; then
+    echo Not enough lines of output when searching for "$2" 1>&2
+   fi
+   return 99
+  fi
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
+ fi
+ join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: compare_lines 1>&2
+ fi
+ compare_lines
+ digits_of_similarity=$?
+ if [ $digits_of_similarity -eq 99 ]; then
+  if [ $verbose -gt 0 ]; then
+   echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
+  fi
+  digits_of_similarity=99
+ else
+  if [ $verbose -gt 0 ]; then
+   echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
+  fi
+ fi
+ rm tmp1.txt tmp2.txt tmp3.txt
+
+ return $digits_of_similarity
+}
+
+dashnum()
+{
+# dashnum n1 n2 n3 ...
+#
+#  print numbers using %3i format or "--" if number = 99
+for num in $@
+do
+ if [ $num = 99 ]; then
+  printf ' --'
+ else
+  printf '%3i' $num
+ fi
+done
+}
+
+testoutput()
+{
+# testoutput diretory subdir
+#
+#  test output in "directory"
+
+if [ $debug -gt 0 ]; then
+ echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
+fi
+testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
+if [ $debug -gt 0 ]; then
+ echo testoutput: cg2dres=$cg2dres 1>&2
+fi
+
+if [ $longtest -gt 0 ]; then
+testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
+testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
+testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
+testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
+testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
+testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
+testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
+testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
+testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
+testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
+testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
+testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
+testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
+testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
+testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
+testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
+else
+testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
+testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
+testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
+testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
+testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
+testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
+testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
+testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
+fi
+
+dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+        $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+#printf '%3i' $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+#       $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+}
+
+genmakemodel()
+{
+# genmakemodel directory
+ ( cd $1;
+   if [ $quick -eq 0 -o ! -r Makefile ]; then
+    printf 'genmake ... ' 1>&2
+    if [ $ieee -eq 0 ]; then
+     ../../../tools/genmake -mods=../code > make.log 2>&1
+    else
+     ../../../tools/genmake -ieee -mods=../code > make.log 2>&1
+    fi
+    if [ $? -ne 0 ]; then
+     tail make.log
+     echo genmakemodel: genmake failed 1>&2
+     return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makecleancompile()
+{
+# makecleancompile directory
+ ( cd $1;
+   if [ $force -gt 0 ]; then
+    rm -f output.txt
+    printf 'make clean ... ' 2>&1
+    make CLEAN >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makecleancompile: make clean failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makecleanupafter()
+{
+# makeupafter directory
+ ( cd $1;
+   if [ $clean -gt 0 ]; then
+    rm -f output.txt
+    printf 'make clean ... ' 2>&1
+    make CLEAN >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makeupafter: make clean failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makedependmodel()
+{
+# makedependmodel directory
+ ( cd $1;
+   if [ $quick -eq 0 -o ! -r Makefile ]; then
+    printf 'make depend ... ' 1>&2
+    make cleanlinks >> make.log 2>&1
+    make depend >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makemodel: make depend failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makemodel()
+{
+# makemodel directory
+ ( cd $1;
+   if [ -r Makefile ]; then
+    printf 'make ... ' 1>&2
+    make >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+     tail make.log
+     echo failed 1>&2
+     return 1
+    else
+     echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+linkdata()
+{
+# linkdata flag 
+# symbolically link data files to run directory
+ if [ $1 -ne 0 ]; then
+# if [ ! -r $2 ]; then
+#  mkdir $2
+# fi
+  ( cd $2; ln -sf ../input/* .)
+ fi
+}
+
+runmodel()
+{
+# runmodel directory exe
+#
+#  runs the model "exe" in "directory" (exe is relative to directory)
+
+ ( cd $1
+   if [ -x $2 ]; then
+    if [ $quick -eq 0 ]; then
+     rm -f output.txt
+    fi
+    printf 'runmodel: ' 1>&2
+     make output.txt && return 0
+     return 1
+   fi
+ )
+}
+
+createcodelet()
+{
+# create codelet for comparing model output
+cat > tmp_cmpnum.f <<EOFA
+      program cmpnum
+      implicit none
+      real*8 a,b,diff
+      integer linnum,best
+      best=-16
+  99  read(*,*,end=70,err=60) linnum,a,b
+      diff=0.5*(abs(a)+abs(b))
+c     print *,a,b,diff,abs(a-b)/diff
+      if (diff.gt.1.e-12) then
+        diff=abs(a-b)/diff
+        if (diff.gt.0.) then
+c         print *,int(log10(diff)),diff
+          linnum=int(log10(diff))
+          best=max(best,linnum)
+        endif
+      else
+        if (best.eq.-16.and.diff.ne.0.) best=-22
       endif
-     end
-     if (! $?acc) set acc=0
-     if ($acc < $bestacc) then
-       set worstline=($line $vals[1] $vals[2] `printf '%23.'$acc'e' $vals[1]` $acc)
-       set bestacc=$acc
-     endif
-#    echo $line $vals[1] $vals[2] `printf '%23.'$acc'e' $vals[1]` $acc
-    end
-    if ($nlines == 0) then
-     set bestacc=0
-     set worstline="Could find no lines in output.txt to compare"
-    endif
-    if ($bestacc == 14 & $?exactmatch) then
-     set exactmatch=Yes
-     echo " "
-     echo "                        ***** EXACT MATCH *****"
-    endif
-    if ($bestacc > $digitsOfAccuracy) then
-     set pass=Pass
-     echo " "
-     echo "                      ***** PASS (digits=$bestacc) *****"
-     echo " "
-     echo " worstline: " $worstline
-    else
-     echo " "
-     echo "                      ***** FAIL (digits=$bestacc) *****"
-     echo " "
-     echo " worstline: " $worstline
-    endif
-    set accuracy=$bestacc
+      goto 99
+  60  stop 'cmpnum: An error occured reading a,b'
+  70  print *,-best
+      end
+EOFA
+f77 tmp_cmpnum.f
+if [ -x ./a.out ]; then
+ return 0
+else
+ echo createcodelet: failed to compile codelet 1>&2
+ exit 1
+fi
+}
+
+formatresults()
+{
+# formatresults expt genmake depend make run results*
+
+ nm=$1
+ printf '%s %s %s %s' $2 $3 $4 $5
+ shift; shift; shift; shift; shift;
+ printf '%3s' $@
+
+ if [ $1 = '--' ]; then
+  printf ' N/O '
+ else
+  if [ $1 -gt 12 ]; then
+   printf ' pass'
   else
-    echo " error"
-    echo "               ***** An error occured running the model *****"
-    tail output.txt
-    echo The runtime error occured in \"$dr\"
-    if ($?on_error_die) exit 1
-    set pass=-
-  endif
-  rm -f high.txt oldhigh.txt low.txt oldlow.txt output.txt make.log
-  endif
-  cd ..
- endif
- echo ""
- cd ..
-# Pretty summary
- printresults:
- printf '  %s\t  %s\t  %s\t  %s\t  %s\t %s\t %s\t%s\n' $config $makedepend $compile $execute $exactmatch $accuracy $pass $dr >> summary.txt
-end
-echo ==============================================================================
-echo ""
+   printf ' FAIL'
+  fi
+ fi
+ printf '  %s' $nm
+ printf '\n'
+
+}
+
+show_help()
+{
+cat - << EOF
+$0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
+ -help      Show this help message
+ -quick     Skip "genmake" and "make depend" if the Makefile exists
+ -quiet     Reduce the amount of output
+ -verbose   Produce copious amounts of output
+ -debug     Produce even more output which will mean nothing to most
+ -force     Do "make CLEAN" before compiling. This forces a complete rebuild.
+ -clean     Do "make CLEAN" after compiling and testing.
+ -shorttest Don't compare numeric output for mean and s.d. of variables.
+ -noieee    By default, $0 uses the -ieee option for genmake. This turns it off.
+ -cleanup   Aggresively removes all model output, executables and object files
+            and then exits. Use with care.
+
+Normal usage:
+ $0 *       Configure, compile, run and analyze in all experiment directories
+EOF
+}
+
+scandirs()
+{
+if [ $# -eq 0 ]; then
+ for arg in *
+  do
+   test -d $arg/input && echo $arg
+ done
+else
+ echo $*
+fi
+}
+
+clean_up()
+{
+# Find all executables, object files, CPP'd source and model output
+# and DELETE it.
+for opt in '-name "mitgcmuv*"' '-name "*.o"' '-name "*.f"' '-name "*.c"' '-name "fort.*"' '-name "make.log"' '-path *results -prune -o -name "*.meta"' '-path *results -prune -o -name "*.data"' '-type l'
+ do
+   echo Cleaning: find . $opt -exec rm {} \;
+   find . $opt -exec rm {} \;
+done
+}
+
+###############################################################################
+###############################################################################
+###############################################################################
+
+# Main function
+
+# Default properties
+debug=0
+verbose=1
+quick=0
+force=0
+clean=0
+ieee=1
+longtest=1
+expts=''
+
+# Process arguments
+for arg in $@
+do
+ case $arg in
+   -cleanup) clean_up; exit 0;;
+   -quick) quick=1;;
+   -verbose) verbose=2;;
+   -debug) debug=1;;
+   -force) force=1;;
+   -clean) clean=1;;
+   -noieee) ieee=0;;
+   -shorttest) longtest=0;;
+   -quiet) verbose=0;;
+   -help) show_help; exit 0;;
+   -*) echo Unrecognized option:$arg; exit 9;;
+   *) test -d $arg && expts=`echo $expts $arg`;;
+ esac
+done
+
+if [ $force -gt 0 -a $quick -gt 0 ]; then
+ echo You specified -quick and -force together which conflict.
+ echo Please specify either -quick or -force or neither but not both.
+ exit 1
+fi
+
+#if [ ${#expts} -eq 0 ]; then
+# echo Scanning all directories
+# for arg in *
+#  do
+#   test -d $arg && expts=`echo $expts $arg`
+# done
+#fi
+expts=`scandirs $expts`
+
+createcodelet
+
+date > summary.txt
+if [ $longtest -gt 0 ]; then
+cat << EOF >> summary.txt
+                T           S           U           V
+C D M    c        m  s        m  s        m  s        m  s
+n p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
+f n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
+g d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
+
+EOF
+else
+cat << EOF >> summary.txt
+              T     S     U     V
+C D M    c                        
+n p a R  g  m  m  m  m  m  m  m  m
+f n k u  2  i  a  i  a  i  a  i  a
+g d e n  d  n  x  n  x  n  x  n  x
+
+EOF
+fi
+
+# Now configue, make, run and test in each directory
+for dir in $expts
+do
+ if [ -r $dir/results/output.txt ]; then
+ echo -------------------------------------------------------------------------------
+ echo
+ echo Experiment: $dir
+ echo
+ unset genmake makedepend make run
+ if [ $longtest -gt 0 ]; then
+  results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+ else
+  results='-- -- -- -- -- -- -- -- --'
+ fi
+ if [ -r $dir/build ]; then
+  seperatebuilddir=1
+  builddir=build
+  rundir=build
+  (cd $dir/input; rm -f *.{o,f,c,F} work* output.txt Make* make.log; )
+  (cd $dir/build; ln -sf ../input/* .)
+ else
+  seperatebuilddir=0
+  builddir=input
+  rundir=input
+ fi
+ genmakemodel $dir/$builddir && genmake=Y \
+ && makecleancompile $dir/$builddir \
+ && makedependmodel $dir/$builddir && makedepend=Y \
+ && makemodel $dir/$builddir && make=Y \
+ && linkdata $seperatebuilddir $dir/$rundir \
+ && runmodel $dir/$builddir mitgcmuv && run=Y \
+ && results=`testoutput $dir $rundir` \
+ && makecleanupafter $dir/$builddir
+ echo
+ formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results
+ echo
+ formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> summary.txt
+ fi
+done
+
+rm tmp_cmpnum.f a.out
 
-echo "Summary:"
-echo ""
+echo -------------------------------------------------------------------------------
+echo 
+date >> summary.txt
 cat summary.txt