--- MITgcm/verification/testscript	2001/06/29 18:28:44	1.7
+++ MITgcm/verification/testscript	2001/08/03 20:18:40	1.8
@@ -1,180 +1,396 @@
-#!/bin/csh -f
+#!/bin/sh
 
-# Run this script from the verification directory
-# It will automatically configure, compile, run and verify all experiments
-# in the verification directory for whcih there is an results/output.txt
-# file.
-
-# This is the number of least-significant digits allows to be
-# in error before the test is classified as a "fail".
-
-set on_error_die
-set digitsOfAccuracy=8
-set SKIP=( )
-printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' " " Make " " " " Exact "# of" " " " " > summary.txt
-printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' Config Depend Compile Execute Match Digits Status Experiment >> summary.txt
-
-# Pass any arguments
-foreach arg ($argv)
- switch ($arg)
-  case -clean:
-   echo Cleaning ...
-   foreach dr ([a-zA-Z01-9]*)
-    if (-d $dr/input) then
-      echo Entering $dr
-      cd $dr/input
-      echo ""
-      make CLEAN
-      \rm -f {*output,*high,*low}.txt *.log
-      \rm -f ?ake*
-      cd ../..
-    endif
-   end
-   rm -f summary.txt
-   exit 0
-   breaksw
-  case -nodie:
-   unset on_error_die
-   breaksw
-  case -skip=*:
-   set SKIP = ( $SKIP `echo $arg | sed 's/-skip=//' | sed 's/,/ /g' `)
-   breaksw
-  default:
-   echo Unknown argument given to $0
-   exit 1
- endsw
-end
-
-# Need to make sure we don't meet non-model problems ...
-unlimit
-
-foreach dr ([a-zA-Z01-9]*)
-
- if (! -d $dr/input ) continue
- foreach skip ($SKIP)
-  if ( `echo $dr | sed s/$skip.'*'//` == '' ) continue;continue
- end
- set config=-
- set makedepend=-
- set compile=-
- set execute=-
- set exactmatch=-
- set accuracy=-
- set pass=FAIL
- echo ==============================================================================
- echo Entering $dr
- cd $dr
- echo ""
- if (! -r results/output.txt) then
-  echo "                ***** No results to compare with *****"
-  if (! $?untested) set untested
-  set untested=($untested $dr)
+similar()
+{
+# similar digits a b
+#
+#  compare the floating point number a and b and successively reduced
+#  truncation until a match is found
+ for digits in 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
+ do
+   a=`printf '%22.'$digits'e\n' $1`
+   b=`printf '%22.'$digits'e\n' $2`
+   if [ $a = $b ]; then
+    if [ $verbose -gt 1 ]; then
+     echo $1 $2 $a 1>&2
+    fi
+    break
+   fi
+ done
+ return $digits
+}
+
+compare_lines()
+{
+# compare_lines digits < file
+#
+#  read each line of file and test the 2nd and 3rd column values for
+#  similarity based on the function similar()
+ worst_so_far=$1
+ while read linenum a b
+ do
+  if [ $debug -gt 0 ]; then
+   echo compare_lines: similar a=$a b=$b 1>&2
+  fi
+  similar $a $b
+  digits=$?
+  if [ $digits -lt $worst_so_far ]; then
+   worst_so_far=$digits
+  fi
+ done
+  if [ $debug -gt 0 ]; then
+   echo compare_lines: worst_so_far=$worst_so_far 1>&2
+  fi
+ return $worst_so_far
+}
+
+compare_lines2()
+{
+# use codelet to compare lines
+ return `./a.out < tmp3.txt`
+}
+
+testoutput_for_prop()
+{
+# testoutput_for_prop dir s1 label
+#
+#  compares files in $dir/input/output.txt and $dir/results.output.txt
+#  using search strings s1 and text label 
+ 
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: grep "$2" $1/input/output.txt 1>&2
+ fi
+ if [ -r $1/input/output.txt ]; then
+  grep "$2" $1/input/output.txt | sed 's/.*=//' | nl > tmp1.txt
  else
-  set config=NO
-  cd input
-  echo -n "  generating Makefile ..."
-  ../../../tools/genmake -mods=../code >&! make.log
-  if ($status == 0) set config=Yes
-  set makedepend=NO
-  echo " done."
-  echo -n "  make depend ..."
-  make cleanlinks depend >>& make.log
-  if ($status == 0) set makedepend=Yes
-  echo " done."
-  echo -n "  make ..."
-  make >>& make.log
-  if ($status) then
-   echo "               ***** An error occurred during make *****"
-   cat make.log
-   echo The error during compilation occured in \"$dr\"
-   set compile=NO
-   if ($?on_error_die) exit 1
-  else
-   set compile=Yes
-   set execute=NO
-  endif
-  echo " done."
-  echo -n "  running model ..."
-  if ($compile == 'Yes') then
-  ./mitgcmuv > & output.txt
-  if ($status == 0) then
-    set execute=Yes
-    set exactmatch=NO
-    set accuracy=-
-    echo " done."
-
-# Compare output based on cg2d residual
-    grep "cg2d_init_res" output.txt | sed 's/.*=//' | nl > high.txt
-    grep "cg2d_init_res" ../results/output.txt | sed 's/.*=//' | nl > oldhigh.txt
-    join high.txt oldhigh.txt > low.txt
-    set nlines=`wc -l low.txt | awk '{print $1}'`
-    echo "  comparing $nlines lines of output"
-    @ line = 1
-    set bestacc=99;
-    set worstline='Exact match'
-    while ($line <= $nlines)
-     set vals=(`grep "^$line" low.txt | awk '{print $2,$3}'`)
-     @ line += 1
-     unset acc
-     foreach dp (14 13 12 11 10 9 8 7 6 5 4 3 2 1 0)
-      set cmp=(`printf '%23.'$dp'e %23.'$dp'e\n' $vals[1] $vals[2]`)
-      if ($cmp[1] =~ $cmp[2]) then
-       if (! $?acc) then
-        set acc=$dp
-        break
-       endif
-      endif
-     end
-     if (! $?acc) set acc=0
-     if ($acc < $bestacc) then
-       set worstline=($line $vals[1] $vals[2] `printf '%23.'$acc'e' $vals[1]` $acc)
-       set bestacc=$acc
-     endif
-#    echo $line $vals[1] $vals[2] `printf '%23.'$acc'e' $vals[1]` $acc
-    end
-    if ($nlines == 0) then
-     set bestacc=0
-     set worstline="Could find no lines in output.txt to compare"
-    endif
-    if ($bestacc == 14 & $?exactmatch) then
-     set exactmatch=Yes
-     echo " "
-     echo "                        ***** EXACT MATCH *****"
-    endif
-    if ($bestacc > $digitsOfAccuracy) then
-     set pass=Pass
-     echo " "
-     echo "                      ***** PASS (digits=$bestacc) *****"
-     echo " "
-     echo " worstline: " $worstline
+  echo testoutput_for_prop: output.txt from model run was not readable 1>&2
+  return 99
+ fi
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
+ fi
+ grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
+ fi
+ join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
+ if [ $debug -gt 0 ]; then
+  echo testoutput_for_prop: compare_lines 1>&2
+ fi
+ #compare_lines 99 < tmp3.txt
+ compare_lines2
+ digits_of_similarity=$?
+ if [ $digits_of_similarity -eq 99 ]; then
+  if [ $verbose -gt 0 ]; then
+   echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
+  fi
+  digits_of_similarity=99
+ else
+  if [ $verbose -gt 0 ]; then
+   echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
+  fi
+ fi
+ rm tmp1.txt tmp2.txt tmp3.txt
+
+ return $digits_of_similarity
+}
+
+dashnum()
+{
+# dashnum n1 n2 n3 ...
+#
+#  print numbers using %3i format or "--" if number = 99
+for num in $@
+do
+ if [ $num = 99 ]; then
+  printf ' --'
+ else
+  printf '%3i' $num
+ fi
+done
+}
+
+testoutput()
+{
+# testoutput diretory
+#
+#  test output in "directory"
+
+if [ $debug -gt 0 ]; then
+ echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
+fi
+testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual"; cg2dres=$?
+if [ $debug -gt 0 ]; then
+ echo testoutput: cg2dres=$cg2dres 1>&2
+fi
+
+if [ $testmore -gt 0 ]; then
+testoutput_for_prop $1 "dynstat_theta_min" "theta minimum"; tmin=$?
+testoutput_for_prop $1 "dynstat_theta_max" "theta maximum"; tmax=$?
+testoutput_for_prop $1 "dynstat_theta_mean" "theta mean"; tmean=$?
+testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d."; tsd=$?
+testoutput_for_prop $1 "dynstat_salt_min" "salt minimum"; smin=$?
+testoutput_for_prop $1 "dynstat_salt_max" "salt maximum"; smax=$?
+testoutput_for_prop $1 "dynstat_salt_mean" "salt mean"; smean=$?
+testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d."; ssd=$?
+testoutput_for_prop $1 "dynstat_uvel_min" "U minimum"; umin=$?
+testoutput_for_prop $1 "dynstat_uvel_max" "U maximum"; umax=$?
+testoutput_for_prop $1 "dynstat_uvel_mean" "U mean"; umean=$?
+testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d."; usd=$?
+testoutput_for_prop $1 "dynstat_vvel_min" "V minimum"; vmin=$?
+testoutput_for_prop $1 "dynstat_vvel_max" "V maximum"; vmax=$?
+testoutput_for_prop $1 "dynstat_vvel_mean" "V mean"; vmean=$?
+testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d."; vsd=$?
+else
+testoutput_for_prop $1 "dynstat_theta_min" "theta minimum"; tmin=$?
+testoutput_for_prop $1 "dynstat_theta_max" "theta maximum"; tmax=$?
+testoutput_for_prop $1 "dynstat_salt_min" "salt minimum"; smin=$?
+testoutput_for_prop $1 "dynstat_salt_max" "salt maximum"; smax=$?
+testoutput_for_prop $1 "dynstat_uvel_min" "U minimum"; umin=$?
+testoutput_for_prop $1 "dynstat_uvel_max" "U maximum"; umax=$?
+testoutput_for_prop $1 "dynstat_vvel_min" "V minimum"; vmin=$?
+testoutput_for_prop $1 "dynstat_vvel_max" "V maximum"; vmax=$?
+fi
+
+dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+        $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+#printf '%3i' $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
+#       $umin $umax $umean $usd $vmin $vmax $vmean $vsd
+}
+
+genmakemodel()
+{
+# genmakemodel directory
+ ( cd $1;
+   if [ $quick -eq 0 -o ! -r Makefile ]; then
+    printf 'genmake ... ' 1>&2
+    ../../../tools/genmake -mods=../code > make.log 2>&1
+    if [ $? -ne 0 ]; then
+     tail make.log
+     echo makemodel: genmake failed 1>&2
+     return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makedependmodel()
+{
+# makedependmodel directory
+ ( cd $1;
+   if [ $quick -eq 0 -o ! -r Makefile ]; then
+    printf 'make depend ... ' 1>&2
+    make cleanlinks >> make.log 2>&1
+    make depend >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+      tail make.log
+      echo makemodel: make depend failed 1>&2
+      return 1
+    else
+      echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+makemodel()
+{
+# makemodel directory
+ ( cd $1;
+   if [ -r Makefile ]; then
+    printf 'make ... ' 1>&2
+    make >> make.log 2>&1
+    if [ $? -ne 0 ]; then
+     tail make.log
+     echo failed 1>&2
+     return 1
+    else
+     echo succesful 1>&2
+    fi
+   fi
+ )
+}
+
+runmodel()
+{
+# runmodel directory exe
+#
+#  runs the model "exe" in "directory" (exe is relative to directory)
+
+ ( cd $1
+   if [ -x $2 ]; then
+    if [ ! -r output.txt -o $quick -eq 0 ]; then
+     echo runmodel: running... 1>&2
+     ( ./$2 > output.txt 2>&1 ) && return 0
+     return 1
     else
-     echo " "
-     echo "                      ***** FAIL (digits=$bestacc) *****"
-     echo " "
-     echo " worstline: " $worstline
-    endif
-    set accuracy=$bestacc
+     echo runmodel: output.txt is newer than executable 1>&2
+     return 0
+    fi
+   else
+    echo runmodel: executable \"$1/$2\" is missing 1>&2
+    return 1
+   fi
+ )
+}
+
+createcodelet()
+{
+# create codelet for comparing model output
+cat > tmp_cmpnum.f <<EOFA
+      program cmpnum
+      implicit none
+      real*8 a,b,diff
+      integer linnum,best
+      best=-16
+  99  read(*,*,end=70,err=60) linnum,a,b
+      diff=0.5*(abs(a)+abs(b))
+      if (diff.gt.0.) then
+        diff=abs(a-b)/diff
+        if (diff.gt.0.) then
+c         print *,int(log10(diff)),diff
+          linnum=int(log10(diff))
+          best=max(best,linnum)
+c       else
+c         print *,-16
+        endif
+c     else
+c       diff=0.
+c       print *,-16
+      endif
+      goto 99
+  60  stop 'cmpnum: An error occured reading a,b'
+  70  print *,-best
+      end
+EOFA
+f77 tmp_cmpnum.f
+if [ -x ./a.out ]; then
+ return 0
+else
+ echo createcodelet: failed to compile codelet 1>&2
+ exit 1
+fi
+}
+
+formatresults()
+{
+# formatresults expt genmake depend make run results*
+
+ nm=$1
+ printf '%s %s %s %s' $2 $3 $4 $5
+ shift; shift; shift; shift; shift;
+ printf '%3s' $@
+
+ if [ $1 = '--' ]; then
+  printf ' N/O '
+ else
+  if [ $1 -gt 12 ]; then
+   printf ' pass'
   else
-    echo " error"
-    echo "               ***** An error occured running the model *****"
-    tail output.txt
-    echo The runtime error occured in \"$dr\"
-    if ($?on_error_die) exit 1
-    set pass=-
-  endif
-  rm -f high.txt oldhigh.txt low.txt oldlow.txt output.txt make.log
-  endif
-  cd ..
- endif
- echo ""
- cd ..
-# Pretty summary
- printresults:
- printf '  %s\t  %s\t  %s\t  %s\t  %s\t %s\t %s\t%s\n' $config $makedepend $compile $execute $exactmatch $accuracy $pass $dr >> summary.txt
-end
-echo ==============================================================================
-echo ""
+   printf ' FAIL'
+  fi
+ fi
+ printf '  %s' $nm
+ printf '\n'
+
+}
+
+show_help()
+{
+cat - << EOF
+$0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
+ -help      Show this help message
+ -quick     Skip "genmake" and "make depend" if the Makefile exists
+ -quiet     Reduce the amount of output
+ -verbose   Produce copious amounts of output
+
+Normal usage:
+ $0 *       Configure, compile, run and analyze in all experiment directories
+EOF
+}
+
+###############################################################################
+
+# Main function
+
+# Default properties
+debug=0
+verbose=1
+quick=0
+testmore=0
+expts=''
+
+# Process arguments
+for arg in $@
+do
+ case $arg in
+   -quick) quick=1;;
+   -verbose) verbose=2;;
+   -debug) debug=1;;
+   -testmore) testmore=1;;
+   -quiet) verbose=0;;
+   -help) show_help; exit 0;;
+   *) test -d $arg && expts=`echo $expts $arg`;;
+ esac
+done
+
+if [ ${#expts} -eq 0 ]; then
+ echo Scanning all directories
+ for arg in *
+  do
+   test -d $arg && expts=`echo $expts $arg`
+ done
+fi
+
+createcodelet
+
+if [ $testmore -eq 0 ]; then
+cat << EOF > summary.txt
+                T           S           U           V
+C D M    c        m  s        m  s        m  s        m  s
+n p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
+f n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
+g d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
+
+EOF
+else
+cat << EOF > summary.txt
+              T     S     U     V
+C D M    c                        
+n p a R  g  m  m  m  m  m  m  m  m
+f n k u  2  i  a  i  a  i  a  i  a
+g d e n  d  n  x  n  x  n  x  n  x
+
+EOF
+fi
+
+# Now configue, make, run and test in each directory
+for dir in $expts
+do
+ if [ -r $dir/results/output.txt ]; then
+ echo -------------------------------------------------------------------------------
+ echo
+ echo Experiment: $dir
+ echo
+ unset genmake makedepend make run
+ results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
+ results='-- -- -- -- -- -- -- -- --'
+ genmakemodel $dir/input && genmake=Y \
+ && makedependmodel $dir/input && makedepend=Y \
+ && makemodel $dir/input && make=Y \
+ && runmodel $dir/input mitgcmuv && run=Y \
+ && results=`testoutput $dir`
+ echo
+ formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results
+ echo
+ formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> summary.txt
+ fi
+done
+
+rm tmp_cmpnum.f a.out
 
-echo "Summary:"
-echo ""
+echo -------------------------------------------------------------------------------
+echo 
 cat summary.txt