/[MITgcm]/MITgcm/verification/testscript
ViewVC logotype

Diff of /MITgcm/verification/testscript

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.2 by adcroft, Fri Mar 9 19:05:26 2001 UTC revision 1.24 by jmc, Mon Jun 23 22:13:48 2003 UTC
# Line 1  Line 1 
1  #!/bin/csh  #!/bin/sh
2    
3  # Run this script from the verification directory  compare_lines()
4  # It will automatically configure, compile, run and verify all experiments  {
5  # in the verification directory for whcih there is an results/output.txt  # use codelet to compare lines
6  # file.   if [ $verbose -gt 1 ]; then
7      cat tmp3.txt 1>&2
8  # This is the number of least-significant digits allows to be   fi
9  # in error before the test is classified as a "fail".   return `./a.out < tmp3.txt`
10    }
11  set on_error_die  
12  set passaccuracy=7  testoutput_for_prop()
13  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' " " Make " " " " Exact "# of" " " > summary.txt  {
14  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' Config Depend Compile Execute Match Digits Experiment >> summary.txt  # testoutput_for_prop dir s1 label subdir
15    #
16  # Pass any arguments  #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
17  foreach arg ($argv)  #  using search strings s1 and text label
18   switch ($arg)  
19    case -clean:   if [ $debug -gt 0 ]; then
20     echo Cleaning ...    echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
21     foreach dr ([a-zA-Z01-9]*)   fi
22      if (-d $dr/input) then   if [ -r $1/$4/output.txt ]; then
23        echo Entering $dr    grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
24        cd $dr/input    lncnt=`wc -l tmp1.txt | awk '{print $1}' `
25        echo ""    if [ $lncnt -lt 3 ]; then
26        make CLEAN     if [ $verbose -gt 0 ]; then
27        \rm -f {*output,*high,*low}.txt *.log      echo Not enough lines of output when searching for "$2" 1>&2
28        \rm -f ?ake*     fi
29        cd ../..     return 99
30      endif    fi
31     end   else
32     rm -f summary.txt    echo testoutput_for_prop: output.txt from model run was not readable 1>&2
33     exit 0    return 99
34     breaksw   fi
35    case -nodie:   if [ $debug -gt 0 ]; then
36     unset on_error_die    echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
37     breaksw   fi
38    default:   grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
39     echo Unknown argument given to $0    lncnt=`wc -l tmp2.txt | awk '{print $1}' `
40     exit 1    if [ $lncnt -lt 3 ]; then
41   endsw     if [ $verbose -gt 0 ]; then
42  end      echo Not enough lines of output when searching for "$2" 1>&2
43       fi
44  foreach dr ([a-zA-Z01-9]*)     return 99
45   if (! -d $dr/input ) continue    fi
46   set config=-   if [ $debug -gt 0 ]; then
47   set makedepend=-    echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
48   set compile=-   fi
49   set execute=-   join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
50   set exactmatch=-   if [ $debug -gt 0 ]; then
51   set accuracy=-    echo testoutput_for_prop: compare_lines 1>&2
52   echo ==============================================================================   fi
53   echo Entering $dr   compare_lines
54   cd $dr   digits_of_similarity=$?
55   echo ""   if [ $digits_of_similarity -eq 99 ]; then
56   if (! -r results/output.txt) then    if [ $verbose -gt 0 ]; then
57    echo "                ***** No results to compare with *****"     echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
58    if (! $?untested) set untested    fi
59    set untested=($untested $dr)    digits_of_similarity=99
60   else   else
61    set config=NO    if [ $verbose -gt 0 ]; then
62    cd input     echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
63    echo -n "  generating Makefile ..."    fi
64    ../../../tools/genmake -mods=../code >&! make.log   fi
65    if ($status == 0) set config=Yes   rm tmp1.txt tmp2.txt tmp3.txt
66    set makedepend=NO  
67    echo " done."   return $digits_of_similarity
68    echo -n "  make depend ..."  }
69    make cleanlinks depend >>& make.log  
70    if ($status == 0) set makedepend=Yes  dashnum()
71    echo " done."  {
72    echo -n "  make ..."  # dashnum n1 n2 n3 ...
73    make >>& make.log  #
74    if ($status) then  #  print numbers using %3i format or "--" if number = 99
75     echo "               ***** An error occurred during make *****"  for num in $@
76     cat make.log  do
77     echo The error during compilation occured in \"$dr\"   if [ $num = 99 ]; then
78     set compile=NO    printf ' --'
79     if ($?on_error_die) exit 1   else
80    else    printf '%3i' $num
81     set compile=Yes   fi
82     set execute=NO  done
83    endif  }
84    echo " done."  
85    echo -n "  running model ..."  testoutput()
86    if ($compile == 'Yes') then  {
87    ./mitgcmuv | & grep "D iters" > output.txt  # testoutput diretory subdir
88    if ($status == 0) then  #
89      set execute=Yes  #  test output in "directory"
90      set exactmatch=NO  
91      set accuracy=-  if [ $debug -gt 0 ]; then
92      echo " done."   echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
93      sed 's/.*D iters, err =//' output.txt \  fi
94      | grep "   0    " \  testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
95      > high.txt  if [ $debug -gt 0 ]; then
96      grep "D iters" ../results/output.txt \   echo testoutput: cg2dres=$cg2dres 1>&2
97       | sed 's/.*D iters, err =//' \  fi
98       | grep "   0    " \  
99       > oldhigh.txt  if [ $longtest -gt 0 ]; then
100      diff oldhigh.txt high.txt > /dev/null  testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
101      if ($status) then  testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
102        echo "  output differs:"  testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
103        diff oldhigh.txt high.txt  testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
104        echo ""  testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
105        @ lvl=0  testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
106        set accuracy='.'  testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
107        set fail  testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
108        echo -n Trying reduced accuracy  testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
109        while ($lvl <= $passaccuracy)  testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
110          @ lvl+=1  testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
111          echo -n " " $lvl  testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
112          sed s/{$accuracy}E/E/ high.txt > low.txt  testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
113          sed s/{$accuracy}E/E/ oldhigh.txt > oldlow.txt  testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
114          diff oldlow.txt low.txt > /dev/null  testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
115          if ($status == 0) then  testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
116            unset fail  else
117            break  testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
118    testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
119    testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
120    testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
121    testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
122    testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
123    testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
124    testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
125    fi
126    
127    dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
128            $umin $umax $umean $usd $vmin $vmax $vmean $vsd
129    #printf '%3i' $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
130    #       $umin $umax $umean $usd $vmin $vmax $vmean $vsd
131    }
132    
133    genmakemodel()
134    {
135    # genmakemodel directory
136     ( cd $1;
137       if [ $quick -eq 0 -o ! -r Makefile ]; then
138        printf 'genmake ... ' 1>&2
139        if [ $ieee -eq 0 ]; then
140         ../../../tools/genmake -mods=../code > make.log 2>&1
141        else
142         ../../../tools/genmake -ieee -mods=../code > make.log 2>&1
143        fi
144        if [ $? -ne 0 ]; then
145         tail make.log
146         echo genmakemodel: genmake failed 1>&2
147         return 1
148        else
149          echo succesful 1>&2
150        fi
151       fi
152     )
153    }
154    
155    makecleancompile()
156    {
157    # makecleancompile directory
158     ( cd $1;
159       if [ $force -gt 0 ]; then
160        rm -f output.txt
161        printf 'make clean ... ' 2>&1
162        make CLEAN >> make.log 2>&1
163        if [ $? -ne 0 ]; then
164          tail make.log
165          echo makecleancompile: make clean failed 1>&2
166          return 1
167        else
168          echo succesful 1>&2
169        fi
170       fi
171     )
172    }
173    
174    makecleanupafter()
175    {
176    # makeupafter directory
177     ( cd $1;
178       if [ $clean -gt 0 ]; then
179        rm -f output.txt
180        printf 'make clean ... ' 2>&1
181        make CLEAN >> make.log 2>&1
182        if [ $? -ne 0 ]; then
183          tail make.log
184          echo makeupafter: make clean failed 1>&2
185          return 1
186        else
187          echo succesful 1>&2
188        fi
189       fi
190     )
191    }
192    
193    makedependmodel()
194    {
195    # makedependmodel directory
196     ( cd $1;
197       if [ $quick -eq 0 -o ! -r Makefile ]; then
198        printf 'make depend ... ' 1>&2
199        make cleanlinks >> make.log 2>&1
200        make depend >> make.log 2>&1
201        if [ $? -ne 0 ]; then
202          tail make.log
203          echo makemodel: make depend failed 1>&2
204          return 1
205        else
206          echo succesful 1>&2
207        fi
208       fi
209     )
210    }
211    
212    makemodel()
213    {
214    # makemodel directory
215     ( cd $1;
216       if [ -r Makefile ]; then
217        printf 'make ... ' 1>&2
218        make >> make.log 2>&1
219        if [ $? -ne 0 ]; then
220         tail make.log
221         echo failed 1>&2
222         return 1
223        else
224         echo succesful 1>&2
225        fi
226       fi
227     )
228    }
229    
230    linkdata()
231    {
232    # linkdata flag
233    # symbolically link data files to run directory
234     if [ $1 -ne 0 ]; then
235    # if [ ! -r $2 ]; then
236    #  mkdir $2
237    # fi
238      ( cd $2; ln -sf ../input/* .)
239     fi
240    }
241    
242    runmodel()
243    {
244    # runmodel directory exe
245    #
246    #  runs the model "exe" in "directory" (exe is relative to directory)
247    
248     ( cd $1
249       if [ -x $2 ]; then
250        if [ $quick -eq 0 ]; then
251         rm -f output.txt
252        fi
253        printf 'runmodel: ' 1>&2
254         make output.txt && return 0
255         return 1
256       fi
257     )
258    }
259    
260    createcodelet()
261    {
262    # create codelet for comparing model output
263    cat > tmp_cmpnum.f <<EOFA
264          program cmpnum
265          implicit none
266          real*8 a,b,diff
267          integer linnum,best
268          best=-16
269      99  read(*,*,end=70,err=60) linnum,a,b
270          diff=0.5*(abs(a)+abs(b))
271    c     print *,a,b,diff,abs(a-b)/diff
272          if (diff.gt.1.e-12) then
273            diff=abs(a-b)/diff
274            if (diff.gt.0.) then
275    c         print *,int(log10(diff)),diff
276              linnum=int(log10(diff))
277              best=max(best,linnum)
278          endif          endif
         set accuracy=`echo $accuracy | sed 's/\./../'`  
       end  
       echo ""  
       if ($?fail ) then  
         if (! $?fails) set fails  
         set fails=($fails $dr)  
         echo "                        ***** FAIL *****"  
279        else        else
280          if (! $?passes) set passes          if (best.eq.-16.and.diff.ne.0.) best=-22
         set passes=($passes $dr)  
         echo ""  
         echo "                   ***** PASS (grade $lvl) *****"  
281        endif        endif
282        set accuracy=$lvl        goto 99
283      else    60  stop 'cmpnum: An error occured reading a,b'
284        set exactmatch=Yes    70  print *,-best
285        echo ""        end
286        echo Model passed at highest accuracy.  EOFA
287        echo ""  f77 tmp_cmpnum.f
288        echo "                        ***** PASS *****"  if [ -x ./a.out ]; then
289        if (! $?passes) set passes   return 0
290        set passes=($passes $dr)  else
291      endif   echo createcodelet: failed to compile codelet 1>&2
292     exit 1
293    fi
294    }
295    
296    formatresults()
297    {
298    # formatresults expt genmake depend make run results*
299    
300     nm=$1
301     printf '%s %s %s %s' $2 $3 $4 $5
302     shift; shift; shift; shift; shift;
303     printf '%3s' $@
304    
305     if [ $1 = '--' ]; then
306      printf ' N/O '
307     else
308      if [ $1 -gt 12 ]; then
309       printf ' pass'
310    else    else
311      echo " error"     printf ' FAIL'
312      echo "               ***** An error occured running the model *****"    fi
313      tail output.txt   fi
314      echo The runtime error occured in \"$dr\"   printf '  %s' $nm
315      if ($?on_error_die) exit 1   printf '\n'
316    endif  
317    endif  }
318    cd ..  
319   endif  show_help()
320   echo ""  {
321   cd ..  cat - << EOF
322  # Pretty summary  $0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
323   printresults:   -help      Show this help message
324   printf '  %s\t  %s\t  %s\t  %s\t  %s\t  %s\t%s\n' $config $makedepend $compile $execute $exactmatch $accuracy $dr >> summary.txt   -quick     Skip "genmake" and "make depend" if the Makefile exists
325  end   -quiet     Reduce the amount of output
326  echo ==============================================================================   -verbose   Produce copious amounts of output
327  echo ""   -debug     Produce even more output which will mean nothing to most
328     -force     Do "make CLEAN" before compiling. This forces a complete rebuild.
329     -clean     Do "make CLEAN" after compiling and testing.
330     -shorttest Don't compare numeric output for mean and s.d. of variables.
331     -noieee    By default, $0 uses the -ieee option for genmake. This turns it off.
332     -cleanup   Aggresively removes all model output, executables and object files
333                and then exits. Use with care.
334    
335    Normal usage:
336     $0 *       Configure, compile, run and analyze in all experiment directories
337    EOF
338    }
339    
340    scandirs()
341    {
342    if [ $# -eq 0 ]; then
343     for arg in *
344      do
345       test -d $arg/input && echo $arg
346     done
347    else
348     echo $*
349    fi
350    }
351    
352    clean_up()
353    {
354    # Find all executables, object files, CPP'd source and model output
355    # and DELETE it.
356    for opt in '-name "mitgcmuv*"' '-name "*.o"' '-name "*.f"' '-name "*.c"' '-name "fort.*"' '-name "make.log"' '-path *results -prune -o -name "*.meta"' '-path *results -prune -o -name "*.data"' '-type l'
357     do
358       echo Cleaning: find . $opt -exec rm {} \;
359       find . $opt -exec rm {} \;
360    done
361    }
362    
363    ###############################################################################
364    ###############################################################################
365    ###############################################################################
366    
367    # Main function
368    
369    # Default properties
370    debug=0
371    verbose=1
372    quick=0
373    force=0
374    clean=0
375    ieee=1
376    longtest=1
377    expts=''
378    
379    # Process arguments
380    for arg in $@
381    do
382     case $arg in
383       -cleanup) clean_up; exit 0;;
384       -quick) quick=1;;
385       -verbose) verbose=2;;
386       -debug) debug=1;;
387       -force) force=1;;
388       -clean) clean=1;;
389       -noieee) ieee=0;;
390       -shorttest) longtest=0;;
391       -quiet) verbose=0;;
392       -help) show_help; exit 0;;
393       -*) echo Unrecognized option:$arg; exit 9;;
394       *) test -d $arg && expts=`echo $expts $arg`;;
395     esac
396    done
397    
398    if [ $force -gt 0 -a $quick -gt 0 ]; then
399     echo You specified -quick and -force together which conflict.
400     echo Please specify either -quick or -force or neither but not both.
401     exit 1
402    fi
403    
404    #if [ ${#expts} -eq 0 ]; then
405    # echo Scanning all directories
406    # for arg in *
407    #  do
408    #   test -d $arg && expts=`echo $expts $arg`
409    # done
410    #fi
411    expts=`scandirs $expts`
412    
413    createcodelet
414    
415    date > summary.txt
416    if [ $longtest -gt 0 ]; then
417    cat << EOF >> summary.txt
418                    T           S           U           V
419    C D M    c        m  s        m  s        m  s        m  s
420    n p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
421    f n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
422    g d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
423    
424    EOF
425    else
426    cat << EOF >> summary.txt
427                  T     S     U     V
428    C D M    c                        
429    n p a R  g  m  m  m  m  m  m  m  m
430    f n k u  2  i  a  i  a  i  a  i  a
431    g d e n  d  n  x  n  x  n  x  n  x
432    
433    EOF
434    fi
435    
436    # Now configue, make, run and test in each directory
437    for dir in $expts
438    do
439     if [ -r $dir/results/output.txt ]; then
440     echo -------------------------------------------------------------------------------
441     echo
442     echo Experiment: $dir
443     echo
444     unset genmake makedepend make run
445     if [ $longtest -gt 0 ]; then
446      results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
447     else
448      results='-- -- -- -- -- -- -- -- --'
449     fi
450     if [ -r $dir/build ]; then
451      seperatebuilddir=1
452      builddir=build
453      rundir=build
454      (cd $dir/input; rm -f *.{o,f,c,F} work* output.txt Make* make.log; )
455      (cd $dir/build; ln -sf ../input/* .)
456     else
457      seperatebuilddir=0
458      builddir=input
459      rundir=input
460     fi
461     genmakemodel $dir/$builddir && genmake=Y \
462     && makecleancompile $dir/$builddir \
463     && makedependmodel $dir/$builddir && makedepend=Y \
464     && makemodel $dir/$builddir && make=Y \
465     && linkdata $seperatebuilddir $dir/$rundir \
466     && runmodel $dir/$builddir mitgcmuv && run=Y \
467     && results=`testoutput $dir $rundir` \
468     && makecleanupafter $dir/$builddir
469     echo
470     formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results
471     echo
472     formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> summary.txt
473     fi
474    done
475    
476    rm tmp_cmpnum.f a.out
477    
478  echo "Summary:"  echo -------------------------------------------------------------------------------
479  echo ""  echo
480    date >> summary.txt
481  cat summary.txt  cat summary.txt

Legend:
Removed from v.1.2  
changed lines
  Added in v.1.24

  ViewVC Help
Powered by ViewVC 1.1.22