/[MITgcm]/MITgcm/verification/testscript
ViewVC logotype

Diff of /MITgcm/verification/testscript

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.6 by adcroft, Wed Jun 6 12:24:19 2001 UTC revision 1.27 by adcroft, Wed Jan 21 20:38:22 2004 UTC
# Line 1  Line 1 
1  #!/bin/csh -f  #!/bin/sh
2    
3  # Run this script from the verification directory  compare_lines()
4  # It will automatically configure, compile, run and verify all experiments  {
5  # in the verification directory for whcih there is an results/output.txt  # use codelet to compare lines
6  # file.   if [ $verbose -gt 1 ]; then
7      cat tmp3.txt 1>&2
8  # This is the number of least-significant digits allows to be   fi
9  # in error before the test is classified as a "fail".   return `./a.out < tmp3.txt`
10    }
11  set on_error_die  
12  set passaccuracy=7  testoutput_for_prop()
13  set SKIP=( )  {
14  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' " " Make " " " " Exact "# of" " " " " > summary.txt  # testoutput_for_prop dir s1 label subdir
15  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' Config Depend Compile Execute Match Digits Status Experiment >> summary.txt  #
16    #  compares files in $dir/$subdir/output.txt and $dir/results/output.txt
17  # Pass any arguments  #  using search strings s1 and text label
18  foreach arg ($argv)  
19   switch ($arg)   if [ $debug -gt 0 ]; then
20    case -clean:    echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
21     echo Cleaning ...   fi
22     foreach dr ([a-zA-Z01-9]*)   if [ -r $1/$4/output.txt ]; then
23      if (-d $dr/input) then    grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
24        echo Entering $dr    lncnt=`wc -l tmp1.txt | awk '{print $1}' `
25        cd $dr/input    if [ $lncnt -lt 3 ]; then
26        echo ""     if [ $verbose -gt 0 ]; then
27        make CLEAN      echo Not enough lines of output when searching for "$2" 1>&2
28        \rm -f {*output,*high,*low}.txt *.log     fi
29        \rm -f ?ake*     return 99
30        cd ../..    fi
31      endif   else
32     end    echo testoutput_for_prop: output.txt from model run was not readable 1>&2
33     rm -f summary.txt    return 99
34     exit 0   fi
35     breaksw   if [ $debug -gt 0 ]; then
36    case -nodie:    echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
37     unset on_error_die   fi
38     breaksw   grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
39    case -skip=*:    lncnt=`wc -l tmp2.txt | awk '{print $1}' `
40     set SKIP = ( $SKIP `echo $arg | sed 's/-skip=//' | sed 's/,/ /g' `)    if [ $lncnt -lt 3 ]; then
41     breaksw     if [ $verbose -gt 0 ]; then
42    default:      echo Not enough lines of output when searching for "$2" 1>&2
43     echo Unknown argument given to $0     fi
44     exit 1     return 99
45   endsw    fi
46  end   if [ $debug -gt 0 ]; then
47      echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
48  # Need to make sure we don't meet non-model problems ...   fi
49  unlimit   join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
50     if [ $debug -gt 0 ]; then
51  foreach dr ([a-zA-Z01-9]*)    echo testoutput_for_prop: compare_lines 1>&2
52   if (! -d $dr/input ) continue   fi
53   foreach skip ($SKIP)   compare_lines
54    if ( `echo $dr | sed s/$skip.'*'//` == '' ) continue;continue   digits_of_similarity=$?
55   end   if [ $digits_of_similarity -eq 99 ]; then
56   set config=-    if [ $verbose -gt 0 ]; then
57   set makedepend=-     echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
58   set compile=-    fi
59   set execute=-    digits_of_similarity=99
60   set exactmatch=-   else
61   set accuracy=-    if [ $verbose -gt 0 ]; then
62   set pass=FAIL     echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
63   echo ==============================================================================    fi
64   echo Entering $dr   fi
65   cd $dr   rm tmp1.txt tmp2.txt tmp3.txt
66   echo ""  
67   if (! -r results/output.txt) then   return $digits_of_similarity
68    echo "                ***** No results to compare with *****"  }
69    if (! $?untested) set untested  
70    set untested=($untested $dr)  dashnum()
71   else  {
72    set config=NO  # dashnum n1 n2 n3 ...
73    cd input  #
74    echo -n "  generating Makefile ..."  #  print numbers using %3i format or "--" if number = 99
75    ../../../tools/genmake -mods=../code >&! make.log  for num in $@
76    if ($status == 0) set config=Yes  do
77    set makedepend=NO   if [ $num = 99 ]; then
78    echo " done."    printf ' --'
79    echo -n "  make depend ..."   else
80    make cleanlinks depend >>& make.log    printf '%3i' $num
81    if ($status == 0) set makedepend=Yes   fi
82    echo " done."  done
83    echo -n "  make ..."  }
84    make >>& make.log  
85    if ($status) then  testoutput()
86     echo "               ***** An error occurred during make *****"  {
87     cat make.log  # testoutput diretory subdir
88     echo The error during compilation occured in \"$dr\"  #
89     set compile=NO  #  test output in "directory"
90     if ($?on_error_die) exit 1  
91    else  if [ $debug -gt 0 ]; then
92     set compile=Yes   echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
93     set execute=NO  fi
94    endif  testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
95    echo " done."  if [ $debug -gt 0 ]; then
96    echo -n "  running model ..."   echo testoutput: cg2dres=$cg2dres 1>&2
97    if ($compile == 'Yes') then  fi
98  # ./mitgcmuv | & grep "D iters" > output.txt  
99    ./mitgcmuv > & output.txt  if [ $longtest -gt 0 ]; then
100    if ($status == 0) then  testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
101      set execute=Yes  testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
102      set exactmatch=NO  testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
103      set accuracy=-  testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
104      echo " done."  testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
105      grep "D iters" output.txt | sed 's/.*D iters, err =//' \  testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
106      | grep "   0    " \  testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
107      > high.txt  testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
108      grep "D iters" ../results/output.txt \  testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
109       | sed 's/.*D iters, err =//' \  testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
110       | grep "   0    " \  testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
111       > oldhigh.txt  testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
112      diff oldhigh.txt high.txt > /dev/null  testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
113      if ($status) then  testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
114        echo "  output differs:"  testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
115        diff oldhigh.txt high.txt  testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
116        echo ""  else
117        @ lvl=0  testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
118        set accuracy='.'  testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
119        set fail  testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
120        echo -n Trying reduced accuracy  testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
121        while ($lvl <= $passaccuracy)  testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
122          @ lvl+=1  testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
123          echo -n " " $lvl  testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
124          sed s/{$accuracy}E/E/ high.txt > low.txt  testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
125          sed s/{$accuracy}E/E/ oldhigh.txt > oldlow.txt  fi
126          diff oldlow.txt low.txt > /dev/null  
127          if ($status == 0) then  dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
128            unset fail          $umin $umax $umean $usd $vmin $vmax $vmean $vsd
129            break  #printf '%3i' $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
130    #       $umin $umax $umean $usd $vmin $vmax $vmean $vsd
131    }
132    
133    genmakemodel()
134    {
135    # genmakemodel directory
136     ( cd $1;
137       if [ $quick -eq 0 -o ! -r Makefile ]; then
138        printf 'genmake ... ' 1>&2
139        if [ $ieee -eq 0 ]; then
140         ../../../tools/genmake2 -mods=../code > make.log 2>&1
141        else
142         ../../../tools/genmake2 -ieee -mods=../code > make.log 2>&1
143        fi
144        if [ $? -ne 0 ]; then
145         tail make.log
146         echo genmakemodel: $genmake failed 1>&2
147         return 1
148        else
149          echo succesful 1>&2
150        fi
151       fi
152     )
153    }
154    
155    makecleancompile()
156    {
157    # makecleancompile directory
158     ( cd $1;
159       if [ $force -gt 0 ]; then
160        rm -f output.txt
161        printf 'make clean ... ' 2>&1
162        make CLEAN >> make.log 2>&1
163        if [ $? -ne 0 ]; then
164          tail make.log
165          echo makecleancompile: make clean failed 1>&2
166          return 1
167        else
168          echo succesful 1>&2
169        fi
170       fi
171     )
172    }
173    
174    makecleanupafter()
175    {
176    # makeupafter directory
177     ( cd $1;
178       if [ $clean -gt 0 ]; then
179        rm -f output.txt
180        printf 'make clean ... ' 2>&1
181        make CLEAN >> make.log 2>&1
182        if [ $? -ne 0 ]; then
183          tail make.log
184          echo makeupafter: make clean failed 1>&2
185          return 1
186        else
187          echo succesful 1>&2
188        fi
189       fi
190     )
191    }
192    
193    makedependmodel()
194    {
195    # makedependmodel directory
196     ( cd $1;
197       if [ $quick -eq 0 -o ! -r Makefile ]; then
198        printf 'make depend ... ' 1>&2
199        make cleanlinks >> make.log 2>&1
200        make depend >> make.log 2>&1
201        if [ $? -ne 0 ]; then
202          tail make.log
203          echo makemodel: make depend failed 1>&2
204          return 1
205        else
206          echo succesful 1>&2
207        fi
208       fi
209     )
210    }
211    
212    makemodel()
213    {
214    # makemodel directory
215     ( cd $1;
216       if [ -r Makefile ]; then
217        printf 'make ... ' 1>&2
218        make >> make.log 2>&1
219        if [ $? -ne 0 ]; then
220         tail make.log
221         echo failed 1>&2
222         return 1
223        else
224         echo succesful 1>&2
225        fi
226       fi
227     )
228    }
229    
230    linkdata()
231    {
232        # linkdata flag
233        #
234        # symbolically link data files to run directory
235        if test "x$1" = x1 ; then
236            (
237                cd $2
238                if test "x$ADM" = x ; then
239                    files=`( cd ../input ; ls -1 | grep -v CVS )`
240                    for i in $files ; do
241                        if test ! -d "../input/"$i ; then
242                            ln -sf "../input/"$i $i
243                        fi
244                    done
245                else
246                    files=`( cd ../input ; ls -1 *.bin | grep -v CVS )`
247                    for i in $files ; do
248                        if test ! -d "../input/"$i ; then
249                            ln -sf "../input/"$i $i
250                        fi
251                    done
252                    files=`( cd ../input_ad ; ls -1 | grep -v CVS )`
253                    for i in $files ; do
254                        if test ! -d "../input_ad/"$i ; then
255                            ln -sf "../input_ad/"$i $i
256                        fi
257                    done
258                fi
259            )
260        fi
261    }
262    
263    runmodel()
264    {
265    # runmodel directory exe
266    #
267    #  runs the model "exe" in "directory" (exe is relative to directory)
268    
269     ( cd $1
270       if [ -x $2 ]; then
271        if [ $quick -eq 0 ]; then
272         rm -f output.txt
273        fi
274       if [ $debug -gt 0 ]; then
275        echo -n runmodel: pwd= 1>&2
276        pwd 1>&2
277       fi
278        printf 'runmodel: ' 1>&2
279         make output.txt && return 0
280         return 1
281       fi
282     )
283    }
284    
285    copynewresults()
286    {
287    # copynewresults directory
288    #
289    #  runs the model "exe" in "directory" (exe is relative to directory)
290    
291     ( cd $1
292       if [ $newresults -eq 1 ]; then
293        if [ -r output.txt ]; then
294         cp -f output.txt ../results/
295         return 0
296        fi
297       fi
298     )
299    }
300    
301    createcodelet()
302    {
303    # create codelet for comparing model output
304    cat > tmp_cmpnum.f <<EOFA
305          program cmpnum
306          implicit none
307          real*8 a,b,diff
308          integer linnum,best
309          best=-16
310      99  read(*,*,end=70,err=60) linnum,a,b
311          diff=0.5*(abs(a)+abs(b))
312    c     print *,a,b,diff,abs(a-b)/diff
313          if (diff.gt.1.e-12) then
314            diff=abs(a-b)/diff
315            if (diff.gt.0.) then
316    c         print *,int(log10(diff)),diff
317              linnum=int(log10(diff))
318              best=max(best,linnum)
319          endif          endif
         set accuracy=`echo $accuracy | sed 's/\./../'`  
       end  
       echo ""  
       if ($?fail ) then  
         if (! $?fails) set fails  
         set fails=($fails $dr)  
         set pass=FAIL  
         echo "                        ***** FAIL *****"  
320        else        else
321          if (! $?passes) set passes          if (best.eq.-16.and.diff.ne.0.) best=-22
         set passes=($passes $dr)  
         set pass=Pass  
         echo ""  
         echo "                   ***** PASS (grade $lvl) *****"  
322        endif        endif
323        set accuracy=$lvl        goto 99
324      else    60  stop 'cmpnum: An error occured reading a,b'
325        set exactmatch=Yes    70  print *,-best
326        set pass=Pass        end
327        echo ""  EOFA
328        echo Model passed at highest accuracy.  f77 tmp_cmpnum.f
329        echo ""  if [ -x ./a.out ]; then
330        echo "                        ***** PASS *****"   return 0
331        if (! $?passes) set passes  else
332        set passes=($passes $dr)   echo createcodelet: failed to compile codelet 1>&2
333      endif   exit 1
334    fi
335    }
336    
337    formatresults()
338    {
339    # formatresults expt genmake depend make run results*
340    
341     nm=$1
342     printf '%s %s %s %s' $2 $3 $4 $5
343     shift; shift; shift; shift; shift;
344     printf '%3s' $@
345    
346     if [ $1 = '--' ]; then
347      printf ' N/O '
348     else
349      if [ $1 -gt 12 ]; then
350       printf ' pass'
351    else    else
352      echo " error"     printf ' FAIL'
353      echo "               ***** An error occured running the model *****"    fi
354      tail output.txt   fi
355      echo The runtime error occured in \"$dr\"   printf '  %s' $nm
356      if ($?on_error_die) exit 1   printf '\n'
357      set pass=-  
358    endif  }
359    rm -f high.txt oldhigh.txt low.txt oldlow.txt output.txt make.log  
360    endif  show_help()
361    cd ..  {
362   endif  cat - << EOF
363   echo ""  $0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
364   cd ..   -help      Show this help message
365  # Pretty summary   -quick     Skip "genmake" and "make depend" if the Makefile exists
366   printresults:   -newresults Copy the output.txt into results/
367   printf '  %s\t  %s\t  %s\t  %s\t  %s\t %s\t %s\t%s\n' $config $makedepend $compile $execute $exactmatch $accuracy $pass $dr >> summary.txt   -quiet     Reduce the amount of output
368  end   -verbose   Produce copious amounts of output
369  echo ==============================================================================   -debug     Produce even more output which will mean nothing to most
370  echo ""   -force     Do "make CLEAN" before compiling. This forces a complete rebuild.
371     -clean     Do "make CLEAN" after compiling and testing.
372     -shorttest Don't compare numeric output for mean and s.d. of variables.
373     -noieee    By default, $0 uses the -ieee option for genmake. This turns it off.
374     -cleanup   Aggresively removes all model output, executables and object files
375                and then exits. Use with care.
376    
377    Normal usage:
378     $0 *       Configure, compile, run and analyze in all experiment directories
379    EOF
380    }
381    
382    scandirs()
383    {
384    if [ $# -eq 0 ]; then
385     for arg in *
386      do
387       test -d $arg/input && echo $arg
388     done
389    else
390     echo $*
391    fi
392    }
393    
394    clean_up()
395    {
396    # Find all executables, object files, CPP'd source and model output
397    # and DELETE it.
398    for opt in '-name "mitgcmuv*"' '-name "*.o"' '-name "*.f"' '-name "*.c"' '-name "fort.*"' '-name "make.log"' '-path *results -prune -o -name "*.meta"' '-path *results -prune -o -name "*.data"' '-type l'
399     do
400       echo Cleaning: find . $opt -exec rm {} \;
401       find . $opt -exec rm {} \;
402    done
403    }
404    
405    ###############################################################################
406    ###############################################################################
407    ###############################################################################
408    
409    # Main function
410    
411    # Default properties
412    debug=0
413    verbose=1
414    quick=0
415    newresults=0
416    force=0
417    clean=0
418    ieee=1
419    longtest=1
420    expts=''
421    genmake=genmake
422    
423    # Process arguments
424    for arg in $@
425    do
426     case $arg in
427       -cleanup) clean_up; exit 0;;
428       -quick) quick=1;;
429       -newresults) newresults=1;;
430       -verbose) verbose=2;;
431       -debug) debug=1;;
432       -force) force=1;;
433       -clean) clean=1;;
434       -noieee) ieee=0;;
435       -shorttest) longtest=0;;
436       -quiet) verbose=0;;
437       -help) show_help; exit 0;;
438       -*) echo Unrecognized option:$arg; exit 9;;
439       *) test -d $arg && expts=`echo $expts $arg`;;
440     esac
441    done
442    
443    if [ $force -gt 0 -a $quick -gt 0 ]; then
444     echo You specified -quick and -force together which conflict.
445     echo Please specify either -quick or -force or neither but not both.
446     exit 1
447    fi
448    
449    #if [ ${#expts} -eq 0 ]; then
450    # echo Scanning all directories
451    # for arg in *
452    #  do
453    #   test -d $arg && expts=`echo $expts $arg`
454    # done
455    #fi
456    expts=`scandirs $expts`
457    
458    createcodelet
459    
460    date > summary.txt
461    if [ $longtest -gt 0 ]; then
462    cat << EOF >> summary.txt
463                    T           S           U           V
464    C D M    c        m  s        m  s        m  s        m  s
465    n p a R  g  m  m  e  .  m  m  e  .  m  m  e  .  m  m  e  .
466    f n k u  2  i  a  a  d  i  a  a  d  i  a  a  d  i  a  a  d
467    g d e n  d  n  x  n  .  n  x  n  .  n  x  n  .  n  x  n  .
468    
469    EOF
470    else
471    cat << EOF >> summary.txt
472                  T     S     U     V
473    C D M    c                        
474    n p a R  g  m  m  m  m  m  m  m  m
475    f n k u  2  i  a  i  a  i  a  i  a
476    g d e n  d  n  x  n  x  n  x  n  x
477    
478    EOF
479    fi
480    
481    # Now configue, make, run and test in each directory
482    for dir in $expts
483    do
484     if [ -r $dir/results/output.txt ]; then
485     echo -------------------------------------------------------------------------------
486     echo
487     echo Experiment: $dir
488     echo
489     unset genmakepass makedepend make run
490     if [ $longtest -gt 0 ]; then
491      results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
492     else
493      results='-- -- -- -- -- -- -- -- --'
494     fi
495     if [ -r $dir/build ]; then
496      seperatebuilddir=1
497      builddir=build
498      rundir=build
499      (cd $dir/input; rm -f *.{o,f,c,F,h} work* output.txt Make* make.log; )
500     else
501      seperatebuilddir=0
502      builddir=input
503      rundir=input
504     fi
505     genmakemodel $dir/$builddir && genmakepass=Y \
506     && makecleancompile $dir/$builddir \
507     && makedependmodel $dir/$builddir && makedepend=Y \
508     && makemodel $dir/$builddir && make=Y \
509     && linkdata $seperatebuilddir $dir/$rundir \
510     && runmodel $dir/$builddir mitgcmuv && run=Y \
511     && results=`testoutput $dir $rundir` \
512     && copynewresults $dir/$rundir \
513     && makecleanupafter $dir/$builddir
514     echo
515     formatresults $dir ${genmakepass:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results
516     echo
517     formatresults $dir ${genmakepass:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> summary.txt
518     fi
519    done
520    
521    rm tmp_cmpnum.f a.out
522    
523  echo "Summary:"  echo -------------------------------------------------------------------------------
524  echo ""  echo
525    date >> summary.txt
526  cat summary.txt  cat summary.txt

Legend:
Removed from v.1.6  
changed lines
  Added in v.1.27

  ViewVC Help
Powered by ViewVC 1.1.22