/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (hide annotations) (download)
Wed Aug 27 22:12:31 2003 UTC (20 years, 7 months ago) by edhill
Branch: MAIN
Small cleanups to genmake2 and its "optfiles".  Initial check-in of
"testreport" which functions similarly to "testscript" but supports
more genmake2 options and emails results to a central address for
automated testing.

1 edhill 1.1 #!/bin/bash
2     #
3     # $Header$
4     #
5    
6     usage()
7     {
8     echo
9     echo "Usage: $0 [OPTIONS]"
10     echo
11     echo "where possible OPTIONS are:"
12     echo " (-help|-h) print usage"
13     echo " (-optfile=|-of=)STRING list of genmake2 \"optfiles\""
14     echo " (-a|-addr)STRING list of email recipients"
15     echo " (-t|-tdir)STRING list of \"test\" dirs"
16     echo
17     exit 1
18     }
19    
20     # build the mpack utility
21     build_mpack()
22     {
23     echo -n "building the mpack utility... "
24     if test ! -x "$MPACKDIR/mpack" ; then
25     if test ! -d $MPACKDIR ; then
26     echo "Error: can't find \"$MPACKDIR\""
27     echo " are you sure this program is being run in the correct "
28     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
29     exit 1
30     fi
31     echo -n "building mpack... "
32     ( cd $MPACKDIR && ./configure && make ) > build_mpack.out 2>&1
33     RETVAL=$?
34     if test "x$RETVAL" != x0 ; then
35     echo
36     echo "Error building the mpack tools at: $MPACK_DIR"
37     exit 1
38     fi
39     fi
40     echo "OK"
41     }
42    
43     compare_lines()
44     {
45     # use codelet to compare lines
46     if [ $verbose -gt 1 ]; then
47     cat tmp3.txt 1>&2
48     fi
49     return `./a.out < tmp3.txt`
50     }
51    
52     testoutput_for_prop()
53     {
54     # testoutput_for_prop dir s1 label subdir
55     #
56     # compares files in $dir/$subdir/output.txt and $dir/results/output.txt
57     # using search strings s1 and text label
58    
59     if [ $debug -gt 0 ]; then
60     echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
61     fi
62     if [ -r $1/$4/output.txt ]; then
63     grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt
64     lncnt=`wc -l tmp1.txt | awk '{print $1}' `
65     if [ $lncnt -lt 3 ]; then
66     if [ $verbose -gt 0 ]; then
67     echo Not enough lines of output when searching for "$2" 1>&2
68     fi
69     return 99
70     fi
71     else
72     echo testoutput_for_prop: output.txt from model run was not readable 1>&2
73     return 99
74     fi
75     if [ $debug -gt 0 ]; then
76     echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2
77     fi
78     grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt
79     lncnt=`wc -l tmp2.txt | awk '{print $1}' `
80     if [ $lncnt -lt 3 ]; then
81     if [ $verbose -gt 0 ]; then
82     echo Not enough lines of output when searching for "$2" 1>&2
83     fi
84     return 99
85     fi
86     if [ $debug -gt 0 ]; then
87     echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
88     fi
89     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
90     if [ $debug -gt 0 ]; then
91     echo testoutput_for_prop: compare_lines 1>&2
92     fi
93     compare_lines
94     digits_of_similarity=$?
95     if [ $digits_of_similarity -eq 99 ]; then
96     if [ $verbose -gt 0 ]; then
97     echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
98     fi
99     digits_of_similarity=99
100     else
101     if [ $verbose -gt 0 ]; then
102     echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
103     fi
104     fi
105     rm tmp1.txt tmp2.txt tmp3.txt
106    
107     return $digits_of_similarity
108     }
109    
110     dashnum()
111     {
112     # dashnum n1 n2 n3 ...
113     #
114     # print numbers using %3i format or "--" if number = 99
115    
116     for num in $@ ; do
117     if [ $num = 99 ]; then
118     printf ' --'
119     else
120     printf '%3i' $num
121     fi
122     done
123     }
124    
125     testoutput()
126     {
127     # testoutput diretory subdir
128     #
129     # test output in "directory"
130    
131     if [ $debug -gt 0 ]; then
132     echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
133     fi
134     testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$?
135     if [ $debug -gt 0 ]; then
136     echo testoutput: cg2dres=$cg2dres 1>&2
137     fi
138    
139     testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$?
140     testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$?
141     testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$?
142     testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$?
143     testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$?
144     testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$?
145     testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$?
146     testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$?
147     testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$?
148     testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$?
149     testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$?
150     testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$?
151     testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$?
152     testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$?
153     testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$?
154     testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$?
155    
156     dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \
157     $umin $umax $umean $usd $vmin $vmax $vmean $vsd
158     }
159    
160     genmakemodel()
161     {
162     # genmakemodel directory
163     GENMAKE2="../../../tools/genmake2"
164     (
165     cd $1;
166     printf 'genmake ... ' 1>&2
167     # ../../../tools/genmake -ieee -mods=../code > make.log 2>&1
168     $GENMAKE2 -ds --mods=../code "--optfile="$OPTFILE > make.log 2>&1
169     RETVAL=$?
170     for i in gm_state gm_optfile gm_local Makefile ; do
171     if test -r $i ; then
172     cp $i $CDIR
173     fi
174     done
175     if test "x$RETVAL" != x0 ; then
176     tail make.log
177     echo "genmakemodel: genmake failed" 1>&2
178     cp make.log $CDIR
179     return 1
180     else
181     echo "succesful" 1>&2
182     fi
183     )
184     }
185    
186     makeclean()
187     {
188     # makeclean directory
189     (
190     cd $1;
191     rm -f output.txt
192     printf 'make CLEAN ... ' 2>&1
193     if test -r Makefile ; then
194     make CLEAN >> make.log 2>&1
195     RETVAL=$?
196     if test "x$RETVAL" != x0 ; then
197     tail make.log
198     echo "makeclean: \"make CLEAN\" failed" 1>&2
199     cp make.log $CDIR"/make.log"
200     return 1
201     fi
202     fi
203     echo succesful 1>&2
204     exit 0
205     )
206     }
207    
208     makedependmodel()
209     {
210     # makedependmodel directory
211     (
212     cd $1;
213     printf 'make depend ... ' 1>&2
214     make depend >> make.log 2>&1
215     RETVAL=$?
216     if test "x$RETVAL" != x0 ; then
217     tail make.log
218     echo "makedependmodel: make depend failed" 1>&2
219     cp make.log $CDIR"/make.log"
220     return 1
221     else
222     echo succesful 1>&2
223     fi
224     )
225     }
226    
227     makemodel()
228     {
229     # makemodel directory
230     (
231     cd $1;
232     if test -r Makefile ; then
233     printf 'make ... ' 1>&2
234     make >> make.log 2>&1
235     RETVAL=$?
236     if test "x$RETVAL" != x0 ; then
237     tail make.log
238     echo failed 1>&2
239     cp make.log $CDIR"/make.log"
240     return 1
241     else
242     echo succesful 1>&2
243     fi
244     fi
245     )
246     }
247    
248     linkdata()
249     {
250     # linkdata flag
251     #
252     # symbolically link data files to run directory
253     if [ $1 -ne 0 ]; then
254     ( cd $2 ; ln -sf ../input/* . )
255     fi
256     }
257    
258     runmodel()
259     {
260     # runmodel directory exe
261     #
262     # runs the model "exe" in "directory" (exe is relative to directory)
263     (
264     cd $1
265     if [ -x $2 ]; then
266     if [ $quick -eq 0 ]; then
267     rm -f output.txt
268     fi
269     printf 'runmodel: ' 1>&2
270     make output.txt
271     RETVAL=$?
272     if test "x$RETVAL" = x0 ; then
273     cp output.txt $CDIR"/output.txt"
274     return 0
275     else
276     return 1
277     fi
278     fi
279     )
280     }
281    
282     createcodelet()
283     {
284     # create codelet for comparing model output
285    
286     echo -n "creating the comparison code... "
287     cat > tmp_cmpnum.f <<EOFA
288     program cmpnum
289     implicit none
290     real*8 a,b,diff
291     integer linnum,best
292     best=-16
293     99 read(*,*,end=70,err=60) linnum,a,b
294     diff=0.5*(abs(a)+abs(b))
295     c print *,a,b,diff,abs(a-b)/diff
296     if (diff.gt.1.e-12) then
297     diff=abs(a-b)/diff
298     if (diff.gt.0.) then
299     c print *,int(log10(diff)),diff
300     linnum=int(log10(diff))
301     best=max(best,linnum)
302     endif
303     else
304     if (best.eq.-16.and.diff.ne.0.) best=-22
305     endif
306     goto 99
307     60 stop 'cmpnum: An error occured reading a,b'
308     70 print *,-best
309     end
310     EOFA
311    
312     f77 tmp_cmpnum.f
313     if [ -x ./a.out ]; then
314     echo "OK"
315     return 0
316     else
317     echo
318     echo "createcodelet: failed to compile codelet" | tee
319     exit 1
320     fi
321     }
322    
323     formatresults()
324     {
325     # formatresults expt genmake depend make run results*
326    
327     nm=$1
328     printf '%s %s %s %s' $2 $3 $4 $5
329     shift; shift; shift; shift; shift;
330     printf '%3s' $@
331    
332     if [ $1 = '--' ]; then
333     printf ' N/O '
334     else
335     if [ $1 -gt 12 ]; then
336     printf ' pass'
337     else
338     printf ' FAIL'
339     fi
340     fi
341     printf ' %s' $nm
342     printf '\n'
343    
344     }
345    
346     show_help()
347     {
348     cat - << EOF
349     $0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
350     -help Show this help message
351     -quick Skip "genmake" and "make depend" if the Makefile exists
352     -quiet Reduce the amount of output
353     -verbose Produce copious amounts of output
354     -debug Produce even more output which will mean nothing to most
355     -force Do "make CLEAN" before compiling. This forces a complete rebuild.
356     -clean Do "make CLEAN" after compiling and testing.
357     -noieee By default, $0 uses the -ieee option for genmake. This turns it off.
358     -cleanup Aggresively removes all model output, executables and object files
359     and then exits. Use with care.
360    
361     Normal usage:
362     $0 * Configure, compile, run and analyze in all experiment directories
363     EOF
364     }
365    
366     scandirs()
367     {
368     if [ $# -eq 0 ]; then
369     for arg in * ; do
370     test -d $arg/input && echo $arg
371     done
372     else
373     echo $*
374     fi
375     }
376    
377    
378     ###############################################################################
379     ###############################################################################
380     ###############################################################################
381    
382    
383     # Default properties
384     debug=0
385     verbose=1
386     quick=0
387     clean=0
388     ieee=1
389     expts=''
390    
391     OPTFILES=
392     ADDRESSES=edhill@mitgcm.org
393     TESTDIRS=
394     MPACKDIR="../tools/mpack-1.6"
395     MPACK="$MPACKDIR/mpack"
396    
397     echo -n "parsing options... "
398    
399     ac_prev=
400     for ac_option ; do
401    
402     # If the previous option needs an argument, assign it.
403     if test -n "$ac_prev"; then
404     eval "$ac_prev=\$ac_option"
405     ac_prev=
406     continue
407     fi
408    
409     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
410    
411     case $ac_option in
412    
413     -help | --help | -h | --h)
414     usage ;;
415    
416     -optfile | --optfile | -f | --f)
417     ac_prev=OPTFILES ;;
418     -optfile=* | --optfile=*)
419     OPTFILES=$ac_optarg ;;
420    
421     -addr | --addr | -a | --a)
422     ac_prev=ADDRESSES ;;
423     -addr=* | --addr=*)
424     ADDRESSES=$ac_optarg ;;
425    
426     -tdir | --tdir | -t | --t)
427     ac_prev=TESTDIRS ;;
428     -tdir=* | --tdir=*)
429     TESTDIRS=$ac_optarg ;;
430    
431     -quick) quick=1 ;;
432     -verbose) verbose=2 ;;
433     -debug) debug=1 ;;
434     -clean) clean=1 ;;
435     -noieee) ieee=0 ;;
436     -quiet) verbose=0 ;;
437    
438     -*)
439     echo "Error: unrecognized option: "$ac_option
440     usage
441     ;;
442    
443     *)
444     echo "Error: unrecognized argument: "$ac_option
445     usage
446     ;;
447    
448     esac
449    
450     done
451    
452     if test "x$TESTDIRS" = x ; then
453     TESTDIRS=`scandirs`
454     fi
455    
456     echo "OK"
457    
458     # create the FORTRAN comparison code
459     createcodelet
460    
461     # build the mpack utility
462     build_mpack
463    
464     # Create a uniquely named directory to store results
465     MACH=`hostname`
466     DATE=`date +%Y%m%d`
467     BASE=$MACH"_"$DATE"_"
468     DNUM=0
469     DRESULTS="$BASE$DNUM"
470     while test -e $DRESULTS ; do
471     DNUM=$(( $DNUM + 1 ))
472     DRESULTS="$BASE$DNUM"
473     done
474     mkdir $DRESULTS
475     RETVAL=$?
476     if test "x$RETVAL" != x0 ; then
477     echo "Error: can't create results directory \"./$DRESULTS\""
478     exit 1
479     fi
480     SUMMARY="$DRESULTS/summary.txt"
481     date > $SUMMARY
482     cat << EOF >> $SUMMARY
483     T S U V
484     G D M c m s m s m s m s
485     E p a R g m m e . m m e . m m e . m m e .
486     N n k u 2 i a a d i a a d i a a d i a a d
487     2 d e n d n x n . n x n . n x n . n x n .
488    
489     EOF
490    
491     NDIR=0
492    
493     # For each optfile...
494     for OPTFILE in $OPTFILES ; do
495    
496     OPTFILE=`pwd`"/$OPTFILE"
497     if test ! -r $OPTFILE ; then
498     echo "Error: can't read optfile \"$OPTFILE\""
499     exit 1
500     fi
501     echo
502     echo "OPTFILE=$OPTFILE" >> $SUMMARY
503     echo >> $SUMMARY
504    
505     # ...and each test directory...
506     for dir in $TESTDIRS ; do
507    
508     # Create an output dir for each OPTFILE/tdir combination
509     CDIR=$DRESULTS"/"$NDIR
510     mkdir $CDIR
511     CDIR=`pwd`"/$CDIR"
512    
513     # ...configue, make, run, and compare the output.
514     echo "-------------------------------------------------------------------------------"
515     echo
516     echo "Experiment: $dir"
517     echo
518     unset genmake makedepend make run
519     results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
520     ( cd $dir/input; rm -f *.{o,f,c,F} *.[f,F]90 work* output.txt Make* make.log; )
521     if [ -r $dir/build ]; then
522     seperatebuilddir=1
523     builddir=build
524     rundir=build
525     ( cd $dir/build; ln -sf ../input/* . )
526     else
527     seperatebuilddir=0
528     builddir=input
529     rundir=input
530     fi
531     genmakemodel $dir/$builddir && genmake=Y \
532     && makeclean $dir/$builddir \
533     && makedependmodel $dir/$builddir && makedepend=Y \
534     && makemodel $dir/$builddir && make=Y \
535     && linkdata $seperatebuilddir $dir/$rundir \
536     && runmodel $dir/$builddir mitgcmuv && run=Y \
537     && results=`testoutput $dir $rundir`
538     echo
539     formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results
540     echo
541     formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> $SUMMARY
542    
543     ( cd $DRESULTS"/"$NDIR ; gzip * )
544    
545     tar -cf $DRESULTS"/"$NDIR".tar" $DRESULTS"/"$NDIR
546     $MPACK -s MITgcm-test -m 1000000 $DRESULTS"/"$NDIR".tar" $ADDRESSES
547    
548     NDIR=$(( $NDIR + 1 ))
549    
550     done
551     done
552    
553     # rm tmp_cmpnum.f a.out
554    
555     echo -------------------------------------------------------------------------------
556     echo
557     echo >> $SUMMARY
558     date >> $SUMMARY
559     cat $SUMMARY
560    
561     # tar -cf $DRESULTS".tar" $DRESULTS
562     #
563     # $MPACK -s MITgcm-test -m 1000000 $DRESULTS".tar" $ADDRESSES
564    

  ViewVC Help
Powered by ViewVC 1.1.22