/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.76 - (hide annotations) (download)
Sun Feb 26 17:40:37 2006 UTC (18 years, 1 month ago) by jmc
Branch: MAIN
CVS Tags: checkpoint58b_post
Changes since 1.75: +15 -6 lines
fix adm testing (broken with previous modifs)

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.76 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.75 2006/02/24 23:15:25 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 edhill 1.7 echo " (-mpi) use MPI input files"
15 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
16 edhill 1.41 echo " (DEF=\"-ieee\")"
17 edhill 1.6 echo " (-optfile=|-of=)STRING list of optfiles to use"
18 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
19 edhill 1.6 echo " (DEF=\"edhill@mitgcm.org\")"
20 edhill 1.20 echo " (-t|-tdir) STRING list of test dirs to use"
21 edhill 1.6 echo " (DEF=\"\" which builds all)"
22 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
23     echo " Bourne-compatible \"sh\" shell"
24     echo " (DEF=\"\" for \"bash\")"
25 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
26 edhill 1.20 echo " (-command) STRING command to run"
27 edhill 1.6 echo " (DEF=\"make output.txt\")"
28 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
29 edhill 1.8 echo " (DEF=\"make\")"
30 jmc 1.70 echo " (-odir) STRING used to build output directory name"
31     echo " (DEF=\"hostname\")"
32 edhill 1.50 echo " (-ptracers|-ptr) STRING specify which ptracers to test"
33     echo " (DEF=\"1 2 3 4 5\")"
34 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
35 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
36     echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
37     echo " (-nogenmake|-ng) skip the genmake stage"
38     echo " (-noclean|-nc) skip the \"make clean\" stage"
39     echo " (-nodepend|-nd) skip the \"make depend\" stage"
40 edhill 1.40 echo " (-deldir|-dd) on success, delete the output directory"
41 edhill 1.6 echo
42 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
43 edhill 1.6 echo "such as:"
44 edhill 1.50 echo
45 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
46     echo " -addr='abc@123.com testing@home.org'"
47 edhill 1.1 echo
48 edhill 1.50 echo "provided that the expression is properly quoted within the current"
49     echo "shell (note the use of single quotes to protect white space)."
50     echo
51 edhill 1.1 exit 1
52     }
53    
54     # build the mpack utility
55     build_mpack()
56     {
57 edhill 1.34 printf "building the mpack utility... "
58 edhill 1.1 if test ! -x "$MPACKDIR/mpack" ; then
59     if test ! -d $MPACKDIR ; then
60 edhill 1.20 echo
61 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
62     echo " are you sure this program is being run in the correct "
63     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
64 edhill 1.20 echo
65     HAVE_MPACK=f
66 edhill 1.1 fi
67 edhill 1.34 printf "building mpack... "
68 edhill 1.26 if test "x$CC" = x ; then
69     export CC=cc
70     fi
71 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
72 edhill 1.1 RETVAL=$?
73     if test "x$RETVAL" != x0 ; then
74     echo
75     echo "Error building the mpack tools at: $MPACK_DIR"
76 edhill 1.20 echo
77     HAVE_MPACK=f
78     else
79 edhill 1.25 rm -f tr_build_mpack.out
80 edhill 1.20 HAVE_MPACK=t
81 edhill 1.1 fi
82 edhill 1.20 else
83     HAVE_MPACK=t
84 edhill 1.1 fi
85     echo "OK"
86     }
87    
88     testoutput_for_prop()
89     {
90 edhill 1.34 # testoutput_for_prop dir s1 label subdir extension
91 edhill 1.1 #
92     # compares files in $dir/$subdir/output.txt and $dir/results/output.txt
93     # using search strings s1 and text label
94    
95     if [ $debug -gt 0 ]; then
96     echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2
97     fi
98     if [ -r $1/$4/output.txt ]; then
99 edhill 1.15 grep "$2" $1/$4/output.txt | sed 's/.*=//' | cat -n > tmp1.txt
100 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
101     if [ $lncntA -lt 3 ]; then
102 edhill 1.1 if [ $verbose -gt 0 ]; then
103     echo Not enough lines of output when searching for "$2" 1>&2
104     fi
105     return 99
106     fi
107     else
108     echo testoutput_for_prop: output.txt from model run was not readable 1>&2
109     return 99
110     fi
111     if [ $debug -gt 0 ]; then
112 edhill 1.34 echo testoutput_for_prop: grep "$2" $1/results/output.txt$5 1>&2
113 edhill 1.1 fi
114 edhill 1.34 grep "$2" $1/results/output.txt$5 | sed 's/.*=//' | cat -n > tmp2.txt
115 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
116     if [ $lncntB -lt 3 ]; then
117 edhill 1.1 if [ $verbose -gt 0 ]; then
118     echo Not enough lines of output when searching for "$2" 1>&2
119     fi
120     return 99
121     fi
122 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
123     if [ $verbose -gt 0 ]; then
124     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
125     fi
126     return 99
127     fi
128 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
129     if [ $has_nan -gt 0 ] ; then
130     echo testoutput_for_prop: output.txt contains $has_nan NaN values 1>&2
131     return 99
132     fi
133     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
134     if [ $has_inf -gt 0 ] ; then
135     echo testoutput_for_prop: output.txt contains $has_inf Inf values 1>&2
136 jmc 1.71 return 99
137     fi
138 edhill 1.1 if [ $debug -gt 0 ]; then
139     echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2
140     fi
141     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
142     if [ $debug -gt 0 ]; then
143     echo testoutput_for_prop: compare_lines 1>&2
144     fi
145 edhill 1.22 if [ $verbose -gt 1 ]; then
146     cat tmp3.txt 1>&2
147     fi
148     echo "-1" >> tmp3.txt
149 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
150     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
151     digits_of_similarity=`./tmp_cmpnum < tmp4.txt`
152 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
153     if [ $verbose -gt 0 ]; then
154     echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2
155     fi
156     digits_of_similarity=99
157     else
158     if [ $verbose -gt 0 ]; then
159     echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
160     fi
161     fi
162 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
163 edhill 1.1
164     return $digits_of_similarity
165     }
166    
167     dashnum()
168     {
169     # dashnum n1 n2 n3 ...
170     #
171     # print numbers using %3i format or "--" if number = 99
172    
173     for num in $@ ; do
174     if [ $num = 99 ]; then
175     printf ' --'
176     else
177     printf '%3i' $num
178     fi
179     done
180     }
181    
182 edhill 1.24 testoutput_ad()
183     {
184     grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $5}' > t05.txt
185     grep $3 $1/$2/output.txt_adm | awk '{print NR " " $5}' > t15.txt
186     grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $6}' > t06.txt
187     grep $3 $1/$2/output.txt_adm | awk '{print NR " " $6}' > t16.txt
188     join t05.txt t15.txt > t5.txt
189     join t06.txt t16.txt > t6.txt
190     echo "-1" >> t5.txt
191     echo "-1" >> t6.txt
192     digits_5=`./tmp_cmpnum < t5.txt`
193     digits_6=`./tmp_cmpnum < t6.txt`
194     dashnum $digits_5 $digits_6
195     rm -f t[01][56].txt t[56].txt
196     }
197    
198 edhill 1.49 check_for_add_mon_output()
199     {
200     # Check for additional types of monitor output
201     if test "x$1" = x ; then
202     return
203     fi
204    
205 edhill 1.50 for ii in $PTRACERS_NUM ; do
206     eval "HAVE_PTR0"$ii"=f"
207     done
208    
209 edhill 1.49 ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
210     ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
211 edhill 1.50 for ii in $PTRACERS_NUM ; do
212 edhill 1.49 for jj in $ptr_add ; do
213     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
214     tst=`grep $name $1 | wc -l | awk '{print $1}'`
215     if test ! "x$tst" = x0 ; then
216     eval "HAVE_PTR0"$ii"=t"
217     fi
218     done
219     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
220     done
221     }
222    
223 edhill 1.1 testoutput()
224     {
225 edhill 1.34 # testoutput directory subdir extension
226 edhill 1.1 #
227     # test output in "directory"
228 edhill 1.24 if test "x$ADM" = x ; then
229     if [ $debug -gt 0 ]; then
230     echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2
231     fi
232 edhill 1.34 testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2 $3; cg2dres=$?
233 edhill 1.24 if [ $debug -gt 0 ]; then
234     echo testoutput: cg2dres=$cg2dres 1>&2
235     fi
236 edhill 1.34 testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2 $3; tmin=$?
237     testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2 $3; tmax=$?
238     testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2 $3; tmean=$?
239     testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2 $3; tsd=$?
240     testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2 $3; smin=$?
241     testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2 $3; smax=$?
242     testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2 $3; smean=$?
243     testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2 $3; ssd=$?
244     testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2 $3; umin=$?
245     testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2 $3; umax=$?
246     testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2 $3; umean=$?
247     testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2 $3; usd=$?
248     testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2 $3; vmin=$?
249     testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$?
250     testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$?
251     testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$?
252 edhill 1.49
253 edhill 1.50 # This is for PTRACERS
254     for ii in $PTRACERS_NUM ; do
255     eval `echo "p0"$ii"_min=99"`
256     eval `echo "p0"$ii"_max=99"`
257     eval `echo "p0"$ii"_mean=99"`
258     eval `echo "p0"$ii"_sd=99"`
259     tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
260     #echo 'tst = '$tst
261     if test "x$tst" = xt ; then
262     a="trcstat_ptracer0"
263     testoutput_for_prop $1 "$a"$ii"_min" "p0"$ii"_min" $2 $3
264     RETVAL=$? ; eval `echo "p0"$ii"_min="$RETVAL`
265     testoutput_for_prop $1 "$a"$ii"_max" "p0"$ii"_max" $2 $3
266     RETVAL=$? ; eval `echo "p0"$ii"_max="$RETVAL`
267     testoutput_for_prop $1 "$a"$ii"_mean" "p0"$ii"_mean" $2 $3
268     RETVAL=$? ; eval `echo "p0"$ii"_mean="$RETVAL`
269     testoutput_for_prop $1 "$a"$ii"_sd" "p0"$ii"_sd" $2 $3
270     RETVAL=$? ; eval `echo "p0"$ii"_sd="$RETVAL`
271     fi
272 edhill 1.49 done
273    
274     allargs="$cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd"
275     allargs="$allargs $umin $umax $umean $usd $vmin $vmax $vmean $vsd"
276     allargs="$allargs $p01_min $p01_max $p01_mean $p01_sd"
277     allargs="$allargs $p02_min $p02_max $p02_mean $p02_sd"
278     allargs="$allargs $p03_min $p03_max $p03_mean $p03_sd"
279     allargs="$allargs $p04_min $p04_max $p04_mean $p04_sd"
280     allargs="$allargs $p05_min $p05_max $p05_mean $p05_sd"
281    
282     eval "dashnum $allargs"
283    
284 edhill 1.24 else
285     testoutput_ad $1 $2 "precision_grdchk_result"
286 edhill 1.1 fi
287     }
288    
289     genmakemodel()
290     {
291     # genmakemodel directory
292 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
293     echo "genmake skipped!"
294     else
295 edhill 1.34 if test "x$BASH" = x ; then
296     GENMAKE2="../../../tools/genmake2"
297     else
298     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
299     fi
300 edhill 1.10 (
301     cd $1;
302 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
303     if test "x$ADM" = x ; then
304     command="$command --mods=../code"
305     else
306     command="$command --mods=../code_ad"
307     command="$command -adof=../../../tools/adjoint_options/adjoint_staf"
308     fi
309 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
310     command="$command --optfile=$OPTFILE"
311     fi
312     if test "x$IEEE" != x ; then
313     command="$command -ieee"
314     fi
315 edhill 1.66 if test "x$MPI" = xt ; then
316     command="$command -mpi"
317     fi
318 edhill 1.10 printf 'genmake ... ' 1>&2
319     $command > make.log 2>&1
320     RETVAL=$?
321 edhill 1.44 # Reduce the size of the testing emails!
322 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
323 edhill 1.10 if test "x$RETVAL" != x0 ; then
324     tail make.log
325     echo "genmakemodel: genmake failed" 1>&2
326 edhill 1.21 cp genmake_* make.log $CDIR
327 edhill 1.10 return 1
328     else
329 edhill 1.20 echo "successful" 1>&2
330 edhill 1.1 fi
331 edhill 1.10 )
332     fi
333 edhill 1.1 }
334    
335     makeclean()
336     {
337     # makeclean directory
338 edhill 1.10 if test "x$NOCLEAN" = xt ; then
339     echo "make CLEAN skipped!"
340     else
341     (
342     cd $1;
343     if test -e output.txt ; then
344     rm -f output.txt
345     fi
346     printf 'make CLEAN ... ' 2>&1
347     if test -r Makefile ; then
348     $MAKE CLEAN >> make.log 2>&1
349     RETVAL=$?
350     if test "x$RETVAL" != x0 ; then
351     tail make.log
352     echo "makeclean: \"make CLEAN\" failed" 1>&2
353     cp make.log $CDIR"/make.log"
354     return 1
355     fi
356     fi
357 edhill 1.20 echo successful 1>&2
358 edhill 1.10 exit 0
359     )
360     fi
361     }
362    
363 edhill 1.68 postclean()
364     {
365     # postclean directory
366     if test "x$POSTCLEAN" = xt ; then
367     (
368     cd $1;
369     if test -r Makefile ; then
370     $MAKE CLEAN >> /dev/null 2>&1
371     fi
372     exit 0
373     )
374     fi
375     }
376    
377 edhill 1.10 makedependmodel()
378     {
379     # makedependmodel directory
380     if test "x$NODEPEND" = xt ; then
381     echo "make depend skipped!"
382     else
383     (
384     cd $1;
385     printf 'make depend ... ' 1>&2
386     $MAKE depend >> make.log 2>&1
387 edhill 1.1 RETVAL=$?
388     if test "x$RETVAL" != x0 ; then
389     tail make.log
390 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
391 edhill 1.1 cp make.log $CDIR"/make.log"
392     return 1
393 edhill 1.10 else
394 edhill 1.20 echo successful 1>&2
395 edhill 1.1 fi
396 edhill 1.10 )
397     fi
398 edhill 1.1 }
399    
400     makemodel()
401     {
402     # makemodel directory
403     (
404     cd $1;
405     if test -r Makefile ; then
406     printf 'make ... ' 1>&2
407 edhill 1.24 if test "x$ADM" = x ; then
408 edhill 1.43 if test "x$JOBS" = x ; then
409     $MAKE >> make.log 2>&1
410     else
411     $MAKE -j $JOBS >> make.log 2>&1
412     fi
413 edhill 1.24 else
414     $MAKE adall >> make.log 2>&1
415     fi
416 edhill 1.1 RETVAL=$?
417     if test "x$RETVAL" != x0 ; then
418     tail make.log
419     echo failed 1>&2
420     cp make.log $CDIR"/make.log"
421     return 1
422     else
423 edhill 1.20 echo successful 1>&2
424 edhill 1.1 fi
425     fi
426     )
427     }
428    
429 edhill 1.27 symlink_mpifiles()
430     {
431     # Put special links so that MPI specific files are used
432     # This MUST be invoked between makeclean and makelinks because
433     # the Makefile will link to non-mpi files by default
434    
435     dir=$1
436     code_dir=$2
437     BUILD_DIR=$dir/$3
438     CODE_DIR=$dir/$code_dir
439 edhill 1.45
440 edhill 1.27 # These are files that should replace their counter-part when using -mpi
441     MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
442    
443     # Is this an MPI run?
444     if test "x$MPI" = xt ; then
445     # YES: We symbolically link these files to the build
446     # dir so long as there is no real file in place
447     for ii in $MPI_FILES ; do
448     i=`echo $ii | sed 's:^\./::'`
449     name=`echo $i | sed 's:_mpi::' `
450     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
451     RETVAL=$?
452     if test "x$RETVAL" != x0 ; then
453     if ! test -f $BUILD_DIR/$i ; then
454 edhill 1.45 #echo Linking $name to $i
455 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
456     fi
457     fi
458     done
459     else
460 edhill 1.45 # NO: We undo any _mpi symbolically linked files
461 edhill 1.27 for ii in $MPI_FILES ; do
462     i=`echo $ii | sed 's:^\./::'`
463     name=`echo $i | sed 's:_mpi::' `
464     if test -L $BUILD_DIR/$name ; then
465 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
466     RETVAL=$?
467     if test "x$RETVAL" = x0 ; then
468     #echo Un-linking $name from $linktarg
469 edhill 1.27 rm -f $BUILD_DIR/$name
470     fi
471     fi
472     done
473     fi
474    
475     }
476    
477 edhill 1.1 linkdata()
478     {
479     # linkdata flag
480     #
481     # symbolically link data files to run directory
482 edhill 1.12 if test "x$1" = x1 ; then
483     (
484     cd $2
485 edhill 1.24 if test "x$ADM" = x ; then
486     files=`( cd ../input ; ls -1 | grep -v CVS )`
487     for i in $files ; do
488 edhill 1.60 if test ! -d "../input/"$i -a ! -f $i ; then
489 edhill 1.24 ln -sf "../input/"$i $i
490     fi
491     done
492     else
493     files=`( cd ../input ; ls -1 *.bin | grep -v CVS )`
494     for i in $files ; do
495     if test ! -d "../input/"$i ; then
496     ln -sf "../input/"$i $i
497     fi
498     done
499     files=`( cd ../input_ad ; ls -1 | grep -v CVS )`
500     for i in $files ; do
501     if test ! -d "../input_ad/"$i ; then
502     ln -sf "../input_ad/"$i $i
503     fi
504     done
505     fi
506 edhill 1.12 )
507 edhill 1.1 fi
508     }
509    
510     runmodel()
511     {
512 edhill 1.6 # runmodel directory
513 edhill 1.1 #
514 edhill 1.24 # runs "$COMMAND in "directory"
515 edhill 1.6 # (where "$COMMAND" is relative to "directory")
516 edhill 1.1 (
517     cd $1
518 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
519 edhill 1.6 # make output.txt
520 edhill 1.38 echo
521 jmc 1.46 rm -f run.log
522 jmc 1.76 if test $OUTPUTFILE -ot $EXECUTABLE ; then
523 jmc 1.75 ( eval $COMMAND ) > run.log 2>&1
524     RETVAL=$?
525     else
526 jmc 1.76 echo " $OUTPUTFILE is up to date " > run.log 2>&1
527 jmc 1.75 RETVAL=0
528     fi
529 edhill 1.39 # echo "COMMAND='$COMMAND'"
530     # echo "pwd='"`pwd`"'"
531 edhill 1.6 if test "x$RETVAL" = x0 ; then
532 jmc 1.46 tail run.log
533 edhill 1.20 echo successful 1>&2
534 edhill 1.44 # === Reduce the size of the testing emails!
535     # if test "x$ADM" = x ; then
536     # cp output.txt $CDIR"/output.txt"
537     # else
538     # cp output.txt_adm $CDIR"/output.txt_adm"
539     # fi
540 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
541 edhill 1.6 return 0
542     else
543 edhill 1.20 tail run.log
544     echo failed 1>&2
545     cp run.log $CDIR"/run.log"
546 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
547 edhill 1.6 return 1
548 edhill 1.1 fi
549     )
550     }
551    
552     createcodelet()
553     {
554     # create codelet for comparing model output
555    
556 edhill 1.34 printf "creating the comparison code... "
557 edhill 1.22 cat > tmp_cmpnum.c <<EOF
558     #include <stdio.h>
559     #include <math.h>
560     int main( int argc, char** argv ) {
561 adcroft 1.33 int linnum,best,lncnt;
562 edhill 1.52 double a,b,abave,relerr;
563 jmc 1.57 best = -22;
564 adcroft 1.33 lncnt = 0;
565     while( 1 & (lncnt+=1) < 999 ) {
566 edhill 1.22 scanf("%d", &linnum);
567     if (linnum == -1) break;
568     scanf("%lf", &a); scanf("%lf", &b);
569 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
570     if (abave > 0.0) {
571     relerr=fabs(a-b)/abave;
572 edhill 1.64 if (relerr > 0.0) { linnum = (int)rint(log10(relerr)); }
573 jmc 1.57 else { linnum = -16 ; }
574     best = (best > linnum) ? best : linnum;
575 edhill 1.22 }
576     }
577 adcroft 1.33 if (lncnt == 999) best=-29;
578 edhill 1.22 printf("%d\n", -best);
579     return 0;
580     }
581     EOF
582 edhill 1.58 $CC -o tmp_cmpnum tmp_cmpnum.c -lm
583 edhill 1.1
584 edhill 1.22 if [ -x ./tmp_cmpnum ]; then
585 edhill 1.1 echo "OK"
586     return 0
587     else
588     echo
589 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
590     echo " a C compiler using the CC environment variable."
591 edhill 1.1 exit 1
592     fi
593     }
594    
595     formatresults()
596     {
597     # formatresults expt genmake depend make run results*
598    
599     nm=$1
600     printf '%s %s %s %s' $2 $3 $4 $5
601     shift; shift; shift; shift; shift;
602     printf '%3s' $@
603    
604     if [ $1 = '--' ]; then
605     printf ' N/O '
606     else
607     if [ $1 -gt 12 ]; then
608     printf ' pass'
609     else
610     printf ' FAIL'
611     fi
612     fi
613     printf ' %s' $nm
614     printf '\n'
615    
616     }
617    
618     show_help()
619     {
620     cat - << EOF
621     $0 [-help] [-quick] [-verbose] dir1 [dir2] [...]
622 edhill 1.10
623     -help|-h Show this help message
624 edhill 1.1 -quiet Reduce the amount of output
625     -verbose Produce copious amounts of output
626     -debug Produce even more output which will mean nothing to most
627     -force Do "make CLEAN" before compiling. This forces a complete rebuild.
628     -clean Do "make CLEAN" after compiling and testing.
629     -cleanup Aggresively removes all model output, executables and object files
630     and then exits. Use with care.
631    
632     Normal usage:
633     $0 * Configure, compile, run and analyze in all experiment directories
634     EOF
635     }
636    
637     scandirs()
638     {
639     if [ $# -eq 0 ]; then
640     for arg in * ; do
641     test -d $arg/input && echo $arg
642     done
643     else
644     echo $*
645     fi
646     }
647    
648    
649     ###############################################################################
650    
651    
652     # Default properties
653     debug=0
654     verbose=1
655     clean=0
656     expts=''
657 edhill 1.6 # ieee=1
658 edhill 1.10
659 edhill 1.40 IEEE=true
660 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
661     IEEE=$MITGCM_IEEE
662     fi
663    
664    
665     CLEANUP=f
666     QUICK=f
667     NOGENMAKE=f
668     NOCLEAN=f
669     NODEPEND=f
670 edhill 1.68 POSTCLEAN=f
671 edhill 1.1
672 edhill 1.4 BASH=
673 edhill 1.10 OPTFILE=NONE
674     ADDRESSES=
675 edhill 1.1 TESTDIRS=
676     MPACKDIR="../tools/mpack-1.6"
677 edhill 1.20 HAVE_MPACK=
678 edhill 1.1 MPACK="$MPACKDIR/mpack"
679 edhill 1.24 COMMAND=
680 edhill 1.59 if test "x$MAKE" = x ; then
681     MAKE=make
682     fi
683     if test "x$CC" = x ; then
684     CC=cc
685     fi
686 edhill 1.43 JOBS=
687 edhill 1.7 MPI=f
688 jmc 1.70 OUTDIR=
689 edhill 1.40 DELDIR=
690 edhill 1.1
691 edhill 1.24 ADM=
692    
693 edhill 1.49 # Additional monitor types
694 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
695 edhill 1.49
696 edhill 1.34 printf "parsing options... "
697 edhill 1.1
698     ac_prev=
699     for ac_option ; do
700    
701     # If the previous option needs an argument, assign it.
702     if test -n "$ac_prev"; then
703     eval "$ac_prev=\$ac_option"
704     ac_prev=
705     continue
706     fi
707    
708     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
709    
710     case $ac_option in
711    
712     -help | --help | -h | --h)
713     usage ;;
714    
715 edhill 1.2 -optfile | --optfile | -of | --of)
716 edhill 1.10 ac_prev=OPTFILE ;;
717 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
718 edhill 1.10 OPTFILE=$ac_optarg ;;
719 edhill 1.1
720     -addr | --addr | -a | --a)
721     ac_prev=ADDRESSES ;;
722     -addr=* | --addr=*)
723     ADDRESSES=$ac_optarg ;;
724    
725     -tdir | --tdir | -t | --t)
726     ac_prev=TESTDIRS ;;
727     -tdir=* | --tdir=*)
728     TESTDIRS=$ac_optarg ;;
729 edhill 1.4
730     -bash | --bash | -b | --b)
731     ac_prev=BASH ;;
732     -bash=* | --bash=*)
733     BASH=$ac_optarg ;;
734 edhill 1.5
735 edhill 1.6 -command | --command | -c | --c)
736     ac_prev=COMMAND ;;
737     -command=* | --command=*)
738     COMMAND=$ac_optarg ;;
739 edhill 1.8
740     -make | --make | -m | --m)
741     ac_prev=MAKE ;;
742     -make=* | --make=*)
743     MAKE=$ac_optarg ;;
744 edhill 1.1
745 jmc 1.70 -odir | --odir)
746     ac_prev=OUTDIR ;;
747     -odir=* | --odir=*)
748     OUTDIR=$ac_optarg ;;
749    
750 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
751     ac_prev=PTRACERS_NUM ;;
752     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
753     PTRACERS_NUM=$ac_optarg ;;
754    
755 edhill 1.43 -j) ac_prev=JOBS ;;
756     -j=*) JOBS=$ac_optarg ;;
757    
758 edhill 1.10 -clean | --clean)
759 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
760 edhill 1.10
761     -quick | --quick | -q | --q)
762     QUICK=t ;;
763     -nogenmake | --nogenmake | -ng | --ng)
764     NOGENMAKE=t ;;
765     -noclean | --noclean | -nc | --nc)
766     NOCLEAN=t ;;
767     -nodepend | --nodepend | -nd | --nd)
768     NODEPEND=t ;;
769    
770 edhill 1.68 -postclean | --postclean | -pc | --pc)
771     POSTCLEAN=t ;;
772    
773 edhill 1.18 -mpi) MPI=t ;;
774 edhill 1.10
775 edhill 1.24 -adm | -ad) ADM=t ;;
776    
777 edhill 1.10 -ieee) IEEE=true ;;
778     -noieee) IEEE= ;;
779    
780 edhill 1.1 -verbose) verbose=2 ;;
781     -debug) debug=1 ;;
782     -quiet) verbose=0 ;;
783    
784 edhill 1.40 -deldir | -dd) DELDIR=t ;;
785    
786 edhill 1.1 -*)
787     echo "Error: unrecognized option: "$ac_option
788     usage
789     ;;
790    
791     *)
792     echo "Error: unrecognized argument: "$ac_option
793     usage
794     ;;
795    
796     esac
797    
798     done
799    
800 edhill 1.10 if test "x$QUICK" = xt ; then
801     NOGENMAKE=t
802     NOCLEAN=t
803     NODEPEND=t
804     fi
805    
806 edhill 1.1 if test "x$TESTDIRS" = x ; then
807     TESTDIRS=`scandirs`
808     fi
809    
810 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
811     OPTFILE=$MITGCM_OF
812     fi
813    
814 jmc 1.76 if test "x$ADM" = xt ; then
815     EXECUTABLE="mitgcmuv_ad"
816     OUTPUTFILE="output.txt_adm"
817     else
818     EXECUTABLE="mitgcmuv"
819     OUTPUTFILE="output.txt"
820     fi
821    
822 edhill 1.24 if test "x$ADM" = xt -a "x$COMMAND" = x ; then
823 jmc 1.76 COMMAND="./$EXECUTABLE > $OUTPUTFILE 2>&1"
824 edhill 1.24 fi
825    
826     if test "x$COMMAND" = x ; then
827 jmc 1.76 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
828 edhill 1.24 fi
829    
830 jmc 1.76 #echo "OK"
831     echo "OK (COMMAND= $COMMAND )"
832 edhill 1.1
833     # create the FORTRAN comparison code
834     createcodelet
835    
836     # build the mpack utility
837 edhill 1.31 if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
838 edhill 1.32 echo "skipping mpack build"
839     else
840 edhill 1.31 build_mpack
841     fi
842 edhill 1.1
843     # Create a uniquely named directory to store results
844     MACH=`hostname`
845 edhill 1.2 UNAMEA=`uname -a`
846 edhill 1.1 DATE=`date +%Y%m%d`
847 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
848 jmc 1.70 if test "x$OUTDIR" != x ; then
849     BASE="tr_"$OUTDIR"_"$DATE"_"
850     fi
851 edhill 1.1 DNUM=0
852     DRESULTS="$BASE$DNUM"
853     while test -e $DRESULTS ; do
854     DNUM=$(( $DNUM + 1 ))
855     DRESULTS="$BASE$DNUM"
856     done
857     mkdir $DRESULTS
858     RETVAL=$?
859     if test "x$RETVAL" != x0 ; then
860 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
861 edhill 1.1 exit 1
862     fi
863     SUMMARY="$DRESULTS/summary.txt"
864 edhill 1.34 printf "Start time: " >> $SUMMARY
865 edhill 1.16 start_date=`date`
866 edhill 1.17 echo $start_date > $SUMMARY
867 edhill 1.1
868 edhill 1.11 of_path=
869 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
870     if test -r $OPTFILE ; then
871 edhill 1.11 # get the path
872     path=${OPTFILE%/*}
873     if test "x$path" = x ; then
874     of_path=`pwd`
875     else
876     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
877     fi
878     file=${OPTFILE##*/}
879     OPTFILE=$of_path/$file
880 edhill 1.21 cp $OPTFILE $DRESULTS
881     echo >> $SUMMARY
882     echo " OPTFILE=$OPTFILE" >> $SUMMARY
883 edhill 1.11 else
884 edhill 1.21 echo | tee $SUMMARY
885     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
886     exit 1
887 edhill 1.10 fi
888 edhill 1.21 else
889     echo >> $SUMMARY
890     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
891     echo " so the genmake default will be used." >> $SUMMARY
892 edhill 1.10 fi
893     echo
894     echo >> $SUMMARY
895 edhill 1.24 if test "x$ADM" = x ; then
896 edhill 1.49 line_0=" ----T----- ----S----- ----U----- ----V-----"
897     line_1="G D M c m s m s m s m s"
898     line_2="E p a R g m m e . m m e . m m e . m m e ."
899     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
900     line_4="2 d e n d n x n . n x n . n x n . n x n ."
901 edhill 1.50 for ii in $PTRACERS_NUM ; do
902 edhill 1.49 # tst=`eval 'echo $HAVE_PTR0'$ii`
903     # if test "x$tst" = xt ; then
904     line_0="$line_0 --PTR 0"$ii"--"
905     line_1="$line_1 m s"
906     line_2="$line_2 m m e ."
907     line_3="$line_3 i a a d"
908     line_4="$line_4 n x n ."
909     # fi
910     done
911     echo "$line_0" | tee -a $SUMMARY
912     echo "$line_1" | tee -a $SUMMARY
913     echo "$line_2" | tee -a $SUMMARY
914     echo "$line_3" | tee -a $SUMMARY
915     echo "$line_4" | tee -a $SUMMARY
916     echo " " | tee -a $SUMMARY
917 edhill 1.24 else
918     echo "ADJOINT=true" >> $SUMMARY
919     echo >> $SUMMARY
920     cat << EOF | tee -a $SUMMARY
921     G D M C G
922     E p a R o r
923     N n k u s a
924     2 d e n t d
925    
926     EOF
927     fi
928 edhill 1.1
929 edhill 1.10 # ...and each test directory...
930     for dir in $TESTDIRS ; do
931    
932     # Cleanup only!
933     if test "x$CLEANUP" = xt ; then
934     if test -r $dir/build/Makefile ; then
935     ( cd $dir/build ; make CLEAN )
936     fi
937     if test -r $dir/input/Makefile ; then
938     ( cd $dir/input ; make CLEAN )
939     fi
940 edhill 1.62 (
941     cd $dir
942     rm -rf tr_run.*
943     )
944 edhill 1.10 continue
945 edhill 1.1 fi
946 edhill 1.3
947 edhill 1.10 # Verify that the testdir exists and contains previous
948     # results in the correct location--or skip this directory!
949 edhill 1.24 fout=
950     if test "x$ADM" = x ; then
951     fout=$dir"/results/output.txt"
952     else
953     fout=$dir"/results_ad/output.txt_adm"
954     fi
955     if test ! -r $fout ; then
956     echo "can't read \"$fout\" -- skipping $dir"
957 edhill 1.10 continue
958     fi
959 edhill 1.53 if test "x$ADM" = x ; then
960     check_for_add_mon_output $fout
961     fi
962 edhill 1.7
963 edhill 1.49 # Check for additional types of monitor output
964    
965 edhill 1.12 builddir="input"
966     rundir="input"
967     use_seperate_build=0
968     if test -d $dir/build -a -r $dir/build ; then
969     builddir="build"
970     rundir="build"
971     use_seperate_build=1
972     linkdata $use_seperate_build $dir/$rundir
973 edhill 1.10 fi
974    
975 edhill 1.34 # Check whether there are "extra runs" for this testdir
976     extra_runs=
977     if test "x$ADM" = x -a "x$use_seperate_build" = x1 ; then
978     ex_run_dirs=`( cd $dir ; echo input.* )`
979     echo "ex_run_dirs='$ex_run_dirs'"
980     for exd in $ex_run_dirs ; do
981     name=`echo $exd | sed -e 's/input.//g'`
982     outf="$dir/results/output.txt.$name"
983     if test -f $outf -a -r $outf ; then
984     extra_runs="$extra_runs $name"
985     fi
986     done
987     fi
988    
989 edhill 1.24 if test "x$ADM" = x ; then
990 edhill 1.27 code_dir=code
991 edhill 1.24 CODE_DIR=$dir/code
992     else
993 edhill 1.27 code_dir=code_ad
994 edhill 1.24 CODE_DIR=$dir/code_ad
995     fi
996 edhill 1.10 BUILD_DIR=$dir/$builddir
997 edhill 1.28
998 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
999 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1000     continue
1001     fi
1002    
1003     echo "-------------------------------------------------------------------------------"
1004     echo
1005     echo "Experiment: $dir"
1006     echo
1007     unset genmake makedepend make run
1008 jmc 1.73 results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
1009 edhill 1.10
1010     # Create an output dir for each OPTFILE/tdir combination
1011 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1012     mkdir $rel_CDIR
1013     CDIR=`pwd`"/$rel_CDIR"
1014 edhill 1.10
1015     if test "x$CLEANUP" = xt ; then
1016     makeclean $dir/$builddir
1017     else
1018 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1019     && makeclean $dir/$builddir \
1020 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1021 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1022     && makemodel $dir/$builddir && make=Y \
1023 edhill 1.12 && linkdata $use_seperate_build $dir/$rundir \
1024     && runmodel $dir/$rundir && run=Y \
1025 edhill 1.1 && results=`testoutput $dir $rundir`
1026 edhill 1.10 fi
1027    
1028     echo
1029 edhill 1.24 if test "x$ADM" = x ; then
1030     fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1031 edhill 1.34 echo
1032     echo "$fres" >> $SUMMARY
1033     echo "fresults='$fres'" > $CDIR"/summary.txt"
1034     echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1035     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1036     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1037     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1038    
1039     for ex in $extra_runs ; do
1040 jmc 1.73 unset run
1041     results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'
1042     # Create an output dir for each OPTFILE/tdir.ex combination
1043     rel_CDIR=$DRESULTS"/"$dir"."$ex
1044     mkdir $rel_CDIR
1045     CDIR=`pwd`"/$rel_CDIR"
1046 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1047 edhill 1.62 for ldir in input.$ex input ; do
1048 edhill 1.61 (
1049     cd "$dir/$ldir" > /dev/null 2>&1
1050     ls -1 2>/dev/null \
1051     | sed -e 's|^CVS$||g' | sed -e 's|^output.txt$||g'
1052     ) > tr_exrun_links
1053     (
1054     cd "$dir/tr_run.$ex"
1055     cat ../../tr_exrun_links | while read i ; do
1056     if test ! "x$i" = x ; then
1057 edhill 1.62 test ! -r $i && ln -s "../"$ldir"/"$i $i
1058 edhill 1.61 fi
1059     done
1060     )
1061     test -e tr_exrun_links && rm -f tr_exrun_links
1062     done
1063 edhill 1.62 ldir=build
1064     (
1065 jmc 1.75 cd $dir/tr_run.$ex
1066     test ! -e mitgcmuv && ln -s "../"$ldir"/"mitgcmuv .
1067 edhill 1.62 )
1068 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1069     && results=`testoutput $dir tr_run.$ex "."$ex`
1070 edhill 1.34 fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
1071     fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1072     fres="$fres.$ex"
1073     echo
1074     echo "$fres" >> $SUMMARY
1075     echo "fresults='$fres'" > $CDIR"/summary.txt"
1076     echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1077     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1078     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1079 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1080 edhill 1.34 done
1081 edhill 1.24 else
1082     fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
1083     fres=$fres"$results $dir"
1084 edhill 1.34 echo
1085     echo "$fres" >> $SUMMARY
1086     echo "fresults='$fres'" > $CDIR"/summary.txt"
1087     echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1088     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1089     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1090     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1091 edhill 1.24 fi
1092 edhill 1.68
1093     postclean $dir/$builddir
1094 edhill 1.10
1095     echo "-------------------------------------------------------------------------------"
1096    
1097 edhill 1.1 done
1098    
1099 edhill 1.34 printf "Start time: " >> $SUMMARY
1100 edhill 1.16 echo $start_date >> $SUMMARY
1101 edhill 1.34 printf "End time: " >> $SUMMARY
1102 edhill 1.13 date >> $SUMMARY
1103 edhill 1.20
1104     # If addresses were supplied and mpack built successfully, then try
1105     # to send email using mpack.
1106     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1107     echo "No results email was sent."
1108     else
1109     if test "x$HAVE_MPACK" = xt ; then
1110     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1111     && gzip $DRESULTS".tar" \
1112 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1113 edhill 1.20 RETVAL=$?
1114     if test "x$RETVAL" != x0 ; then
1115     echo
1116     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1117     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1118     echo " summary of results from the directory \"$DRESULTS\"."
1119     echo
1120     else
1121     echo
1122     echo "An email containing results was sent to the following addresses:"
1123     echo " \"$ADDRESSES\""
1124     echo
1125     fi
1126     test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1127     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1128     fi
1129     fi
1130 edhill 1.13
1131 edhill 1.22 # rm -f tmp_cmpnum.f a.out
1132     rm -f tmp_cmpnum.c tmp_cmpnum
1133 edhill 1.1
1134 edhill 1.12 if test "x$CLEANUP" != xt ; then
1135 jmc 1.65 cat $SUMMARY | sed 's/ -- -- -- --//g'
1136 edhill 1.25 if test -e tr_out.txt ; then
1137     mv tr_out.txt tr_out.txt.old
1138 edhill 1.14 fi
1139 jmc 1.65 cat $SUMMARY | sed 's/ -- -- -- --//g' > tr_out.txt
1140 edhill 1.12 fi
1141 edhill 1.1
1142 edhill 1.40 if test "x$DELDIR" = xt ; then
1143     rm -rf $DRESULTS
1144     fi
1145    

  ViewVC Help
Powered by ViewVC 1.1.22