/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.98 - (hide annotations) (download)
Wed Sep 12 21:50:21 2007 UTC (16 years, 6 months ago) by jmc
Branch: MAIN
Changes since 1.97: +7 -9 lines
- change format of summary : indicate between "> <" the "main" variable
  which decide the "pass" or "FAIL" (now OK with new "make_summary")
- test if output exist before testing if it's older than executable
  (for hp_ux systems)

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.98 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.97 2007/09/11 16:44:52 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.83 echo " (-mth) run multi threaded (using eedata.mth)"
15     echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
19 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
20 edhill 1.6 echo " (DEF=\"edhill@mitgcm.org\")"
21 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
22     echo " (recognized groups: basic, tutorials)"
23     echo " (DEF=\"\" which test all)"
24 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
25     echo " (DEF=\"\" which test all)"
26 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
27     echo " Bourne-compatible \"sh\" shell"
28     echo " (DEF=\"\" for \"bash\")"
29 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
30 edhill 1.20 echo " (-command) STRING command to run"
31 edhill 1.6 echo " (DEF=\"make output.txt\")"
32 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
33 edhill 1.8 echo " (DEF=\"make\")"
34 jmc 1.70 echo " (-odir) STRING used to build output directory name"
35     echo " (DEF=\"hostname\")"
36 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
37 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
38 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
39     echo " (DEF=\"12\")"
40 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
41 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
42 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
43     echo " (-nogenmake|-ng) skip the genmake stage"
44     echo " (-noclean|-nc) skip the \"make clean\" stage"
45     echo " (-nodepend|-nd) skip the \"make depend\" stage"
46     echo " (-deldir|-dd) on success, delete the output directory"
47     echo " (-ts) provide timing information per timestep"
48     echo " (-papis) provide MFlop/s per timestep using PAPI"
49     echo " (-pcls) provide MFlop/s per timestep using PCL"
50 edhill 1.6 echo
51 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
52 edhill 1.6 echo "such as:"
53 edhill 1.50 echo
54 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
55     echo " -addr='abc@123.com testing@home.org'"
56 edhill 1.1 echo
57 edhill 1.50 echo "provided that the expression is properly quoted within the current"
58     echo "shell (note the use of single quotes to protect white space)."
59     echo
60 edhill 1.1 exit 1
61     }
62    
63     # build the mpack utility
64     build_mpack()
65     {
66 edhill 1.34 printf "building the mpack utility... "
67 edhill 1.1 if test ! -x "$MPACKDIR/mpack" ; then
68     if test ! -d $MPACKDIR ; then
69 edhill 1.20 echo
70 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
71     echo " are you sure this program is being run in the correct "
72     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
73 edhill 1.20 echo
74     HAVE_MPACK=f
75 edhill 1.1 fi
76 edhill 1.34 printf "building mpack... "
77 edhill 1.26 if test "x$CC" = x ; then
78     export CC=cc
79     fi
80 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
81 edhill 1.1 RETVAL=$?
82     if test "x$RETVAL" != x0 ; then
83     echo
84     echo "Error building the mpack tools at: $MPACK_DIR"
85 edhill 1.20 echo
86     HAVE_MPACK=f
87     else
88 edhill 1.25 rm -f tr_build_mpack.out
89 edhill 1.20 HAVE_MPACK=t
90 edhill 1.1 fi
91 edhill 1.20 else
92     HAVE_MPACK=t
93 edhill 1.1 fi
94     echo "OK"
95     }
96    
97 jmc 1.93 testoutput_var()
98 edhill 1.1 {
99 jmc 1.93 # testoutput_var dir s1 label subdir extension
100 edhill 1.1 #
101 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
102     # with same output from reference file $dir/results/output.$extension
103     # using search strings s1 and text label
104 edhill 1.1
105     if [ $debug -gt 0 ]; then
106 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
107 edhill 1.1 fi
108 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
109     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
110 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
111     if [ $lncntA -lt 3 ]; then
112 edhill 1.1 if [ $verbose -gt 0 ]; then
113     echo Not enough lines of output when searching for "$2" 1>&2
114     fi
115     return 99
116     fi
117     else
118 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
119 edhill 1.1 return 99
120     fi
121     if [ $debug -gt 0 ]; then
122 jmc 1.93 echo testoutput_var: grep "$2" $1/results/output.$5 1>&2
123 edhill 1.1 fi
124 jmc 1.85 grep "$2" $1/results/output.$5 | sed 's/.*=//' | cat -n > tmp2.txt
125 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
126     if [ $lncntB -lt 3 ]; then
127 edhill 1.1 if [ $verbose -gt 0 ]; then
128     echo Not enough lines of output when searching for "$2" 1>&2
129     fi
130     return 99
131     fi
132 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
133     if [ $verbose -gt 0 ]; then
134     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
135     fi
136     return 99
137     fi
138 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
139     if [ $has_nan -gt 0 ] ; then
140 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
141 edhill 1.72 return 99
142     fi
143     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
144     if [ $has_inf -gt 0 ] ; then
145 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
146 jmc 1.71 return 99
147     fi
148 edhill 1.1 if [ $debug -gt 0 ]; then
149 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
150 edhill 1.1 fi
151     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
152     if [ $debug -gt 0 ]; then
153 jmc 1.93 echo testoutput_var: compare_lines 1>&2
154 edhill 1.1 fi
155 edhill 1.22 if [ $verbose -gt 1 ]; then
156     cat tmp3.txt 1>&2
157     fi
158     echo "-1" >> tmp3.txt
159 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
160     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
161 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
162 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
163     if [ $verbose -gt 0 ]; then
164 jmc 1.93 echo testoutput_var: No comparison was available for \"$2\" 1>&2
165 edhill 1.1 fi
166     digits_of_similarity=99
167     else
168     if [ $verbose -gt 0 ]; then
169     echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
170     fi
171     fi
172 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
173 edhill 1.1
174     return $digits_of_similarity
175     }
176    
177     dashnum()
178     {
179     # dashnum n1 n2 n3 ...
180     #
181     # print numbers using %3i format or "--" if number = 99
182    
183     for num in $@ ; do
184     if [ $num = 99 ]; then
185     printf ' --'
186     else
187     printf '%3i' $num
188     fi
189     done
190     }
191    
192 edhill 1.24 testoutput_ad()
193     {
194 jmc 1.78 grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt
195 jmc 1.87 grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $5}' > t15.txt
196 jmc 1.78 grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt
197 jmc 1.87 grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $6}' > t16.txt
198 edhill 1.24 join t05.txt t15.txt > t5.txt
199     join t06.txt t16.txt > t6.txt
200     echo "-1" >> t5.txt
201     echo "-1" >> t6.txt
202 jmc 1.91 digits_5=`./tr_cmpnum < t5.txt`
203     digits_6=`./tr_cmpnum < t6.txt`
204 edhill 1.24 dashnum $digits_5 $digits_6
205     rm -f t[01][56].txt t[56].txt
206     }
207    
208 edhill 1.49 check_for_add_mon_output()
209     {
210     # Check for additional types of monitor output
211     if test "x$1" = x ; then
212     return
213     fi
214    
215 edhill 1.50 for ii in $PTRACERS_NUM ; do
216     eval "HAVE_PTR0"$ii"=f"
217     done
218    
219 edhill 1.49 ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
220     ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
221 edhill 1.50 for ii in $PTRACERS_NUM ; do
222 edhill 1.49 for jj in $ptr_add ; do
223     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
224     tst=`grep $name $1 | wc -l | awk '{print $1}'`
225     if test ! "x$tst" = x0 ; then
226     eval "HAVE_PTR0"$ii"=t"
227     fi
228     done
229     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
230     done
231     }
232    
233 jmc 1.93 testoutput_run()
234 edhill 1.1 {
235 jmc 1.93 # testoutput_run directory subdir extension
236 edhill 1.1 #
237 jmc 1.93 # test output from 1 run in "directory"
238 edhill 1.24 if test "x$ADM" = x ; then
239 jmc 1.94 # default list of output variables to be checked:
240     # 1rst : main variable used to decide if it pass or FAIL
241     # others : number of matching digits to be printed in summary.txt
242     listChk=$DEF_CHECK_LIST
243     # load experiment-specific list from file "tr_checklist" (if it exist)
244     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
245     sVar=`echo $listChk | awk '{print $1}'`
246     # remove 1rst var and expand the list: + => min max mean s.d
247 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
248     | sed 's/+//g' | sed "s/^$sVar//"`
249 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
250     for ii in 1 2 3 4 5 6 7 8 9 ; do
251     tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
252     #echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
253     if test "x$tst" != xt ; then listVar=`echo "$listVar" | sed "s/ pt$ii..//g"` ; fi
254     done
255     tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){t+=($i==$1)}; print t }'`
256     if test $tst != 1 ; then
257     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
258     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
259     echo "==> WARNING: in checked list:" $listVar 1>&2
260     #- put it back once:
261     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
262     fi
263     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
264     echo "listVar='$listVar'" > $CDIR"/summary.txt"
265     allargs=""
266     for xx in $listVar
267     do
268     case $xx in
269     'PS') if [ $debug -gt 0 ]
270     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
271     testoutput_var $1 "cg2d_init_res" "cg2d init. residual" $2 $3; yy=$?
272     if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
273     'Tmn') testoutput_var $1 "dynstat_theta_min" "theta minimum" $2 $3; yy=$? ;;
274     'Tmx') testoutput_var $1 "dynstat_theta_max" "theta maximum" $2 $3; yy=$? ;;
275     'Tav') testoutput_var $1 "dynstat_theta_mean" "theta mean" $2 $3; yy=$? ;;
276     'Tsd') testoutput_var $1 "dynstat_theta_sd" "theta s.d." $2 $3; yy=$? ;;
277     'Smn') testoutput_var $1 "dynstat_salt_min" "salt minimum" $2 $3; yy=$? ;;
278     'Smx') testoutput_var $1 "dynstat_salt_max" "salt maximum" $2 $3; yy=$? ;;
279     'Sav') testoutput_var $1 "dynstat_salt_mean" "salt mean" $2 $3; yy=$? ;;
280     'Ssd') testoutput_var $1 "dynstat_salt_sd" "salt s.d." $2 $3; yy=$? ;;
281     'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
282     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
283     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
284     'Usd') testoutput_var $1 "dynstat_uvel_sd" "U s.d." $2 $3; yy=$? ;;
285     'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
286     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
287     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
288     'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V s.d." $2 $3; yy=$? ;;
289     'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
290     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
291     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
292     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
293     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
294     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
295     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
296     testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_sd" $2 $3; yy=$? ;;
297     *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
298     esac
299     if test $xx = $sVar
300     then allargs="$allargs > $yy <"
301     else allargs="$allargs $yy"
302     fi
303     done
304    
305     nbVar=`echo $listVar | awk '{print NF}'`
306 jmc 1.97 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
307 jmc 1.94 #-- fill line (up to standard length) with dot:
308 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
309 jmc 1.94 echo $allargs $adNul
310     else
311     echo $allargs
312     fi
313 edhill 1.49
314 edhill 1.24 else
315     testoutput_ad $1 $2 "precision_grdchk_result"
316 edhill 1.1 fi
317     }
318    
319     genmakemodel()
320     {
321     # genmakemodel directory
322 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
323     echo "genmake skipped!"
324     else
325 edhill 1.34 if test "x$BASH" = x ; then
326     GENMAKE2="../../../tools/genmake2"
327     else
328     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
329     fi
330 edhill 1.10 (
331     cd $1;
332 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
333     if test "x$ADM" = x ; then
334     command="$command --mods=../code"
335     else
336     command="$command --mods=../code_ad"
337     fi
338 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
339     command="$command --optfile=$OPTFILE"
340     fi
341     if test "x$IEEE" != x ; then
342     command="$command -ieee"
343     fi
344 edhill 1.66 if test "x$MPI" = xt ; then
345     command="$command -mpi"
346     fi
347 ce107 1.90 if test "x$TS" = xt ; then
348     command="$command -ts"
349     fi
350     if test "x$PAPIS" = xt ; then
351     command="$command -papis"
352     else
353     if test "x$PCLS" = xt ; then
354     command="$command -pcls"
355     fi
356     fi
357 edhill 1.10 printf 'genmake ... ' 1>&2
358     $command > make.log 2>&1
359     RETVAL=$?
360 edhill 1.44 # Reduce the size of the testing emails!
361 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
362 edhill 1.10 if test "x$RETVAL" != x0 ; then
363     tail make.log
364     echo "genmakemodel: genmake failed" 1>&2
365 edhill 1.21 cp genmake_* make.log $CDIR
366 edhill 1.10 return 1
367     else
368 edhill 1.20 echo "successful" 1>&2
369 edhill 1.1 fi
370 edhill 1.10 )
371     fi
372 edhill 1.1 }
373    
374     makeclean()
375     {
376     # makeclean directory
377 edhill 1.10 if test "x$NOCLEAN" = xt ; then
378 jmc 1.83 echo "make Clean skipped!"
379 edhill 1.10 else
380     (
381     cd $1;
382 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
383 edhill 1.10 if test -r Makefile ; then
384 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
385     $MAKE Clean >> make.log 2>&1
386 edhill 1.10 RETVAL=$?
387     if test "x$RETVAL" != x0 ; then
388     tail make.log
389 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
390 edhill 1.10 cp make.log $CDIR"/make.log"
391     return 1
392     fi
393     fi
394 edhill 1.20 echo successful 1>&2
395 edhill 1.10 exit 0
396     )
397     fi
398     }
399    
400 jmc 1.83 run_clean()
401 edhill 1.68 {
402 jmc 1.83 # run_clean directory
403     if test "x$NOCLEAN" = xt ; then
404     echo "run_clean skipped!"
405     else
406 edhill 1.68 (
407     cd $1;
408 jmc 1.83 printf 'clean run-dir ... ' 2>&1
409     # part of what is done after "make clean" when doing "make CLEAN"
410     find . -name "*.meta" -exec rm {} \;
411     find . -name "*.data" -exec rm {} \;
412     find . -name "fort.*" -exec rm {} \;
413     find . -type l -exec rm {} \;
414     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
415     rm -rf mnc_test_*
416     echo successful 1>&2
417 edhill 1.68 exit 0
418     )
419     fi
420     }
421    
422 edhill 1.10 makedependmodel()
423     {
424     # makedependmodel directory
425     if test "x$NODEPEND" = xt ; then
426     echo "make depend skipped!"
427     else
428     (
429     cd $1;
430     printf 'make depend ... ' 1>&2
431     $MAKE depend >> make.log 2>&1
432 edhill 1.1 RETVAL=$?
433     if test "x$RETVAL" != x0 ; then
434     tail make.log
435 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
436 edhill 1.1 cp make.log $CDIR"/make.log"
437     return 1
438 edhill 1.10 else
439 edhill 1.20 echo successful 1>&2
440 edhill 1.1 fi
441 edhill 1.10 )
442     fi
443 edhill 1.1 }
444    
445     makemodel()
446     {
447     # makemodel directory
448     (
449     cd $1;
450     if test -r Makefile ; then
451     printf 'make ... ' 1>&2
452 edhill 1.24 if test "x$ADM" = x ; then
453 edhill 1.43 if test "x$JOBS" = x ; then
454     $MAKE >> make.log 2>&1
455     else
456     $MAKE -j $JOBS >> make.log 2>&1
457     fi
458 edhill 1.24 else
459     $MAKE adall >> make.log 2>&1
460     fi
461 edhill 1.1 RETVAL=$?
462     if test "x$RETVAL" != x0 ; then
463     tail make.log
464     echo failed 1>&2
465     cp make.log $CDIR"/make.log"
466     return 1
467     else
468 edhill 1.20 echo successful 1>&2
469 edhill 1.1 fi
470     fi
471     )
472     }
473    
474 edhill 1.27 symlink_mpifiles()
475     {
476     # Put special links so that MPI specific files are used
477     # This MUST be invoked between makeclean and makelinks because
478     # the Makefile will link to non-mpi files by default
479    
480     dir=$1
481     code_dir=$2
482     BUILD_DIR=$dir/$3
483     CODE_DIR=$dir/$code_dir
484 edhill 1.45
485 edhill 1.27 # These are files that should replace their counter-part when using -mpi
486     MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
487    
488     # Is this an MPI run?
489     if test "x$MPI" = xt ; then
490     # YES: We symbolically link these files to the build
491     # dir so long as there is no real file in place
492     for ii in $MPI_FILES ; do
493     i=`echo $ii | sed 's:^\./::'`
494     name=`echo $i | sed 's:_mpi::' `
495     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
496     RETVAL=$?
497     if test "x$RETVAL" != x0 ; then
498     if ! test -f $BUILD_DIR/$i ; then
499 edhill 1.45 #echo Linking $name to $i
500 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
501     fi
502     fi
503     done
504     else
505 edhill 1.45 # NO: We undo any _mpi symbolically linked files
506 edhill 1.27 for ii in $MPI_FILES ; do
507     i=`echo $ii | sed 's:^\./::'`
508     name=`echo $i | sed 's:_mpi::' `
509     if test -L $BUILD_DIR/$name ; then
510 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
511     RETVAL=$?
512     if test "x$RETVAL" = x0 ; then
513     #echo Un-linking $name from $linktarg
514 edhill 1.27 rm -f $BUILD_DIR/$name
515     fi
516     fi
517     done
518     fi
519    
520     }
521    
522 edhill 1.1 linkdata()
523     {
524 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
525 edhill 1.1 #
526     # symbolically link data files to run directory
527 jmc 1.83 if test -d $1 ; then
528 edhill 1.12 (
529 jmc 1.83 cd $1 ; shift
530     if test -r "../"$1"/eedata.mth" ; then
531     # found eedata.mth in 1rst input dir and it is readable
532     if test "x$MULTI_THREAD" = "xt" ; then
533     # multi-threaded test: remove symbolic link & link eedata.mth
534     if test -h eedata ; then rm -f eedata ; fi
535     if test ! -r eedata ; then
536     ln -sf "../"$1"/eedata.mth" eedata ;
537     printf 'eedata.mth ' 1>&2
538 edhill 1.24 fi
539 jmc 1.83 else
540     # not multi-threaded test: remove eedata symbolic link
541     if test -h eedata ; then rm -f eedata ; fi
542     fi
543     fi
544     for ldir in $* ; do
545     if test -d "../"$ldir ; then
546     printf 'ldir='${ldir} 1>&2
547     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
548     for i in $files ; do
549     if test ! -d "../"$ldir/$i ; then
550     if test ! -r $i ; then
551     printf ' '$i 1>&2
552     ln -sf "../"$ldir"/"$i $i
553     fi
554     fi
555     done
556     if test -x "../"$ldir"/"prepare_run ; then
557     "../"$ldir"/"prepare_run
558 edhill 1.24 fi
559 jmc 1.83 printf ' ; ' 1>&2
560 jmc 1.82 fi
561 jmc 1.83 done
562 edhill 1.12 )
563 edhill 1.1 fi
564     }
565    
566     runmodel()
567     {
568 edhill 1.6 # runmodel directory
569 edhill 1.1 #
570 edhill 1.24 # runs "$COMMAND in "directory"
571 edhill 1.6 # (where "$COMMAND" is relative to "directory")
572 edhill 1.1 (
573     cd $1
574 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
575 edhill 1.6 # make output.txt
576 edhill 1.38 echo
577 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
578 jmc 1.88 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_00
579 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
580     fi
581     if test ! -x $EXECUTABLE ; then
582 jmc 1.88 rm -f run.log ; touch run.log
583     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
584 jmc 1.83 echo " no executable:" $EXECUTABLE >> run.log
585     RETVAL=8
586 jmc 1.88 ENDVAL=-1
587 jmc 1.75 else
588 jmc 1.98 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
589     # output do not exist or is older than executable:
590 jmc 1.88 rm -f run.log ; touch run.log
591     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
592 jmc 1.83 ( eval $COMMAND ) >> run.log 2>&1
593     RETVAL=$?
594     else
595     RETVAL=0
596 jmc 1.88 if test -f run.log ; then
597     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
598     echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1
599     else
600     touch run.log
601     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
602     echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1
603     echo " no previous run.log: assume NORMAL END" >> run.log 2>&1
604     fi
605 jmc 1.83 fi
606 jmc 1.88 ENDVAL=`cat run.log | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
607 jmc 1.75 fi
608 jmc 1.88 rm -f run.log_00
609     #if test "x$RETVAL" = x0 ; then
610     if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
611 jmc 1.46 tail run.log
612 edhill 1.20 echo successful 1>&2
613 edhill 1.44 # === Reduce the size of the testing emails!
614 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
615 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
616 edhill 1.6 return 0
617     else
618 edhill 1.20 tail run.log
619 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
620 edhill 1.20 cp run.log $CDIR"/run.log"
621 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
622 edhill 1.6 return 1
623 edhill 1.1 fi
624     )
625     }
626    
627     createcodelet()
628     {
629     # create codelet for comparing model output
630    
631 edhill 1.34 printf "creating the comparison code... "
632 jmc 1.91 cat > tr_cmpnum.c <<EOF
633 edhill 1.22 #include <stdio.h>
634     #include <math.h>
635     int main( int argc, char** argv ) {
636 jmc 1.91 int linnum,cmplin,best,lncnt;
637 edhill 1.52 double a,b,abave,relerr;
638 jmc 1.57 best = -22;
639 adcroft 1.33 lncnt = 0;
640 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
641 edhill 1.22 scanf("%d", &linnum);
642     if (linnum == -1) break;
643     scanf("%lf", &a); scanf("%lf", &b);
644 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
645 jmc 1.91 if ( abave == abave ) {
646     if (abave > 0.0) {
647     relerr=fabs(a-b)/abave;
648     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
649     else { cmplin = -16 ; }
650     best = (best > cmplin) ? best : cmplin; }
651     else { cmplin = -22 ; }
652     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
653     }
654     else {
655     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
656     break; }
657 edhill 1.22 }
658 adcroft 1.33 if (lncnt == 999) best=-29;
659 jmc 1.91 if (linnum != -1) best=-99;
660 edhill 1.22 printf("%d\n", -best);
661     return 0;
662     }
663     EOF
664 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
665 edhill 1.1
666 jmc 1.91 if [ -x ./tr_cmpnum ]; then
667 edhill 1.1 echo "OK"
668     return 0
669     else
670     echo
671 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
672     echo " a C compiler using the CC environment variable."
673 edhill 1.1 exit 1
674     fi
675     }
676    
677     formatresults()
678     {
679     # formatresults expt genmake depend make run results*
680    
681     nm=$1
682     printf '%s %s %s %s' $2 $3 $4 $5
683     shift; shift; shift; shift; shift;
684 jmc 1.94 listPrt=$@
685     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
686     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
687 jmc 1.97 printf '%3s' $listPrt
688 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
689 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
690 edhill 1.1
691 jmc 1.94 if [ $xx = '..' ]; then
692     printf ' N/O '
693     elif [ $xx = '--' ]; then
694     printf ' N/O '
695     elif [ $xx = 99 ]; then
696 edhill 1.1 printf ' N/O '
697     else
698 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
699 edhill 1.1 printf ' pass'
700     else
701     printf ' FAIL'
702     fi
703     fi
704     printf ' %s' $nm
705     printf '\n'
706    
707     }
708    
709     scandirs()
710     {
711 jmc 1.81 if [ $# -eq 1 ]; then
712     for arg in * ; do
713     test -d $arg/$1 && echo $arg
714     done
715 edhill 1.1 else
716 jmc 1.81 echo $*
717 edhill 1.1 fi
718     }
719    
720    
721     ###############################################################################
722    
723    
724     # Default properties
725     debug=0
726     verbose=1
727     clean=0
728     expts=''
729 edhill 1.6 # ieee=1
730 edhill 1.10
731 edhill 1.40 IEEE=true
732 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
733     IEEE=$MITGCM_IEEE
734     fi
735    
736    
737     CLEANUP=f
738     QUICK=f
739     NOGENMAKE=f
740     NOCLEAN=f
741     NODEPEND=f
742 edhill 1.68 POSTCLEAN=f
743 edhill 1.1
744 edhill 1.4 BASH=
745 edhill 1.10 OPTFILE=NONE
746     ADDRESSES=
747 edhill 1.1 TESTDIRS=
748 jmc 1.89 SKIPDIRS=
749 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
750 edhill 1.20 HAVE_MPACK=
751 edhill 1.1 MPACK="$MPACKDIR/mpack"
752 edhill 1.24 COMMAND=
753 edhill 1.59 if test "x$MAKE" = x ; then
754     MAKE=make
755     fi
756     if test "x$CC" = x ; then
757     CC=cc
758     fi
759 edhill 1.43 JOBS=
760 edhill 1.7 MPI=f
761 jmc 1.83 MULTI_THREAD=f
762 jmc 1.70 OUTDIR=
763 edhill 1.40 DELDIR=
764 edhill 1.1
765 edhill 1.24 ADM=
766    
767 edhill 1.49 # Additional monitor types
768 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
769 edhill 1.49
770 jmc 1.92 MATCH_CRIT=13
771    
772 edhill 1.34 printf "parsing options... "
773 edhill 1.1
774     ac_prev=
775     for ac_option ; do
776    
777     # If the previous option needs an argument, assign it.
778     if test -n "$ac_prev"; then
779     eval "$ac_prev=\$ac_option"
780     ac_prev=
781     continue
782     fi
783    
784     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
785    
786     case $ac_option in
787    
788     -help | --help | -h | --h)
789     usage ;;
790    
791 edhill 1.2 -optfile | --optfile | -of | --of)
792 edhill 1.10 ac_prev=OPTFILE ;;
793 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
794 edhill 1.10 OPTFILE=$ac_optarg ;;
795 edhill 1.1
796     -addr | --addr | -a | --a)
797     ac_prev=ADDRESSES ;;
798     -addr=* | --addr=*)
799     ADDRESSES=$ac_optarg ;;
800    
801     -tdir | --tdir | -t | --t)
802     ac_prev=TESTDIRS ;;
803     -tdir=* | --tdir=*)
804     TESTDIRS=$ac_optarg ;;
805 edhill 1.4
806 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
807     ac_prev=SKIPDIRS ;;
808     -skipdir=* | --skipdir=*)
809     SKIPDIRS=$ac_optarg ;;
810    
811 edhill 1.4 -bash | --bash | -b | --b)
812     ac_prev=BASH ;;
813     -bash=* | --bash=*)
814     BASH=$ac_optarg ;;
815 edhill 1.5
816 edhill 1.6 -command | --command | -c | --c)
817     ac_prev=COMMAND ;;
818     -command=* | --command=*)
819     COMMAND=$ac_optarg ;;
820 edhill 1.8
821     -make | --make | -m | --m)
822     ac_prev=MAKE ;;
823     -make=* | --make=*)
824     MAKE=$ac_optarg ;;
825 edhill 1.1
826 jmc 1.70 -odir | --odir)
827     ac_prev=OUTDIR ;;
828     -odir=* | --odir=*)
829     OUTDIR=$ac_optarg ;;
830    
831 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
832     ac_prev=PTRACERS_NUM ;;
833     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
834     PTRACERS_NUM=$ac_optarg ;;
835    
836 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
837     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
838    
839 edhill 1.43 -j) ac_prev=JOBS ;;
840     -j=*) JOBS=$ac_optarg ;;
841    
842 edhill 1.10 -clean | --clean)
843 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
844 edhill 1.10
845     -quick | --quick | -q | --q)
846     QUICK=t ;;
847     -nogenmake | --nogenmake | -ng | --ng)
848     NOGENMAKE=t ;;
849     -noclean | --noclean | -nc | --nc)
850     NOCLEAN=t ;;
851     -nodepend | --nodepend | -nd | --nd)
852     NODEPEND=t ;;
853    
854 edhill 1.68 -postclean | --postclean | -pc | --pc)
855     POSTCLEAN=t ;;
856    
857 edhill 1.18 -mpi) MPI=t ;;
858 edhill 1.10
859 jmc 1.83 -mth) MULTI_THREAD=t ;;
860    
861 edhill 1.24 -adm | -ad) ADM=t ;;
862    
863 edhill 1.10 -ieee) IEEE=true ;;
864     -noieee) IEEE= ;;
865    
866 edhill 1.1 -verbose) verbose=2 ;;
867     -debug) debug=1 ;;
868     -quiet) verbose=0 ;;
869    
870 edhill 1.40 -deldir | -dd) DELDIR=t ;;
871    
872 ce107 1.90 -ts) TS=t;;
873    
874     -papis) PAPIS=t;;
875    
876     -pcls) PCL=t;;
877    
878 edhill 1.1 -*)
879     echo "Error: unrecognized option: "$ac_option
880     usage
881     ;;
882    
883     *)
884     echo "Error: unrecognized argument: "$ac_option
885     usage
886     ;;
887    
888     esac
889    
890     done
891    
892 edhill 1.10 if test "x$QUICK" = xt ; then
893     NOGENMAKE=t
894     NOCLEAN=t
895     NODEPEND=t
896     fi
897    
898 edhill 1.1 if test "x$TESTDIRS" = x ; then
899 jmc 1.81 if test "x$ADM" = xt ; then
900 jmc 1.89 LIST=`scandirs results_ad`
901 jmc 1.81 else
902 jmc 1.89 LIST=`scandirs results`
903 jmc 1.81 fi
904 jmc 1.84 else
905     #- expand group of experiments:
906     LIST=" "
907     for xx in $TESTDIRS
908     do
909     case $xx in
910 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
911     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
912     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
913 jmc 1.84 ;;
914     'tutorials')
915     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
916     *) LIST=${LIST}" "$xx ;;
917     esac
918     done
919 jmc 1.89 fi
920     #echo 'LIST='${LIST}'<'
921     #- skip dirs, remove duplicate and non-directory:
922     TESTDIRS=" "
923     count=0
924     for xx in $LIST
925     do
926     yy=`echo $SKIPDIRS | grep -c $xx`
927     if test $yy = 0 ; then
928 jmc 1.84 if test -d $xx ; then
929     yy=`echo $TESTDIRS | grep -c $xx`
930     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
931 jmc 1.89 else count=1 ;
932     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
933     fi
934     else
935     if test $count = 1 ; then echo -n ", \"$xx\""
936     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
937 jmc 1.84 fi
938 jmc 1.89 fi
939     done
940     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
941 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
942 edhill 1.1
943 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
944     OPTFILE=$MITGCM_OF
945     fi
946    
947 jmc 1.76 if test "x$ADM" = xt ; then
948     EXECUTABLE="mitgcmuv_ad"
949 jmc 1.78 OUTPUTFILE="output_adm.txt"
950 jmc 1.76 else
951     EXECUTABLE="mitgcmuv"
952     OUTPUTFILE="output.txt"
953     fi
954    
955 jmc 1.87 if test "x$COMMAND" = x ; then
956 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
957 edhill 1.24 fi
958 jmc 1.87 if test "x$MPI" = xt ; then
959     OUTPUTFILE="STDOUT.0000"
960 edhill 1.24 fi
961    
962 jmc 1.76 #echo "OK"
963     echo "OK (COMMAND= $COMMAND )"
964 edhill 1.1
965 jmc 1.94 # set the Default List of output variables to be checked:
966     # (use default or load experiment-specific list from file "tr_checklist")
967     # content : 1rst = main variable used to decide if it pass or FAIL
968     # others = number of matching digits to be printed in summary.txt
969     if test "x$ADM" != xt ; then
970     DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
971     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
972 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
973 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
974 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
975 jmc 1.94 fi
976    
977 edhill 1.1 # create the FORTRAN comparison code
978     createcodelet
979    
980     # build the mpack utility
981 edhill 1.31 if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
982 edhill 1.32 echo "skipping mpack build"
983     else
984 edhill 1.31 build_mpack
985     fi
986 edhill 1.1
987     # Create a uniquely named directory to store results
988     MACH=`hostname`
989 edhill 1.2 UNAMEA=`uname -a`
990 edhill 1.1 DATE=`date +%Y%m%d`
991 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
992 jmc 1.70 if test "x$OUTDIR" != x ; then
993     BASE="tr_"$OUTDIR"_"$DATE"_"
994     fi
995 edhill 1.1 DNUM=0
996     DRESULTS="$BASE$DNUM"
997     while test -e $DRESULTS ; do
998     DNUM=$(( $DNUM + 1 ))
999     DRESULTS="$BASE$DNUM"
1000     done
1001     mkdir $DRESULTS
1002     RETVAL=$?
1003     if test "x$RETVAL" != x0 ; then
1004 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1005 edhill 1.1 exit 1
1006     fi
1007     SUMMARY="$DRESULTS/summary.txt"
1008 edhill 1.34 printf "Start time: " >> $SUMMARY
1009 edhill 1.16 start_date=`date`
1010 edhill 1.17 echo $start_date > $SUMMARY
1011 edhill 1.1
1012 edhill 1.11 of_path=
1013 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1014     if test -r $OPTFILE ; then
1015 edhill 1.11 # get the path
1016     path=${OPTFILE%/*}
1017     if test "x$path" = x ; then
1018     of_path=`pwd`
1019     else
1020     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1021     fi
1022     file=${OPTFILE##*/}
1023     OPTFILE=$of_path/$file
1024 edhill 1.21 cp $OPTFILE $DRESULTS
1025     echo >> $SUMMARY
1026     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1027 edhill 1.11 else
1028 edhill 1.21 echo | tee $SUMMARY
1029     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1030     exit 1
1031 edhill 1.10 fi
1032 edhill 1.21 else
1033     echo >> $SUMMARY
1034     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1035     echo " so the genmake default will be used." >> $SUMMARY
1036 edhill 1.10 fi
1037     echo
1038     echo >> $SUMMARY
1039 edhill 1.24 if test "x$ADM" = x ; then
1040 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1041 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1042     else line_0="default "$MATCH_CRIT ; fi
1043     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1044 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1045 edhill 1.49 line_1="G D M c m s m s m s m s"
1046     line_2="E p a R g m m e . m m e . m m e . m m e ."
1047     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1048     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1049 edhill 1.50 for ii in $PTRACERS_NUM ; do
1050 edhill 1.49 # tst=`eval 'echo $HAVE_PTR0'$ii`
1051     # if test "x$tst" = xt ; then
1052     line_0="$line_0 --PTR 0"$ii"--"
1053     line_1="$line_1 m s"
1054     line_2="$line_2 m m e ."
1055     line_3="$line_3 i a a d"
1056     line_4="$line_4 n x n ."
1057     # fi
1058     done
1059     echo "$line_0" | tee -a $SUMMARY
1060     echo "$line_1" | tee -a $SUMMARY
1061     echo "$line_2" | tee -a $SUMMARY
1062     echo "$line_3" | tee -a $SUMMARY
1063     echo "$line_4" | tee -a $SUMMARY
1064     echo " " | tee -a $SUMMARY
1065 edhill 1.24 else
1066     echo "ADJOINT=true" >> $SUMMARY
1067     echo >> $SUMMARY
1068     cat << EOF | tee -a $SUMMARY
1069     G D M C G
1070     E p a R o r
1071     N n k u s a
1072     2 d e n t d
1073    
1074     EOF
1075     fi
1076 edhill 1.1
1077 edhill 1.10 # ...and each test directory...
1078     for dir in $TESTDIRS ; do
1079    
1080     # Cleanup only!
1081     if test "x$CLEANUP" = xt ; then
1082     if test -r $dir/build/Makefile ; then
1083 jmc 1.83 echo ' ------ clean dir:' $dir/build
1084 edhill 1.10 ( cd $dir/build ; make CLEAN )
1085     fi
1086 jmc 1.83 if test -d $dir/run/CVS ; then
1087     echo ' ------ clean dir:' $dir/run
1088     run_clean $dir/run
1089 edhill 1.10 fi
1090 edhill 1.62 (
1091     cd $dir
1092     rm -rf tr_run.*
1093     )
1094 edhill 1.10 continue
1095 edhill 1.1 fi
1096 edhill 1.3
1097 edhill 1.10 # Verify that the testdir exists and contains previous
1098     # results in the correct location--or skip this directory!
1099 edhill 1.24 fout=
1100     if test "x$ADM" = x ; then
1101     fout=$dir"/results/output.txt"
1102     else
1103 jmc 1.78 fout=$dir"/results_ad/output_adm.txt"
1104 edhill 1.24 fi
1105     if test ! -r $fout ; then
1106     echo "can't read \"$fout\" -- skipping $dir"
1107 edhill 1.10 continue
1108     fi
1109 edhill 1.53 if test "x$ADM" = x ; then
1110     check_for_add_mon_output $fout
1111     fi
1112 edhill 1.7
1113 edhill 1.49 # Check for additional types of monitor output
1114    
1115 jmc 1.83 builddir="build"
1116     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1117     rundir="run"
1118     #rundir=$builddir
1119     if test ! -d $dir/$rundir ; then
1120     rundir=$builddir
1121 edhill 1.10 fi
1122    
1123 edhill 1.24 if test "x$ADM" = x ; then
1124 edhill 1.27 code_dir=code
1125 edhill 1.24 CODE_DIR=$dir/code
1126 jmc 1.83 input_dirs='input'
1127 edhill 1.24 else
1128 edhill 1.27 code_dir=code_ad
1129 edhill 1.24 CODE_DIR=$dir/code_ad
1130 jmc 1.83 input_dirs='input_ad input'
1131 edhill 1.24 fi
1132 edhill 1.10 BUILD_DIR=$dir/$builddir
1133 edhill 1.28
1134 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1135 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1136     continue
1137     fi
1138 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1139     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1140     continue
1141     fi
1142    
1143     # Check whether there are "extra runs" for this testdir
1144     extra_runs=
1145     if test "x$ADM" = x ; then
1146     ex_run_dirs=`( cd $dir ; echo input.* )`
1147     #echo "ex_run_dirs='$ex_run_dirs'"
1148     for exd in $ex_run_dirs ; do
1149     name=`echo $exd | sed -e 's/input.//g'`
1150 jmc 1.85 outf="$dir/results/output.$name.txt"
1151 jmc 1.83 if test -f $outf -a -r $outf ; then
1152     if test "x$MULTI_THREAD" = "xt" ; then
1153     if test -r $dir"/"$exd"/eedata.mth" ; then
1154     extra_runs="$extra_runs $name"
1155     #else echo $dir"/"$exd"/eedata.mth: not found"
1156     fi
1157     else
1158     extra_runs="$extra_runs $name"
1159     fi
1160     fi
1161     done
1162     fi
1163 edhill 1.28
1164     echo "-------------------------------------------------------------------------------"
1165     echo
1166 jmc 1.83 if test "x$extra_runs" = "x" ; then
1167     echo "Experiment: $dir"
1168     else
1169     echo "Experiment: $dir ; extra_runs=$extra_runs"
1170     fi
1171 edhill 1.28 echo
1172     unset genmake makedepend make run
1173 jmc 1.94 results=$EMPTY_RESULTS
1174 edhill 1.10
1175     # Create an output dir for each OPTFILE/tdir combination
1176 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1177     mkdir $rel_CDIR
1178     CDIR=`pwd`"/$rel_CDIR"
1179 edhill 1.10
1180     if test "x$CLEANUP" = xt ; then
1181 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1182     makeclean $dir/$builddir \
1183     && run_clean $dir/$rundir
1184 edhill 1.10 else
1185 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1186     && makeclean $dir/$builddir \
1187 jmc 1.83 && run_clean $dir/$rundir \
1188 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1189 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1190     && makemodel $dir/$builddir && make=Y \
1191 jmc 1.83 && linkdata $dir/$rundir $input_dirs \
1192 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1193 jmc 1.93 && results=`testoutput_run $dir $rundir "txt"`
1194 edhill 1.10 fi
1195    
1196     echo
1197 edhill 1.24 if test "x$ADM" = x ; then
1198     fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1199 edhill 1.34 echo
1200 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1201 jmc 1.94 touch $CDIR"/summary.txt"
1202 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1203 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1204     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1205     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1206     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1207    
1208     for ex in $extra_runs ; do
1209 jmc 1.73 unset run
1210 jmc 1.94 results=$EMPTY_RESULTS
1211 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1212     rel_CDIR=$DRESULTS"/"$dir"."$ex
1213     mkdir $rel_CDIR
1214     CDIR=`pwd`"/$rel_CDIR"
1215 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1216 jmc 1.83 run_clean $dir/tr_run.$ex
1217     linkdata $dir/tr_run.$ex input.$ex input
1218 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1219 jmc 1.93 && results=`testoutput_run $dir tr_run.$ex ${ex}".txt"`
1220 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1221     fres="$fres.$ex"
1222     echo
1223 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1224 jmc 1.94 touch $CDIR"/summary.txt"
1225 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1226 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1227     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1228     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1229 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1230 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1231     run_clean $dir/tr_run.$ex
1232     fi
1233 edhill 1.34 done
1234 edhill 1.24 else
1235     fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
1236     fres=$fres"$results $dir"
1237 edhill 1.34 echo
1238     echo "$fres" >> $SUMMARY
1239 jmc 1.94 touch $CDIR"/summary.txt"
1240     echo "fresults='$fres'" >> $CDIR"/summary.txt"
1241 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1242     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1243     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1244     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1245 jmc 1.77 grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1246     >> $CDIR"/summary.txt"
1247 edhill 1.24 fi
1248 edhill 1.68
1249 jmc 1.83 #postclean $dir/$builddir
1250     if test "x$POSTCLEAN" = xt ; then
1251     makeclean $dir/$builddir \
1252     && run_clean $dir/$rundir
1253     fi
1254 edhill 1.10
1255     echo "-------------------------------------------------------------------------------"
1256    
1257 edhill 1.1 done
1258    
1259 edhill 1.34 printf "Start time: " >> $SUMMARY
1260 edhill 1.16 echo $start_date >> $SUMMARY
1261 edhill 1.34 printf "End time: " >> $SUMMARY
1262 edhill 1.13 date >> $SUMMARY
1263 edhill 1.20
1264     # If addresses were supplied and mpack built successfully, then try
1265     # to send email using mpack.
1266     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1267     echo "No results email was sent."
1268     else
1269     if test "x$HAVE_MPACK" = xt ; then
1270     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1271     && gzip $DRESULTS".tar" \
1272 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1273 edhill 1.20 RETVAL=$?
1274     if test "x$RETVAL" != x0 ; then
1275     echo
1276     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1277     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1278     echo " summary of results from the directory \"$DRESULTS\"."
1279     echo
1280     else
1281     echo
1282     echo "An email containing results was sent to the following addresses:"
1283     echo " \"$ADDRESSES\""
1284     echo
1285     fi
1286     test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1287     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1288     fi
1289     fi
1290 edhill 1.13
1291 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1292 edhill 1.1
1293 edhill 1.12 if test "x$CLEANUP" != xt ; then
1294 jmc 1.94 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \.//'
1295 edhill 1.25 if test -e tr_out.txt ; then
1296     mv tr_out.txt tr_out.txt.old
1297 edhill 1.14 fi
1298 jmc 1.98 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \.//g' > tr_out.txt
1299 edhill 1.12 fi
1300 edhill 1.1
1301 edhill 1.40 if test "x$DELDIR" = xt ; then
1302     rm -rf $DRESULTS
1303     fi
1304    

  ViewVC Help
Powered by ViewVC 1.1.22