/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.95 - (hide annotations) (download)
Fri Sep 7 00:44:37 2007 UTC (16 years, 6 months ago) by jmc
Branch: MAIN
Changes since 1.94: +4 -5 lines
- little fix (header of tr_out.txt)
- disable the "printing improvement" (main var between ><) until someone
  fix the building of testing web-pages

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.95 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.94 2007/09/06 22:31:45 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.83 echo " (-mth) run multi threaded (using eedata.mth)"
15     echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
19 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
20 edhill 1.6 echo " (DEF=\"edhill@mitgcm.org\")"
21 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
22     echo " (recognized groups: basic, tutorials)"
23     echo " (DEF=\"\" which test all)"
24 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
25     echo " (DEF=\"\" which test all)"
26 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
27     echo " Bourne-compatible \"sh\" shell"
28     echo " (DEF=\"\" for \"bash\")"
29 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
30 edhill 1.20 echo " (-command) STRING command to run"
31 edhill 1.6 echo " (DEF=\"make output.txt\")"
32 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
33 edhill 1.8 echo " (DEF=\"make\")"
34 jmc 1.70 echo " (-odir) STRING used to build output directory name"
35     echo " (DEF=\"hostname\")"
36 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
37 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
38 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
39     echo " (DEF=\"12\")"
40 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
41 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
42 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
43     echo " (-nogenmake|-ng) skip the genmake stage"
44     echo " (-noclean|-nc) skip the \"make clean\" stage"
45     echo " (-nodepend|-nd) skip the \"make depend\" stage"
46     echo " (-deldir|-dd) on success, delete the output directory"
47     echo " (-ts) provide timing information per timestep"
48     echo " (-papis) provide MFlop/s per timestep using PAPI"
49     echo " (-pcls) provide MFlop/s per timestep using PCL"
50 edhill 1.6 echo
51 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
52 edhill 1.6 echo "such as:"
53 edhill 1.50 echo
54 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
55     echo " -addr='abc@123.com testing@home.org'"
56 edhill 1.1 echo
57 edhill 1.50 echo "provided that the expression is properly quoted within the current"
58     echo "shell (note the use of single quotes to protect white space)."
59     echo
60 edhill 1.1 exit 1
61     }
62    
63     # build the mpack utility
64     build_mpack()
65     {
66 edhill 1.34 printf "building the mpack utility... "
67 edhill 1.1 if test ! -x "$MPACKDIR/mpack" ; then
68     if test ! -d $MPACKDIR ; then
69 edhill 1.20 echo
70 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
71     echo " are you sure this program is being run in the correct "
72     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
73 edhill 1.20 echo
74     HAVE_MPACK=f
75 edhill 1.1 fi
76 edhill 1.34 printf "building mpack... "
77 edhill 1.26 if test "x$CC" = x ; then
78     export CC=cc
79     fi
80 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
81 edhill 1.1 RETVAL=$?
82     if test "x$RETVAL" != x0 ; then
83     echo
84     echo "Error building the mpack tools at: $MPACK_DIR"
85 edhill 1.20 echo
86     HAVE_MPACK=f
87     else
88 edhill 1.25 rm -f tr_build_mpack.out
89 edhill 1.20 HAVE_MPACK=t
90 edhill 1.1 fi
91 edhill 1.20 else
92     HAVE_MPACK=t
93 edhill 1.1 fi
94     echo "OK"
95     }
96    
97 jmc 1.93 testoutput_var()
98 edhill 1.1 {
99 jmc 1.93 # testoutput_var dir s1 label subdir extension
100 edhill 1.1 #
101 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
102     # with same output from reference file $dir/results/output.$extension
103     # using search strings s1 and text label
104 edhill 1.1
105     if [ $debug -gt 0 ]; then
106 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
107 edhill 1.1 fi
108 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
109     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
110 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
111     if [ $lncntA -lt 3 ]; then
112 edhill 1.1 if [ $verbose -gt 0 ]; then
113     echo Not enough lines of output when searching for "$2" 1>&2
114     fi
115     return 99
116     fi
117     else
118 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
119 edhill 1.1 return 99
120     fi
121     if [ $debug -gt 0 ]; then
122 jmc 1.93 echo testoutput_var: grep "$2" $1/results/output.$5 1>&2
123 edhill 1.1 fi
124 jmc 1.85 grep "$2" $1/results/output.$5 | sed 's/.*=//' | cat -n > tmp2.txt
125 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
126     if [ $lncntB -lt 3 ]; then
127 edhill 1.1 if [ $verbose -gt 0 ]; then
128     echo Not enough lines of output when searching for "$2" 1>&2
129     fi
130     return 99
131     fi
132 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
133     if [ $verbose -gt 0 ]; then
134     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
135     fi
136     return 99
137     fi
138 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
139     if [ $has_nan -gt 0 ] ; then
140 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
141 edhill 1.72 return 99
142     fi
143     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
144     if [ $has_inf -gt 0 ] ; then
145 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
146 jmc 1.71 return 99
147     fi
148 edhill 1.1 if [ $debug -gt 0 ]; then
149 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
150 edhill 1.1 fi
151     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
152     if [ $debug -gt 0 ]; then
153 jmc 1.93 echo testoutput_var: compare_lines 1>&2
154 edhill 1.1 fi
155 edhill 1.22 if [ $verbose -gt 1 ]; then
156     cat tmp3.txt 1>&2
157     fi
158     echo "-1" >> tmp3.txt
159 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
160     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
161 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
162 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
163     if [ $verbose -gt 0 ]; then
164 jmc 1.93 echo testoutput_var: No comparison was available for \"$2\" 1>&2
165 edhill 1.1 fi
166     digits_of_similarity=99
167     else
168     if [ $verbose -gt 0 ]; then
169     echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2
170     fi
171     fi
172 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
173 edhill 1.1
174     return $digits_of_similarity
175     }
176    
177     dashnum()
178     {
179     # dashnum n1 n2 n3 ...
180     #
181     # print numbers using %3i format or "--" if number = 99
182    
183     for num in $@ ; do
184     if [ $num = 99 ]; then
185     printf ' --'
186     else
187     printf '%3i' $num
188     fi
189     done
190     }
191    
192 edhill 1.24 testoutput_ad()
193     {
194 jmc 1.78 grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt
195 jmc 1.87 grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $5}' > t15.txt
196 jmc 1.78 grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt
197 jmc 1.87 grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $6}' > t16.txt
198 edhill 1.24 join t05.txt t15.txt > t5.txt
199     join t06.txt t16.txt > t6.txt
200     echo "-1" >> t5.txt
201     echo "-1" >> t6.txt
202 jmc 1.91 digits_5=`./tr_cmpnum < t5.txt`
203     digits_6=`./tr_cmpnum < t6.txt`
204 edhill 1.24 dashnum $digits_5 $digits_6
205     rm -f t[01][56].txt t[56].txt
206     }
207    
208 edhill 1.49 check_for_add_mon_output()
209     {
210     # Check for additional types of monitor output
211     if test "x$1" = x ; then
212     return
213     fi
214    
215 edhill 1.50 for ii in $PTRACERS_NUM ; do
216     eval "HAVE_PTR0"$ii"=f"
217     done
218    
219 edhill 1.49 ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
220     ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
221 edhill 1.50 for ii in $PTRACERS_NUM ; do
222 edhill 1.49 for jj in $ptr_add ; do
223     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
224     tst=`grep $name $1 | wc -l | awk '{print $1}'`
225     if test ! "x$tst" = x0 ; then
226     eval "HAVE_PTR0"$ii"=t"
227     fi
228     done
229     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
230     done
231     }
232    
233 jmc 1.93 testoutput_run()
234 edhill 1.1 {
235 jmc 1.93 # testoutput_run directory subdir extension
236 edhill 1.1 #
237 jmc 1.93 # test output from 1 run in "directory"
238 edhill 1.24 if test "x$ADM" = x ; then
239 jmc 1.94 # default list of output variables to be checked:
240     # 1rst : main variable used to decide if it pass or FAIL
241     # others : number of matching digits to be printed in summary.txt
242     listChk=$DEF_CHECK_LIST
243     nbDef=`echo $listChk | awk '{print NF-1+3*gsub("+","E")}'`
244     # load experiment-specific list from file "tr_checklist" (if it exist)
245     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
246     sVar=`echo $listChk | awk '{print $1}'`
247     # remove 1rst var and expand the list: + => min max mean s.d
248     listVar=`echo $listChk | awk '{ for(i=2;i<=NF;i++){if (sub("+","",$i)) \
249     {printf " %s %s %s %s",$i"mn",$i"mx",$i"av",$i"sd"} else {printf " %s",$i} }}'`
250     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
251     for ii in 1 2 3 4 5 6 7 8 9 ; do
252     tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
253     #echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
254     if test "x$tst" != xt ; then listVar=`echo "$listVar" | sed "s/ pt$ii..//g"` ; fi
255     done
256     tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){t+=($i==$1)}; print t }'`
257     if test $tst != 1 ; then
258     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
259     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
260     echo "==> WARNING: in checked list:" $listVar 1>&2
261     #- put it back once:
262     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
263     fi
264     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
265     echo "listVar='$listVar'" > $CDIR"/summary.txt"
266     allargs=""
267     for xx in $listVar
268     do
269     case $xx in
270     'PS') if [ $debug -gt 0 ]
271     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
272     testoutput_var $1 "cg2d_init_res" "cg2d init. residual" $2 $3; yy=$?
273     if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
274     'Tmn') testoutput_var $1 "dynstat_theta_min" "theta minimum" $2 $3; yy=$? ;;
275     'Tmx') testoutput_var $1 "dynstat_theta_max" "theta maximum" $2 $3; yy=$? ;;
276     'Tav') testoutput_var $1 "dynstat_theta_mean" "theta mean" $2 $3; yy=$? ;;
277     'Tsd') testoutput_var $1 "dynstat_theta_sd" "theta s.d." $2 $3; yy=$? ;;
278     'Smn') testoutput_var $1 "dynstat_salt_min" "salt minimum" $2 $3; yy=$? ;;
279     'Smx') testoutput_var $1 "dynstat_salt_max" "salt maximum" $2 $3; yy=$? ;;
280     'Sav') testoutput_var $1 "dynstat_salt_mean" "salt mean" $2 $3; yy=$? ;;
281     'Ssd') testoutput_var $1 "dynstat_salt_sd" "salt s.d." $2 $3; yy=$? ;;
282     'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
283     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
284     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
285     'Usd') testoutput_var $1 "dynstat_uvel_sd" "U s.d." $2 $3; yy=$? ;;
286     'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
287     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
288     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
289     'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V s.d." $2 $3; yy=$? ;;
290     'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
291     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
292     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
293     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
294     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
295     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
296     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
297     testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_sd" $2 $3; yy=$? ;;
298     *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
299     esac
300     if test $xx = $sVar
301     then allargs="$allargs > $yy <"
302     else allargs="$allargs $yy"
303     fi
304     done
305    
306     nbVar=`echo $listVar | awk '{print NF}'`
307     if [ $nbVar -lt $nbDef ] ; then
308     #-- fill line (up to standard length) with dot:
309     adNul=`expr $nbDef - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
310     echo $allargs $adNul
311     else
312     echo $allargs
313     fi
314 edhill 1.49
315 edhill 1.24 else
316     testoutput_ad $1 $2 "precision_grdchk_result"
317 edhill 1.1 fi
318     }
319    
320     genmakemodel()
321     {
322     # genmakemodel directory
323 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
324     echo "genmake skipped!"
325     else
326 edhill 1.34 if test "x$BASH" = x ; then
327     GENMAKE2="../../../tools/genmake2"
328     else
329     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
330     fi
331 edhill 1.10 (
332     cd $1;
333 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
334     if test "x$ADM" = x ; then
335     command="$command --mods=../code"
336     else
337     command="$command --mods=../code_ad"
338     fi
339 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
340     command="$command --optfile=$OPTFILE"
341     fi
342     if test "x$IEEE" != x ; then
343     command="$command -ieee"
344     fi
345 edhill 1.66 if test "x$MPI" = xt ; then
346     command="$command -mpi"
347     fi
348 ce107 1.90 if test "x$TS" = xt ; then
349     command="$command -ts"
350     fi
351     if test "x$PAPIS" = xt ; then
352     command="$command -papis"
353     else
354     if test "x$PCLS" = xt ; then
355     command="$command -pcls"
356     fi
357     fi
358 edhill 1.10 printf 'genmake ... ' 1>&2
359     $command > make.log 2>&1
360     RETVAL=$?
361 edhill 1.44 # Reduce the size of the testing emails!
362 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
363 edhill 1.10 if test "x$RETVAL" != x0 ; then
364     tail make.log
365     echo "genmakemodel: genmake failed" 1>&2
366 edhill 1.21 cp genmake_* make.log $CDIR
367 edhill 1.10 return 1
368     else
369 edhill 1.20 echo "successful" 1>&2
370 edhill 1.1 fi
371 edhill 1.10 )
372     fi
373 edhill 1.1 }
374    
375     makeclean()
376     {
377     # makeclean directory
378 edhill 1.10 if test "x$NOCLEAN" = xt ; then
379 jmc 1.83 echo "make Clean skipped!"
380 edhill 1.10 else
381     (
382     cd $1;
383 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
384 edhill 1.10 if test -r Makefile ; then
385 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
386     $MAKE Clean >> make.log 2>&1
387 edhill 1.10 RETVAL=$?
388     if test "x$RETVAL" != x0 ; then
389     tail make.log
390 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
391 edhill 1.10 cp make.log $CDIR"/make.log"
392     return 1
393     fi
394     fi
395 edhill 1.20 echo successful 1>&2
396 edhill 1.10 exit 0
397     )
398     fi
399     }
400    
401 jmc 1.83 run_clean()
402 edhill 1.68 {
403 jmc 1.83 # run_clean directory
404     if test "x$NOCLEAN" = xt ; then
405     echo "run_clean skipped!"
406     else
407 edhill 1.68 (
408     cd $1;
409 jmc 1.83 printf 'clean run-dir ... ' 2>&1
410     # part of what is done after "make clean" when doing "make CLEAN"
411     find . -name "*.meta" -exec rm {} \;
412     find . -name "*.data" -exec rm {} \;
413     find . -name "fort.*" -exec rm {} \;
414     find . -type l -exec rm {} \;
415     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
416     rm -rf mnc_test_*
417     echo successful 1>&2
418 edhill 1.68 exit 0
419     )
420     fi
421     }
422    
423 edhill 1.10 makedependmodel()
424     {
425     # makedependmodel directory
426     if test "x$NODEPEND" = xt ; then
427     echo "make depend skipped!"
428     else
429     (
430     cd $1;
431     printf 'make depend ... ' 1>&2
432     $MAKE depend >> make.log 2>&1
433 edhill 1.1 RETVAL=$?
434     if test "x$RETVAL" != x0 ; then
435     tail make.log
436 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
437 edhill 1.1 cp make.log $CDIR"/make.log"
438     return 1
439 edhill 1.10 else
440 edhill 1.20 echo successful 1>&2
441 edhill 1.1 fi
442 edhill 1.10 )
443     fi
444 edhill 1.1 }
445    
446     makemodel()
447     {
448     # makemodel directory
449     (
450     cd $1;
451     if test -r Makefile ; then
452     printf 'make ... ' 1>&2
453 edhill 1.24 if test "x$ADM" = x ; then
454 edhill 1.43 if test "x$JOBS" = x ; then
455     $MAKE >> make.log 2>&1
456     else
457     $MAKE -j $JOBS >> make.log 2>&1
458     fi
459 edhill 1.24 else
460     $MAKE adall >> make.log 2>&1
461     fi
462 edhill 1.1 RETVAL=$?
463     if test "x$RETVAL" != x0 ; then
464     tail make.log
465     echo failed 1>&2
466     cp make.log $CDIR"/make.log"
467     return 1
468     else
469 edhill 1.20 echo successful 1>&2
470 edhill 1.1 fi
471     fi
472     )
473     }
474    
475 edhill 1.27 symlink_mpifiles()
476     {
477     # Put special links so that MPI specific files are used
478     # This MUST be invoked between makeclean and makelinks because
479     # the Makefile will link to non-mpi files by default
480    
481     dir=$1
482     code_dir=$2
483     BUILD_DIR=$dir/$3
484     CODE_DIR=$dir/$code_dir
485 edhill 1.45
486 edhill 1.27 # These are files that should replace their counter-part when using -mpi
487     MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
488    
489     # Is this an MPI run?
490     if test "x$MPI" = xt ; then
491     # YES: We symbolically link these files to the build
492     # dir so long as there is no real file in place
493     for ii in $MPI_FILES ; do
494     i=`echo $ii | sed 's:^\./::'`
495     name=`echo $i | sed 's:_mpi::' `
496     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
497     RETVAL=$?
498     if test "x$RETVAL" != x0 ; then
499     if ! test -f $BUILD_DIR/$i ; then
500 edhill 1.45 #echo Linking $name to $i
501 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
502     fi
503     fi
504     done
505     else
506 edhill 1.45 # NO: We undo any _mpi symbolically linked files
507 edhill 1.27 for ii in $MPI_FILES ; do
508     i=`echo $ii | sed 's:^\./::'`
509     name=`echo $i | sed 's:_mpi::' `
510     if test -L $BUILD_DIR/$name ; then
511 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
512     RETVAL=$?
513     if test "x$RETVAL" = x0 ; then
514     #echo Un-linking $name from $linktarg
515 edhill 1.27 rm -f $BUILD_DIR/$name
516     fi
517     fi
518     done
519     fi
520    
521     }
522    
523 edhill 1.1 linkdata()
524     {
525 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
526 edhill 1.1 #
527     # symbolically link data files to run directory
528 jmc 1.83 if test -d $1 ; then
529 edhill 1.12 (
530 jmc 1.83 cd $1 ; shift
531     if test -r "../"$1"/eedata.mth" ; then
532     # found eedata.mth in 1rst input dir and it is readable
533     if test "x$MULTI_THREAD" = "xt" ; then
534     # multi-threaded test: remove symbolic link & link eedata.mth
535     if test -h eedata ; then rm -f eedata ; fi
536     if test ! -r eedata ; then
537     ln -sf "../"$1"/eedata.mth" eedata ;
538     printf 'eedata.mth ' 1>&2
539 edhill 1.24 fi
540 jmc 1.83 else
541     # not multi-threaded test: remove eedata symbolic link
542     if test -h eedata ; then rm -f eedata ; fi
543     fi
544     fi
545     for ldir in $* ; do
546     if test -d "../"$ldir ; then
547     printf 'ldir='${ldir} 1>&2
548     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
549     for i in $files ; do
550     if test ! -d "../"$ldir/$i ; then
551     if test ! -r $i ; then
552     printf ' '$i 1>&2
553     ln -sf "../"$ldir"/"$i $i
554     fi
555     fi
556     done
557     if test -x "../"$ldir"/"prepare_run ; then
558     "../"$ldir"/"prepare_run
559 edhill 1.24 fi
560 jmc 1.83 printf ' ; ' 1>&2
561 jmc 1.82 fi
562 jmc 1.83 done
563 edhill 1.12 )
564 edhill 1.1 fi
565     }
566    
567     runmodel()
568     {
569 edhill 1.6 # runmodel directory
570 edhill 1.1 #
571 edhill 1.24 # runs "$COMMAND in "directory"
572 edhill 1.6 # (where "$COMMAND" is relative to "directory")
573 edhill 1.1 (
574     cd $1
575 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
576 edhill 1.6 # make output.txt
577 edhill 1.38 echo
578 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
579 jmc 1.88 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_00
580 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
581     fi
582     if test ! -x $EXECUTABLE ; then
583 jmc 1.88 rm -f run.log ; touch run.log
584     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
585 jmc 1.83 echo " no executable:" $EXECUTABLE >> run.log
586     RETVAL=8
587 jmc 1.88 ENDVAL=-1
588 jmc 1.75 else
589 jmc 1.83 if test $OUTPUTFILE -ot $EXECUTABLE ; then
590 jmc 1.88 rm -f run.log ; touch run.log
591     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
592 jmc 1.83 ( eval $COMMAND ) >> run.log 2>&1
593     RETVAL=$?
594     else
595     RETVAL=0
596 jmc 1.88 if test -f run.log ; then
597     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
598     echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1
599     else
600     touch run.log
601     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
602     echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1
603     echo " no previous run.log: assume NORMAL END" >> run.log 2>&1
604     fi
605 jmc 1.83 fi
606 jmc 1.88 ENDVAL=`cat run.log | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
607 jmc 1.75 fi
608 jmc 1.88 rm -f run.log_00
609     #if test "x$RETVAL" = x0 ; then
610     if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
611 jmc 1.46 tail run.log
612 edhill 1.20 echo successful 1>&2
613 edhill 1.44 # === Reduce the size of the testing emails!
614 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
615 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
616 edhill 1.6 return 0
617     else
618 edhill 1.20 tail run.log
619 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
620 edhill 1.20 cp run.log $CDIR"/run.log"
621 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
622 edhill 1.6 return 1
623 edhill 1.1 fi
624     )
625     }
626    
627     createcodelet()
628     {
629     # create codelet for comparing model output
630    
631 edhill 1.34 printf "creating the comparison code... "
632 jmc 1.91 cat > tr_cmpnum.c <<EOF
633 edhill 1.22 #include <stdio.h>
634     #include <math.h>
635     int main( int argc, char** argv ) {
636 jmc 1.91 int linnum,cmplin,best,lncnt;
637 edhill 1.52 double a,b,abave,relerr;
638 jmc 1.57 best = -22;
639 adcroft 1.33 lncnt = 0;
640     while( 1 & (lncnt+=1) < 999 ) {
641 edhill 1.22 scanf("%d", &linnum);
642     if (linnum == -1) break;
643     scanf("%lf", &a); scanf("%lf", &b);
644 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
645 jmc 1.91 if ( abave == abave ) {
646     if (abave > 0.0) {
647     relerr=fabs(a-b)/abave;
648     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
649     else { cmplin = -16 ; }
650     best = (best > cmplin) ? best : cmplin; }
651     else { cmplin = -22 ; }
652     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
653     }
654     else {
655     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
656     break; }
657 edhill 1.22 }
658 adcroft 1.33 if (lncnt == 999) best=-29;
659 jmc 1.91 if (linnum != -1) best=-99;
660 edhill 1.22 printf("%d\n", -best);
661     return 0;
662     }
663     EOF
664 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
665 edhill 1.1
666 jmc 1.91 if [ -x ./tr_cmpnum ]; then
667 edhill 1.1 echo "OK"
668     return 0
669     else
670     echo
671 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
672     echo " a C compiler using the CC environment variable."
673 edhill 1.1 exit 1
674     fi
675     }
676    
677     formatresults()
678     {
679     # formatresults expt genmake depend make run results*
680    
681     nm=$1
682     printf '%s %s %s %s' $2 $3 $4 $5
683     shift; shift; shift; shift; shift;
684 jmc 1.94 listPrt=$@
685     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
686     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
687 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
688     printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > / /' | sed 's/ < / /'
689 edhill 1.1
690 jmc 1.94 if [ $xx = '..' ]; then
691     printf ' N/O '
692     elif [ $xx = '--' ]; then
693     printf ' N/O '
694     elif [ $xx = 99 ]; then
695 edhill 1.1 printf ' N/O '
696     else
697 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
698 edhill 1.1 printf ' pass'
699     else
700     printf ' FAIL'
701     fi
702     fi
703     printf ' %s' $nm
704     printf '\n'
705    
706     }
707    
708     scandirs()
709     {
710 jmc 1.81 if [ $# -eq 1 ]; then
711     for arg in * ; do
712     test -d $arg/$1 && echo $arg
713     done
714 edhill 1.1 else
715 jmc 1.81 echo $*
716 edhill 1.1 fi
717     }
718    
719    
720     ###############################################################################
721    
722    
723     # Default properties
724     debug=0
725     verbose=1
726     clean=0
727     expts=''
728 edhill 1.6 # ieee=1
729 edhill 1.10
730 edhill 1.40 IEEE=true
731 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
732     IEEE=$MITGCM_IEEE
733     fi
734    
735    
736     CLEANUP=f
737     QUICK=f
738     NOGENMAKE=f
739     NOCLEAN=f
740     NODEPEND=f
741 edhill 1.68 POSTCLEAN=f
742 edhill 1.1
743 edhill 1.4 BASH=
744 edhill 1.10 OPTFILE=NONE
745     ADDRESSES=
746 edhill 1.1 TESTDIRS=
747 jmc 1.89 SKIPDIRS=
748 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
749 edhill 1.20 HAVE_MPACK=
750 edhill 1.1 MPACK="$MPACKDIR/mpack"
751 edhill 1.24 COMMAND=
752 edhill 1.59 if test "x$MAKE" = x ; then
753     MAKE=make
754     fi
755     if test "x$CC" = x ; then
756     CC=cc
757     fi
758 edhill 1.43 JOBS=
759 edhill 1.7 MPI=f
760 jmc 1.83 MULTI_THREAD=f
761 jmc 1.70 OUTDIR=
762 edhill 1.40 DELDIR=
763 edhill 1.1
764 edhill 1.24 ADM=
765    
766 edhill 1.49 # Additional monitor types
767 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
768 edhill 1.49
769 jmc 1.92 MATCH_CRIT=13
770    
771 edhill 1.34 printf "parsing options... "
772 edhill 1.1
773     ac_prev=
774     for ac_option ; do
775    
776     # If the previous option needs an argument, assign it.
777     if test -n "$ac_prev"; then
778     eval "$ac_prev=\$ac_option"
779     ac_prev=
780     continue
781     fi
782    
783     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
784    
785     case $ac_option in
786    
787     -help | --help | -h | --h)
788     usage ;;
789    
790 edhill 1.2 -optfile | --optfile | -of | --of)
791 edhill 1.10 ac_prev=OPTFILE ;;
792 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
793 edhill 1.10 OPTFILE=$ac_optarg ;;
794 edhill 1.1
795     -addr | --addr | -a | --a)
796     ac_prev=ADDRESSES ;;
797     -addr=* | --addr=*)
798     ADDRESSES=$ac_optarg ;;
799    
800     -tdir | --tdir | -t | --t)
801     ac_prev=TESTDIRS ;;
802     -tdir=* | --tdir=*)
803     TESTDIRS=$ac_optarg ;;
804 edhill 1.4
805 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
806     ac_prev=SKIPDIRS ;;
807     -skipdir=* | --skipdir=*)
808     SKIPDIRS=$ac_optarg ;;
809    
810 edhill 1.4 -bash | --bash | -b | --b)
811     ac_prev=BASH ;;
812     -bash=* | --bash=*)
813     BASH=$ac_optarg ;;
814 edhill 1.5
815 edhill 1.6 -command | --command | -c | --c)
816     ac_prev=COMMAND ;;
817     -command=* | --command=*)
818     COMMAND=$ac_optarg ;;
819 edhill 1.8
820     -make | --make | -m | --m)
821     ac_prev=MAKE ;;
822     -make=* | --make=*)
823     MAKE=$ac_optarg ;;
824 edhill 1.1
825 jmc 1.70 -odir | --odir)
826     ac_prev=OUTDIR ;;
827     -odir=* | --odir=*)
828     OUTDIR=$ac_optarg ;;
829    
830 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
831     ac_prev=PTRACERS_NUM ;;
832     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
833     PTRACERS_NUM=$ac_optarg ;;
834    
835 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
836     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
837    
838 edhill 1.43 -j) ac_prev=JOBS ;;
839     -j=*) JOBS=$ac_optarg ;;
840    
841 edhill 1.10 -clean | --clean)
842 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
843 edhill 1.10
844     -quick | --quick | -q | --q)
845     QUICK=t ;;
846     -nogenmake | --nogenmake | -ng | --ng)
847     NOGENMAKE=t ;;
848     -noclean | --noclean | -nc | --nc)
849     NOCLEAN=t ;;
850     -nodepend | --nodepend | -nd | --nd)
851     NODEPEND=t ;;
852    
853 edhill 1.68 -postclean | --postclean | -pc | --pc)
854     POSTCLEAN=t ;;
855    
856 edhill 1.18 -mpi) MPI=t ;;
857 edhill 1.10
858 jmc 1.83 -mth) MULTI_THREAD=t ;;
859    
860 edhill 1.24 -adm | -ad) ADM=t ;;
861    
862 edhill 1.10 -ieee) IEEE=true ;;
863     -noieee) IEEE= ;;
864    
865 edhill 1.1 -verbose) verbose=2 ;;
866     -debug) debug=1 ;;
867     -quiet) verbose=0 ;;
868    
869 edhill 1.40 -deldir | -dd) DELDIR=t ;;
870    
871 ce107 1.90 -ts) TS=t;;
872    
873     -papis) PAPIS=t;;
874    
875     -pcls) PCL=t;;
876    
877 edhill 1.1 -*)
878     echo "Error: unrecognized option: "$ac_option
879     usage
880     ;;
881    
882     *)
883     echo "Error: unrecognized argument: "$ac_option
884     usage
885     ;;
886    
887     esac
888    
889     done
890    
891 edhill 1.10 if test "x$QUICK" = xt ; then
892     NOGENMAKE=t
893     NOCLEAN=t
894     NODEPEND=t
895     fi
896    
897 edhill 1.1 if test "x$TESTDIRS" = x ; then
898 jmc 1.81 if test "x$ADM" = xt ; then
899 jmc 1.89 LIST=`scandirs results_ad`
900 jmc 1.81 else
901 jmc 1.89 LIST=`scandirs results`
902 jmc 1.81 fi
903 jmc 1.84 else
904     #- expand group of experiments:
905     LIST=" "
906     for xx in $TESTDIRS
907     do
908     case $xx in
909 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
910     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
911     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
912 jmc 1.84 ;;
913     'tutorials')
914     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
915     *) LIST=${LIST}" "$xx ;;
916     esac
917     done
918 jmc 1.89 fi
919     #echo 'LIST='${LIST}'<'
920     #- skip dirs, remove duplicate and non-directory:
921     TESTDIRS=" "
922     count=0
923     for xx in $LIST
924     do
925     yy=`echo $SKIPDIRS | grep -c $xx`
926     if test $yy = 0 ; then
927 jmc 1.84 if test -d $xx ; then
928     yy=`echo $TESTDIRS | grep -c $xx`
929     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
930 jmc 1.89 else count=1 ;
931     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
932     fi
933     else
934     if test $count = 1 ; then echo -n ", \"$xx\""
935     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
936 jmc 1.84 fi
937 jmc 1.89 fi
938     done
939     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
940 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
941 edhill 1.1
942 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
943     OPTFILE=$MITGCM_OF
944     fi
945    
946 jmc 1.76 if test "x$ADM" = xt ; then
947     EXECUTABLE="mitgcmuv_ad"
948 jmc 1.78 OUTPUTFILE="output_adm.txt"
949 jmc 1.76 else
950     EXECUTABLE="mitgcmuv"
951     OUTPUTFILE="output.txt"
952     fi
953    
954 jmc 1.87 if test "x$COMMAND" = x ; then
955 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
956 edhill 1.24 fi
957 jmc 1.87 if test "x$MPI" = xt ; then
958     OUTPUTFILE="STDOUT.0000"
959 edhill 1.24 fi
960    
961 jmc 1.76 #echo "OK"
962     echo "OK (COMMAND= $COMMAND )"
963 edhill 1.1
964 jmc 1.94 # set the Default List of output variables to be checked:
965     # (use default or load experiment-specific list from file "tr_checklist")
966     # content : 1rst = main variable used to decide if it pass or FAIL
967     # others = number of matching digits to be printed in summary.txt
968     if test "x$ADM" != xt ; then
969     DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
970     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
971     nbDef=`echo $DEF_CHECK_LIST | awk '{print NF-1+3*gsub("+","E")}'`
972     ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
973     EMPTY_RESULTS=$EMPTY_RESULTS`expr $nbDef - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
974     fi
975    
976 edhill 1.1 # create the FORTRAN comparison code
977     createcodelet
978    
979     # build the mpack utility
980 edhill 1.31 if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
981 edhill 1.32 echo "skipping mpack build"
982     else
983 edhill 1.31 build_mpack
984     fi
985 edhill 1.1
986     # Create a uniquely named directory to store results
987     MACH=`hostname`
988 edhill 1.2 UNAMEA=`uname -a`
989 edhill 1.1 DATE=`date +%Y%m%d`
990 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
991 jmc 1.70 if test "x$OUTDIR" != x ; then
992     BASE="tr_"$OUTDIR"_"$DATE"_"
993     fi
994 edhill 1.1 DNUM=0
995     DRESULTS="$BASE$DNUM"
996     while test -e $DRESULTS ; do
997     DNUM=$(( $DNUM + 1 ))
998     DRESULTS="$BASE$DNUM"
999     done
1000     mkdir $DRESULTS
1001     RETVAL=$?
1002     if test "x$RETVAL" != x0 ; then
1003 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1004 edhill 1.1 exit 1
1005     fi
1006     SUMMARY="$DRESULTS/summary.txt"
1007 edhill 1.34 printf "Start time: " >> $SUMMARY
1008 edhill 1.16 start_date=`date`
1009 edhill 1.17 echo $start_date > $SUMMARY
1010 edhill 1.1
1011 edhill 1.11 of_path=
1012 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1013     if test -r $OPTFILE ; then
1014 edhill 1.11 # get the path
1015     path=${OPTFILE%/*}
1016     if test "x$path" = x ; then
1017     of_path=`pwd`
1018     else
1019     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1020     fi
1021     file=${OPTFILE##*/}
1022     OPTFILE=$of_path/$file
1023 edhill 1.21 cp $OPTFILE $DRESULTS
1024     echo >> $SUMMARY
1025     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1026 edhill 1.11 else
1027 edhill 1.21 echo | tee $SUMMARY
1028     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1029     exit 1
1030 edhill 1.10 fi
1031 edhill 1.21 else
1032     echo >> $SUMMARY
1033     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1034     echo " so the genmake default will be used." >> $SUMMARY
1035 edhill 1.10 fi
1036     echo
1037     echo >> $SUMMARY
1038 edhill 1.24 if test "x$ADM" = x ; then
1039 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1040 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1041     else line_0="default "$MATCH_CRIT ; fi
1042     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1043 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1044 edhill 1.49 line_1="G D M c m s m s m s m s"
1045     line_2="E p a R g m m e . m m e . m m e . m m e ."
1046     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1047     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1048 edhill 1.50 for ii in $PTRACERS_NUM ; do
1049 edhill 1.49 # tst=`eval 'echo $HAVE_PTR0'$ii`
1050     # if test "x$tst" = xt ; then
1051     line_0="$line_0 --PTR 0"$ii"--"
1052     line_1="$line_1 m s"
1053     line_2="$line_2 m m e ."
1054     line_3="$line_3 i a a d"
1055     line_4="$line_4 n x n ."
1056     # fi
1057     done
1058     echo "$line_0" | tee -a $SUMMARY
1059     echo "$line_1" | tee -a $SUMMARY
1060     echo "$line_2" | tee -a $SUMMARY
1061     echo "$line_3" | tee -a $SUMMARY
1062     echo "$line_4" | tee -a $SUMMARY
1063     echo " " | tee -a $SUMMARY
1064 edhill 1.24 else
1065     echo "ADJOINT=true" >> $SUMMARY
1066     echo >> $SUMMARY
1067     cat << EOF | tee -a $SUMMARY
1068     G D M C G
1069     E p a R o r
1070     N n k u s a
1071     2 d e n t d
1072    
1073     EOF
1074     fi
1075 edhill 1.1
1076 edhill 1.10 # ...and each test directory...
1077     for dir in $TESTDIRS ; do
1078    
1079     # Cleanup only!
1080     if test "x$CLEANUP" = xt ; then
1081     if test -r $dir/build/Makefile ; then
1082 jmc 1.83 echo ' ------ clean dir:' $dir/build
1083 edhill 1.10 ( cd $dir/build ; make CLEAN )
1084     fi
1085 jmc 1.83 if test -d $dir/run/CVS ; then
1086     echo ' ------ clean dir:' $dir/run
1087     run_clean $dir/run
1088 edhill 1.10 fi
1089 edhill 1.62 (
1090     cd $dir
1091     rm -rf tr_run.*
1092     )
1093 edhill 1.10 continue
1094 edhill 1.1 fi
1095 edhill 1.3
1096 edhill 1.10 # Verify that the testdir exists and contains previous
1097     # results in the correct location--or skip this directory!
1098 edhill 1.24 fout=
1099     if test "x$ADM" = x ; then
1100     fout=$dir"/results/output.txt"
1101     else
1102 jmc 1.78 fout=$dir"/results_ad/output_adm.txt"
1103 edhill 1.24 fi
1104     if test ! -r $fout ; then
1105     echo "can't read \"$fout\" -- skipping $dir"
1106 edhill 1.10 continue
1107     fi
1108 edhill 1.53 if test "x$ADM" = x ; then
1109     check_for_add_mon_output $fout
1110     fi
1111 edhill 1.7
1112 edhill 1.49 # Check for additional types of monitor output
1113    
1114 jmc 1.83 builddir="build"
1115     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1116     rundir="run"
1117     #rundir=$builddir
1118     if test ! -d $dir/$rundir ; then
1119     rundir=$builddir
1120 edhill 1.10 fi
1121    
1122 edhill 1.24 if test "x$ADM" = x ; then
1123 edhill 1.27 code_dir=code
1124 edhill 1.24 CODE_DIR=$dir/code
1125 jmc 1.83 input_dirs='input'
1126 edhill 1.24 else
1127 edhill 1.27 code_dir=code_ad
1128 edhill 1.24 CODE_DIR=$dir/code_ad
1129 jmc 1.83 input_dirs='input_ad input'
1130 edhill 1.24 fi
1131 edhill 1.10 BUILD_DIR=$dir/$builddir
1132 edhill 1.28
1133 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1134 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1135     continue
1136     fi
1137 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1138     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1139     continue
1140     fi
1141    
1142     # Check whether there are "extra runs" for this testdir
1143     extra_runs=
1144     if test "x$ADM" = x ; then
1145     ex_run_dirs=`( cd $dir ; echo input.* )`
1146     #echo "ex_run_dirs='$ex_run_dirs'"
1147     for exd in $ex_run_dirs ; do
1148     name=`echo $exd | sed -e 's/input.//g'`
1149 jmc 1.85 outf="$dir/results/output.$name.txt"
1150 jmc 1.83 if test -f $outf -a -r $outf ; then
1151     if test "x$MULTI_THREAD" = "xt" ; then
1152     if test -r $dir"/"$exd"/eedata.mth" ; then
1153     extra_runs="$extra_runs $name"
1154     #else echo $dir"/"$exd"/eedata.mth: not found"
1155     fi
1156     else
1157     extra_runs="$extra_runs $name"
1158     fi
1159     fi
1160     done
1161     fi
1162 edhill 1.28
1163     echo "-------------------------------------------------------------------------------"
1164     echo
1165 jmc 1.83 if test "x$extra_runs" = "x" ; then
1166     echo "Experiment: $dir"
1167     else
1168     echo "Experiment: $dir ; extra_runs=$extra_runs"
1169     fi
1170 edhill 1.28 echo
1171     unset genmake makedepend make run
1172 jmc 1.94 results=$EMPTY_RESULTS
1173 edhill 1.10
1174     # Create an output dir for each OPTFILE/tdir combination
1175 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1176     mkdir $rel_CDIR
1177     CDIR=`pwd`"/$rel_CDIR"
1178 edhill 1.10
1179     if test "x$CLEANUP" = xt ; then
1180 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1181     makeclean $dir/$builddir \
1182     && run_clean $dir/$rundir
1183 edhill 1.10 else
1184 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1185     && makeclean $dir/$builddir \
1186 jmc 1.83 && run_clean $dir/$rundir \
1187 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1188 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1189     && makemodel $dir/$builddir && make=Y \
1190 jmc 1.83 && linkdata $dir/$rundir $input_dirs \
1191 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1192 jmc 1.93 && results=`testoutput_run $dir $rundir "txt"`
1193 edhill 1.10 fi
1194    
1195     echo
1196 edhill 1.24 if test "x$ADM" = x ; then
1197     fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1198 edhill 1.34 echo
1199     echo "$fres" >> $SUMMARY
1200 jmc 1.94 touch $CDIR"/summary.txt"
1201     echo "fresults='$fres'" >> $CDIR"/summary.txt"
1202 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1203     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1204     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1205     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1206    
1207     for ex in $extra_runs ; do
1208 jmc 1.73 unset run
1209 jmc 1.94 results=$EMPTY_RESULTS
1210 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1211     rel_CDIR=$DRESULTS"/"$dir"."$ex
1212     mkdir $rel_CDIR
1213     CDIR=`pwd`"/$rel_CDIR"
1214 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1215 jmc 1.83 run_clean $dir/tr_run.$ex
1216     linkdata $dir/tr_run.$ex input.$ex input
1217 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1218 jmc 1.93 && results=`testoutput_run $dir tr_run.$ex ${ex}".txt"`
1219 edhill 1.34 fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
1220     fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1221     fres="$fres.$ex"
1222     echo
1223     echo "$fres" >> $SUMMARY
1224 jmc 1.94 touch $CDIR"/summary.txt"
1225     echo "fresults='$fres'" >> $CDIR"/summary.txt"
1226 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1227     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1228     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1229 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1230 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1231     run_clean $dir/tr_run.$ex
1232     fi
1233 edhill 1.34 done
1234 edhill 1.24 else
1235     fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
1236     fres=$fres"$results $dir"
1237 edhill 1.34 echo
1238     echo "$fres" >> $SUMMARY
1239 jmc 1.94 touch $CDIR"/summary.txt"
1240     echo "fresults='$fres'" >> $CDIR"/summary.txt"
1241 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1242     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1243     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1244     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1245 jmc 1.77 grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1246     >> $CDIR"/summary.txt"
1247 edhill 1.24 fi
1248 edhill 1.68
1249 jmc 1.83 #postclean $dir/$builddir
1250     if test "x$POSTCLEAN" = xt ; then
1251     makeclean $dir/$builddir \
1252     && run_clean $dir/$rundir
1253     fi
1254 edhill 1.10
1255     echo "-------------------------------------------------------------------------------"
1256    
1257 edhill 1.1 done
1258    
1259 edhill 1.34 printf "Start time: " >> $SUMMARY
1260 edhill 1.16 echo $start_date >> $SUMMARY
1261 edhill 1.34 printf "End time: " >> $SUMMARY
1262 edhill 1.13 date >> $SUMMARY
1263 edhill 1.20
1264     # If addresses were supplied and mpack built successfully, then try
1265     # to send email using mpack.
1266     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1267     echo "No results email was sent."
1268     else
1269     if test "x$HAVE_MPACK" = xt ; then
1270     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1271     && gzip $DRESULTS".tar" \
1272 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1273 edhill 1.20 RETVAL=$?
1274     if test "x$RETVAL" != x0 ; then
1275     echo
1276     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1277     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1278     echo " summary of results from the directory \"$DRESULTS\"."
1279     echo
1280     else
1281     echo
1282     echo "An email containing results was sent to the following addresses:"
1283     echo " \"$ADDRESSES\""
1284     echo
1285     fi
1286     test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1287     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1288     fi
1289     fi
1290 edhill 1.13
1291 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1292 edhill 1.1
1293 edhill 1.12 if test "x$CLEANUP" != xt ; then
1294 jmc 1.94 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \.//'
1295 edhill 1.25 if test -e tr_out.txt ; then
1296     mv tr_out.txt tr_out.txt.old
1297 edhill 1.14 fi
1298 jmc 1.95 cat $SUMMARY | sed '11,$ s/ \.//g' > tr_out.txt
1299 edhill 1.12 fi
1300 edhill 1.1
1301 edhill 1.40 if test "x$DELDIR" = xt ; then
1302     rm -rf $DRESULTS
1303     fi
1304    

  ViewVC Help
Powered by ViewVC 1.1.22