/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.100 - (hide annotations) (download)
Mon Sep 17 15:35:46 2007 UTC (16 years, 6 months ago) by jmc
Branch: MAIN
Changes since 1.99: +33 -29 lines
use same scripts for forward & adm testing (testoutput_ad no longer used)
and check AD output against results/output_adm.txt

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.100 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.99 2007/09/15 20:30:05 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.83 echo " (-mth) run multi threaded (using eedata.mth)"
15     echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
19 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
20 edhill 1.6 echo " (DEF=\"edhill@mitgcm.org\")"
21 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
22     echo " (recognized groups: basic, tutorials)"
23     echo " (DEF=\"\" which test all)"
24 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
25     echo " (DEF=\"\" which test all)"
26 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
27     echo " Bourne-compatible \"sh\" shell"
28     echo " (DEF=\"\" for \"bash\")"
29 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
30 edhill 1.20 echo " (-command) STRING command to run"
31 edhill 1.6 echo " (DEF=\"make output.txt\")"
32 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
33 edhill 1.8 echo " (DEF=\"make\")"
34 jmc 1.70 echo " (-odir) STRING used to build output directory name"
35     echo " (DEF=\"hostname\")"
36 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
37 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
38 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
39     echo " (DEF=\"12\")"
40 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
41 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
42 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
43     echo " (-nogenmake|-ng) skip the genmake stage"
44     echo " (-noclean|-nc) skip the \"make clean\" stage"
45     echo " (-nodepend|-nd) skip the \"make depend\" stage"
46     echo " (-deldir|-dd) on success, delete the output directory"
47     echo " (-ts) provide timing information per timestep"
48     echo " (-papis) provide MFlop/s per timestep using PAPI"
49     echo " (-pcls) provide MFlop/s per timestep using PCL"
50 edhill 1.6 echo
51 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
52 edhill 1.6 echo "such as:"
53 edhill 1.50 echo
54 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
55     echo " -addr='abc@123.com testing@home.org'"
56 edhill 1.1 echo
57 edhill 1.50 echo "provided that the expression is properly quoted within the current"
58     echo "shell (note the use of single quotes to protect white space)."
59     echo
60 edhill 1.1 exit 1
61     }
62    
63     # build the mpack utility
64     build_mpack()
65     {
66 edhill 1.34 printf "building the mpack utility... "
67 edhill 1.1 if test ! -x "$MPACKDIR/mpack" ; then
68     if test ! -d $MPACKDIR ; then
69 edhill 1.20 echo
70 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
71     echo " are you sure this program is being run in the correct "
72     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
73 edhill 1.20 echo
74     HAVE_MPACK=f
75 edhill 1.1 fi
76 edhill 1.34 printf "building mpack... "
77 edhill 1.26 if test "x$CC" = x ; then
78     export CC=cc
79     fi
80 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
81 edhill 1.1 RETVAL=$?
82     if test "x$RETVAL" != x0 ; then
83     echo
84     echo "Error building the mpack tools at: $MPACK_DIR"
85 edhill 1.20 echo
86     HAVE_MPACK=f
87     else
88 edhill 1.25 rm -f tr_build_mpack.out
89 edhill 1.20 HAVE_MPACK=t
90 edhill 1.1 fi
91 edhill 1.20 else
92     HAVE_MPACK=t
93 edhill 1.1 fi
94     echo "OK"
95     }
96    
97 jmc 1.93 testoutput_var()
98 edhill 1.1 {
99 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
100 edhill 1.1 #
101 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
102 jmc 1.99 # with same output from reference file $dir/results/$reference_output
103 jmc 1.93 # using search strings s1 and text label
104 edhill 1.1
105     if [ $debug -gt 0 ]; then
106 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
107 edhill 1.1 fi
108 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
109     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
110 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
111 jmc 1.100 if [ $lncntA -lt 2 ]; then
112 edhill 1.1 if [ $verbose -gt 0 ]; then
113     echo Not enough lines of output when searching for "$2" 1>&2
114     fi
115     return 99
116     fi
117     else
118 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
119 edhill 1.1 return 99
120     fi
121     if [ $debug -gt 0 ]; then
122 jmc 1.99 echo testoutput_var: grep "$2" $1/results/$5 1>&2
123 edhill 1.1 fi
124 jmc 1.99 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
125 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
126 jmc 1.100 if [ $lncntB -lt 2 ]; then
127 edhill 1.1 if [ $verbose -gt 0 ]; then
128     echo Not enough lines of output when searching for "$2" 1>&2
129     fi
130     return 99
131     fi
132 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
133     if [ $verbose -gt 0 ]; then
134     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
135     fi
136     return 99
137     fi
138 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
139     if [ $has_nan -gt 0 ] ; then
140 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
141 edhill 1.72 return 99
142     fi
143     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
144     if [ $has_inf -gt 0 ] ; then
145 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
146 jmc 1.71 return 99
147     fi
148 edhill 1.1 if [ $debug -gt 0 ]; then
149 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
150 edhill 1.1 fi
151     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
152     if [ $debug -gt 0 ]; then
153 jmc 1.93 echo testoutput_var: compare_lines 1>&2
154 edhill 1.1 fi
155 edhill 1.22 if [ $verbose -gt 1 ]; then
156     cat tmp3.txt 1>&2
157     fi
158     echo "-1" >> tmp3.txt
159 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
160     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
161 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
162 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
163     if [ $verbose -gt 0 ]; then
164 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
165 edhill 1.1 fi
166     digits_of_similarity=99
167     else
168     if [ $verbose -gt 0 ]; then
169 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
170 edhill 1.1 fi
171     fi
172 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
173 edhill 1.1
174     return $digits_of_similarity
175     }
176    
177     dashnum()
178     {
179     # dashnum n1 n2 n3 ...
180     #
181     # print numbers using %3i format or "--" if number = 99
182    
183     for num in $@ ; do
184     if [ $num = 99 ]; then
185     printf ' --'
186     else
187     printf '%3i' $num
188     fi
189     done
190     }
191    
192 edhill 1.24 testoutput_ad()
193     {
194 jmc 1.78 grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt
195 jmc 1.87 grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $5}' > t15.txt
196 jmc 1.78 grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt
197 jmc 1.87 grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $6}' > t16.txt
198 edhill 1.24 join t05.txt t15.txt > t5.txt
199     join t06.txt t16.txt > t6.txt
200     echo "-1" >> t5.txt
201     echo "-1" >> t6.txt
202 jmc 1.91 digits_5=`./tr_cmpnum < t5.txt`
203     digits_6=`./tr_cmpnum < t6.txt`
204 edhill 1.24 dashnum $digits_5 $digits_6
205     rm -f t[01][56].txt t[56].txt
206     }
207    
208 edhill 1.49 check_for_add_mon_output()
209     {
210     # Check for additional types of monitor output
211     if test "x$1" = x ; then
212     return
213     fi
214    
215 edhill 1.50 for ii in $PTRACERS_NUM ; do
216     eval "HAVE_PTR0"$ii"=f"
217     done
218    
219 edhill 1.49 ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max"
220     ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
221 edhill 1.50 for ii in $PTRACERS_NUM ; do
222 edhill 1.49 for jj in $ptr_add ; do
223     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
224     tst=`grep $name $1 | wc -l | awk '{print $1}'`
225     if test ! "x$tst" = x0 ; then
226     eval "HAVE_PTR0"$ii"=t"
227     fi
228     done
229     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"'
230     done
231     }
232    
233 jmc 1.93 testoutput_run()
234 edhill 1.1 {
235 jmc 1.99 # testoutput_run directory subdir reference_output
236 edhill 1.1 #
237 jmc 1.93 # test output from 1 run in "directory"
238 jmc 1.100 # if test "x$ADM" = x ; then
239 jmc 1.94 # default list of output variables to be checked:
240     # 1rst : main variable used to decide if it pass or FAIL
241     # others : number of matching digits to be printed in summary.txt
242     listChk=$DEF_CHECK_LIST
243     # load experiment-specific list from file "tr_checklist" (if it exist)
244     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
245     sVar=`echo $listChk | awk '{print $1}'`
246     # remove 1rst var and expand the list: + => min max mean s.d
247 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
248     | sed 's/+//g' | sed "s/^$sVar//"`
249 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
250     for ii in 1 2 3 4 5 6 7 8 9 ; do
251     tst=`eval 'echo "$HAVE_PTR0'$ii'"'`
252     #echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
253     if test "x$tst" != xt ; then listVar=`echo "$listVar" | sed "s/ pt$ii..//g"` ; fi
254     done
255     tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){t+=($i==$1)}; print t }'`
256     if test $tst != 1 ; then
257     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
258     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
259     echo "==> WARNING: in checked list:" $listVar 1>&2
260     #- put it back once:
261     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
262     fi
263     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
264     echo "listVar='$listVar'" > $CDIR"/summary.txt"
265     allargs=""
266     for xx in $listVar
267     do
268     case $xx in
269     'PS') if [ $debug -gt 0 ]
270     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
271 jmc 1.99 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
272 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
273 jmc 1.99 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
274     'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
275     'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
276     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
277     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
278     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
279     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
280     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
281     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
282     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
283 jmc 1.94 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
284     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
285     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
286 jmc 1.99 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
287 jmc 1.94 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
288     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
289     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
290 jmc 1.99 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
291 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
292     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
293     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
294     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
295     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
296     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
297     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
298 jmc 1.99 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
299 jmc 1.94 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
300     esac
301     if test $xx = $sVar
302     then allargs="$allargs > $yy <"
303     else allargs="$allargs $yy"
304     fi
305     done
306    
307     nbVar=`echo $listVar | awk '{print NF}'`
308 jmc 1.97 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
309 jmc 1.94 #-- fill line (up to standard length) with dot:
310 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
311 jmc 1.94 echo $allargs $adNul
312     else
313     echo $allargs
314     fi
315 edhill 1.49
316 jmc 1.100 # else
317     # testoutput_ad $1 $2 "precision_grdchk_result"
318     # fi
319 edhill 1.1 }
320    
321     genmakemodel()
322     {
323     # genmakemodel directory
324 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
325     echo "genmake skipped!"
326     else
327 edhill 1.34 if test "x$BASH" = x ; then
328     GENMAKE2="../../../tools/genmake2"
329     else
330     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
331     fi
332 edhill 1.10 (
333     cd $1;
334 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
335     if test "x$ADM" = x ; then
336     command="$command --mods=../code"
337     else
338     command="$command --mods=../code_ad"
339     fi
340 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
341     command="$command --optfile=$OPTFILE"
342     fi
343     if test "x$IEEE" != x ; then
344     command="$command -ieee"
345     fi
346 edhill 1.66 if test "x$MPI" = xt ; then
347     command="$command -mpi"
348     fi
349 ce107 1.90 if test "x$TS" = xt ; then
350     command="$command -ts"
351     fi
352     if test "x$PAPIS" = xt ; then
353     command="$command -papis"
354     else
355     if test "x$PCLS" = xt ; then
356     command="$command -pcls"
357     fi
358     fi
359 edhill 1.10 printf 'genmake ... ' 1>&2
360     $command > make.log 2>&1
361     RETVAL=$?
362 edhill 1.44 # Reduce the size of the testing emails!
363 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
364 edhill 1.10 if test "x$RETVAL" != x0 ; then
365     tail make.log
366     echo "genmakemodel: genmake failed" 1>&2
367 edhill 1.21 cp genmake_* make.log $CDIR
368 edhill 1.10 return 1
369     else
370 edhill 1.20 echo "successful" 1>&2
371 edhill 1.1 fi
372 edhill 1.10 )
373     fi
374 edhill 1.1 }
375    
376     makeclean()
377     {
378     # makeclean directory
379 edhill 1.10 if test "x$NOCLEAN" = xt ; then
380 jmc 1.83 echo "make Clean skipped!"
381 edhill 1.10 else
382     (
383     cd $1;
384 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
385 edhill 1.10 if test -r Makefile ; then
386 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
387     $MAKE Clean >> make.log 2>&1
388 edhill 1.10 RETVAL=$?
389     if test "x$RETVAL" != x0 ; then
390     tail make.log
391 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
392 edhill 1.10 cp make.log $CDIR"/make.log"
393     return 1
394     fi
395     fi
396 edhill 1.20 echo successful 1>&2
397 edhill 1.10 exit 0
398     )
399     fi
400     }
401    
402 jmc 1.83 run_clean()
403 edhill 1.68 {
404 jmc 1.83 # run_clean directory
405     if test "x$NOCLEAN" = xt ; then
406     echo "run_clean skipped!"
407     else
408 edhill 1.68 (
409     cd $1;
410 jmc 1.83 printf 'clean run-dir ... ' 2>&1
411     # part of what is done after "make clean" when doing "make CLEAN"
412     find . -name "*.meta" -exec rm {} \;
413     find . -name "*.data" -exec rm {} \;
414     find . -name "fort.*" -exec rm {} \;
415     find . -type l -exec rm {} \;
416     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
417     rm -rf mnc_test_*
418     echo successful 1>&2
419 edhill 1.68 exit 0
420     )
421     fi
422     }
423    
424 edhill 1.10 makedependmodel()
425     {
426     # makedependmodel directory
427     if test "x$NODEPEND" = xt ; then
428     echo "make depend skipped!"
429     else
430     (
431     cd $1;
432     printf 'make depend ... ' 1>&2
433     $MAKE depend >> make.log 2>&1
434 edhill 1.1 RETVAL=$?
435     if test "x$RETVAL" != x0 ; then
436     tail make.log
437 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
438 edhill 1.1 cp make.log $CDIR"/make.log"
439     return 1
440 edhill 1.10 else
441 edhill 1.20 echo successful 1>&2
442 edhill 1.1 fi
443 edhill 1.10 )
444     fi
445 edhill 1.1 }
446    
447     makemodel()
448     {
449     # makemodel directory
450     (
451     cd $1;
452     if test -r Makefile ; then
453     printf 'make ... ' 1>&2
454 edhill 1.24 if test "x$ADM" = x ; then
455 edhill 1.43 if test "x$JOBS" = x ; then
456     $MAKE >> make.log 2>&1
457     else
458     $MAKE -j $JOBS >> make.log 2>&1
459     fi
460 edhill 1.24 else
461     $MAKE adall >> make.log 2>&1
462     fi
463 edhill 1.1 RETVAL=$?
464     if test "x$RETVAL" != x0 ; then
465     tail make.log
466     echo failed 1>&2
467     cp make.log $CDIR"/make.log"
468     return 1
469     else
470 edhill 1.20 echo successful 1>&2
471 edhill 1.1 fi
472     fi
473     )
474     }
475    
476 edhill 1.27 symlink_mpifiles()
477     {
478     # Put special links so that MPI specific files are used
479     # This MUST be invoked between makeclean and makelinks because
480     # the Makefile will link to non-mpi files by default
481    
482     dir=$1
483     code_dir=$2
484     BUILD_DIR=$dir/$3
485     CODE_DIR=$dir/$code_dir
486 edhill 1.45
487 edhill 1.27 # These are files that should replace their counter-part when using -mpi
488     MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")`
489    
490     # Is this an MPI run?
491     if test "x$MPI" = xt ; then
492     # YES: We symbolically link these files to the build
493     # dir so long as there is no real file in place
494     for ii in $MPI_FILES ; do
495     i=`echo $ii | sed 's:^\./::'`
496     name=`echo $i | sed 's:_mpi::' `
497     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
498     RETVAL=$?
499     if test "x$RETVAL" != x0 ; then
500     if ! test -f $BUILD_DIR/$i ; then
501 edhill 1.45 #echo Linking $name to $i
502 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
503     fi
504     fi
505     done
506     else
507 edhill 1.45 # NO: We undo any _mpi symbolically linked files
508 edhill 1.27 for ii in $MPI_FILES ; do
509     i=`echo $ii | sed 's:^\./::'`
510     name=`echo $i | sed 's:_mpi::' `
511     if test -L $BUILD_DIR/$name ; then
512 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
513     RETVAL=$?
514     if test "x$RETVAL" = x0 ; then
515     #echo Un-linking $name from $linktarg
516 edhill 1.27 rm -f $BUILD_DIR/$name
517     fi
518     fi
519     done
520     fi
521    
522     }
523    
524 edhill 1.1 linkdata()
525     {
526 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
527 edhill 1.1 #
528     # symbolically link data files to run directory
529 jmc 1.83 if test -d $1 ; then
530 edhill 1.12 (
531 jmc 1.83 cd $1 ; shift
532     if test -r "../"$1"/eedata.mth" ; then
533     # found eedata.mth in 1rst input dir and it is readable
534     if test "x$MULTI_THREAD" = "xt" ; then
535     # multi-threaded test: remove symbolic link & link eedata.mth
536     if test -h eedata ; then rm -f eedata ; fi
537     if test ! -r eedata ; then
538     ln -sf "../"$1"/eedata.mth" eedata ;
539     printf 'eedata.mth ' 1>&2
540 edhill 1.24 fi
541 jmc 1.83 else
542     # not multi-threaded test: remove eedata symbolic link
543     if test -h eedata ; then rm -f eedata ; fi
544     fi
545     fi
546     for ldir in $* ; do
547     if test -d "../"$ldir ; then
548     printf 'ldir='${ldir} 1>&2
549     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
550     for i in $files ; do
551     if test ! -d "../"$ldir/$i ; then
552     if test ! -r $i ; then
553     printf ' '$i 1>&2
554     ln -sf "../"$ldir"/"$i $i
555     fi
556     fi
557     done
558     if test -x "../"$ldir"/"prepare_run ; then
559     "../"$ldir"/"prepare_run
560 edhill 1.24 fi
561 jmc 1.83 printf ' ; ' 1>&2
562 jmc 1.82 fi
563 jmc 1.83 done
564 edhill 1.12 )
565 edhill 1.1 fi
566     }
567    
568     runmodel()
569     {
570 edhill 1.6 # runmodel directory
571 edhill 1.1 #
572 edhill 1.24 # runs "$COMMAND in "directory"
573 edhill 1.6 # (where "$COMMAND" is relative to "directory")
574 edhill 1.1 (
575     cd $1
576 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
577 edhill 1.6 # make output.txt
578 edhill 1.38 echo
579 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
580 jmc 1.88 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_00
581 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
582     fi
583     if test ! -x $EXECUTABLE ; then
584 jmc 1.88 rm -f run.log ; touch run.log
585     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
586 jmc 1.83 echo " no executable:" $EXECUTABLE >> run.log
587     RETVAL=8
588 jmc 1.88 ENDVAL=-1
589 jmc 1.75 else
590 jmc 1.98 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
591     # output do not exist or is older than executable:
592 jmc 1.88 rm -f run.log ; touch run.log
593     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
594 jmc 1.83 ( eval $COMMAND ) >> run.log 2>&1
595     RETVAL=$?
596     else
597     RETVAL=0
598 jmc 1.88 if test -f run.log ; then
599     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
600     echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1
601     else
602     touch run.log
603     if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi
604     echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1
605     echo " no previous run.log: assume NORMAL END" >> run.log 2>&1
606     fi
607 jmc 1.83 fi
608 jmc 1.88 ENDVAL=`cat run.log | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
609 jmc 1.75 fi
610 jmc 1.88 rm -f run.log_00
611     #if test "x$RETVAL" = x0 ; then
612     if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
613 jmc 1.46 tail run.log
614 edhill 1.20 echo successful 1>&2
615 edhill 1.44 # === Reduce the size of the testing emails!
616 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
617 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
618 edhill 1.6 return 0
619     else
620 edhill 1.20 tail run.log
621 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
622 edhill 1.20 cp run.log $CDIR"/run.log"
623 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
624 edhill 1.6 return 1
625 edhill 1.1 fi
626     )
627     }
628    
629     createcodelet()
630     {
631     # create codelet for comparing model output
632    
633 edhill 1.34 printf "creating the comparison code... "
634 jmc 1.91 cat > tr_cmpnum.c <<EOF
635 edhill 1.22 #include <stdio.h>
636     #include <math.h>
637     int main( int argc, char** argv ) {
638 jmc 1.91 int linnum,cmplin,best,lncnt;
639 edhill 1.52 double a,b,abave,relerr;
640 jmc 1.57 best = -22;
641 adcroft 1.33 lncnt = 0;
642 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
643 edhill 1.22 scanf("%d", &linnum);
644     if (linnum == -1) break;
645     scanf("%lf", &a); scanf("%lf", &b);
646 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
647 jmc 1.91 if ( abave == abave ) {
648     if (abave > 0.0) {
649     relerr=fabs(a-b)/abave;
650     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
651     else { cmplin = -16 ; }
652     best = (best > cmplin) ? best : cmplin; }
653     else { cmplin = -22 ; }
654     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
655     }
656     else {
657     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
658     break; }
659 edhill 1.22 }
660 adcroft 1.33 if (lncnt == 999) best=-29;
661 jmc 1.91 if (linnum != -1) best=-99;
662 edhill 1.22 printf("%d\n", -best);
663     return 0;
664     }
665     EOF
666 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
667 edhill 1.1
668 jmc 1.91 if [ -x ./tr_cmpnum ]; then
669 edhill 1.1 echo "OK"
670     return 0
671     else
672     echo
673 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
674     echo " a C compiler using the CC environment variable."
675 edhill 1.1 exit 1
676     fi
677     }
678    
679     formatresults()
680     {
681     # formatresults expt genmake depend make run results*
682    
683     nm=$1
684     printf '%s %s %s %s' $2 $3 $4 $5
685     shift; shift; shift; shift; shift;
686 jmc 1.94 listPrt=$@
687     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
688     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
689 jmc 1.97 printf '%3s' $listPrt
690 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
691 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
692 edhill 1.1
693 jmc 1.94 if [ $xx = '..' ]; then
694     printf ' N/O '
695     elif [ $xx = '--' ]; then
696     printf ' N/O '
697     elif [ $xx = 99 ]; then
698 edhill 1.1 printf ' N/O '
699     else
700 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
701 edhill 1.1 printf ' pass'
702     else
703     printf ' FAIL'
704     fi
705     fi
706     printf ' %s' $nm
707     printf '\n'
708    
709     }
710    
711     scandirs()
712     {
713 jmc 1.81 if [ $# -eq 1 ]; then
714     for arg in * ; do
715 jmc 1.99 test -f $arg/$1 && echo $arg
716 jmc 1.81 done
717 edhill 1.1 else
718 jmc 1.81 echo $*
719 edhill 1.1 fi
720     }
721    
722    
723     ###############################################################################
724    
725    
726     # Default properties
727     debug=0
728     verbose=1
729     clean=0
730     expts=''
731 edhill 1.6 # ieee=1
732 edhill 1.10
733 edhill 1.40 IEEE=true
734 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
735     IEEE=$MITGCM_IEEE
736     fi
737    
738    
739     CLEANUP=f
740     QUICK=f
741     NOGENMAKE=f
742     NOCLEAN=f
743     NODEPEND=f
744 edhill 1.68 POSTCLEAN=f
745 edhill 1.1
746 edhill 1.4 BASH=
747 edhill 1.10 OPTFILE=NONE
748     ADDRESSES=
749 edhill 1.1 TESTDIRS=
750 jmc 1.89 SKIPDIRS=
751 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
752 edhill 1.20 HAVE_MPACK=
753 edhill 1.1 MPACK="$MPACKDIR/mpack"
754 edhill 1.24 COMMAND=
755 edhill 1.59 if test "x$MAKE" = x ; then
756     MAKE=make
757     fi
758     if test "x$CC" = x ; then
759     CC=cc
760     fi
761 edhill 1.43 JOBS=
762 edhill 1.7 MPI=f
763 jmc 1.83 MULTI_THREAD=f
764 jmc 1.70 OUTDIR=
765 edhill 1.40 DELDIR=
766 edhill 1.1
767 edhill 1.24 ADM=
768    
769 edhill 1.49 # Additional monitor types
770 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
771 edhill 1.49
772 jmc 1.92 MATCH_CRIT=13
773    
774 edhill 1.34 printf "parsing options... "
775 edhill 1.1
776     ac_prev=
777     for ac_option ; do
778    
779     # If the previous option needs an argument, assign it.
780     if test -n "$ac_prev"; then
781     eval "$ac_prev=\$ac_option"
782     ac_prev=
783     continue
784     fi
785    
786     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
787    
788     case $ac_option in
789    
790     -help | --help | -h | --h)
791     usage ;;
792    
793 edhill 1.2 -optfile | --optfile | -of | --of)
794 edhill 1.10 ac_prev=OPTFILE ;;
795 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
796 edhill 1.10 OPTFILE=$ac_optarg ;;
797 edhill 1.1
798     -addr | --addr | -a | --a)
799     ac_prev=ADDRESSES ;;
800     -addr=* | --addr=*)
801     ADDRESSES=$ac_optarg ;;
802    
803     -tdir | --tdir | -t | --t)
804     ac_prev=TESTDIRS ;;
805     -tdir=* | --tdir=*)
806     TESTDIRS=$ac_optarg ;;
807 edhill 1.4
808 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
809     ac_prev=SKIPDIRS ;;
810     -skipdir=* | --skipdir=*)
811     SKIPDIRS=$ac_optarg ;;
812    
813 edhill 1.4 -bash | --bash | -b | --b)
814     ac_prev=BASH ;;
815     -bash=* | --bash=*)
816     BASH=$ac_optarg ;;
817 edhill 1.5
818 edhill 1.6 -command | --command | -c | --c)
819     ac_prev=COMMAND ;;
820     -command=* | --command=*)
821     COMMAND=$ac_optarg ;;
822 edhill 1.8
823     -make | --make | -m | --m)
824     ac_prev=MAKE ;;
825     -make=* | --make=*)
826     MAKE=$ac_optarg ;;
827 edhill 1.1
828 jmc 1.70 -odir | --odir)
829     ac_prev=OUTDIR ;;
830     -odir=* | --odir=*)
831     OUTDIR=$ac_optarg ;;
832    
833 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
834     ac_prev=PTRACERS_NUM ;;
835     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
836     PTRACERS_NUM=$ac_optarg ;;
837    
838 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
839     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
840    
841 edhill 1.43 -j) ac_prev=JOBS ;;
842     -j=*) JOBS=$ac_optarg ;;
843    
844 edhill 1.10 -clean | --clean)
845 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
846 edhill 1.10
847     -quick | --quick | -q | --q)
848     QUICK=t ;;
849     -nogenmake | --nogenmake | -ng | --ng)
850     NOGENMAKE=t ;;
851     -noclean | --noclean | -nc | --nc)
852     NOCLEAN=t ;;
853     -nodepend | --nodepend | -nd | --nd)
854     NODEPEND=t ;;
855    
856 edhill 1.68 -postclean | --postclean | -pc | --pc)
857     POSTCLEAN=t ;;
858    
859 edhill 1.18 -mpi) MPI=t ;;
860 edhill 1.10
861 jmc 1.83 -mth) MULTI_THREAD=t ;;
862    
863 edhill 1.24 -adm | -ad) ADM=t ;;
864    
865 edhill 1.10 -ieee) IEEE=true ;;
866     -noieee) IEEE= ;;
867    
868 edhill 1.1 -verbose) verbose=2 ;;
869     -debug) debug=1 ;;
870     -quiet) verbose=0 ;;
871    
872 edhill 1.40 -deldir | -dd) DELDIR=t ;;
873    
874 ce107 1.90 -ts) TS=t;;
875    
876     -papis) PAPIS=t;;
877    
878     -pcls) PCL=t;;
879    
880 edhill 1.1 -*)
881     echo "Error: unrecognized option: "$ac_option
882     usage
883     ;;
884    
885     *)
886     echo "Error: unrecognized argument: "$ac_option
887     usage
888     ;;
889    
890     esac
891    
892     done
893    
894 edhill 1.10 if test "x$QUICK" = xt ; then
895     NOGENMAKE=t
896     NOCLEAN=t
897     NODEPEND=t
898     fi
899    
900 jmc 1.99 #- setting for forward or ADM testing
901     if test "x$ADM" = xt ; then
902     code_dir=code_ad
903     input_dirs='input_ad input'
904     ref_outp="output_adm.txt"
905     EXECUTABLE="mitgcmuv_ad"
906     else
907     code_dir=code
908     input_dirs='input'
909     ref_outp="output.txt"
910     EXECUTABLE="mitgcmuv"
911     fi
912    
913 edhill 1.1 if test "x$TESTDIRS" = x ; then
914 jmc 1.100 # if test "x$ADM" = xt ; then
915     # LIST=`scandirs results_ad/$ref_outp`
916     # else
917 jmc 1.99 LIST=`scandirs results/$ref_outp`
918 jmc 1.100 # fi
919 jmc 1.84 else
920     #- expand group of experiments:
921     LIST=" "
922     for xx in $TESTDIRS
923     do
924     case $xx in
925 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
926     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
927     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
928 jmc 1.84 ;;
929     'tutorials')
930     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
931     *) LIST=${LIST}" "$xx ;;
932     esac
933     done
934 jmc 1.89 fi
935     #echo 'LIST='${LIST}'<'
936     #- skip dirs, remove duplicate and non-directory:
937     TESTDIRS=" "
938     count=0
939     for xx in $LIST
940     do
941     yy=`echo $SKIPDIRS | grep -c $xx`
942     if test $yy = 0 ; then
943 jmc 1.84 if test -d $xx ; then
944     yy=`echo $TESTDIRS | grep -c $xx`
945     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
946 jmc 1.89 else count=1 ;
947     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
948     fi
949     else
950     if test $count = 1 ; then echo -n ", \"$xx\""
951     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
952 jmc 1.84 fi
953 jmc 1.89 fi
954     done
955     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
956 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
957 edhill 1.1
958 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
959     OPTFILE=$MITGCM_OF
960     fi
961    
962 jmc 1.99 OUTPUTFILE=$ref_outp
963 jmc 1.87 if test "x$COMMAND" = x ; then
964 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
965 edhill 1.24 fi
966 jmc 1.87 if test "x$MPI" = xt ; then
967     OUTPUTFILE="STDOUT.0000"
968 edhill 1.24 fi
969    
970 jmc 1.76 #echo "OK"
971     echo "OK (COMMAND= $COMMAND )"
972 edhill 1.1
973 jmc 1.94 # set the Default List of output variables to be checked:
974     # (use default or load experiment-specific list from file "tr_checklist")
975     # content : 1rst = main variable used to decide if it pass or FAIL
976     # others = number of matching digits to be printed in summary.txt
977 jmc 1.99 if test "x$ADM" = x ; then
978 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
979     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
980 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
981 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
982 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
983 jmc 1.99 else
984     DEF_CHECK_LIST='Cost Cost Grad'
985     EMPTY_RESULTS='.. ..'
986     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
987 jmc 1.94 fi
988    
989 edhill 1.1 # create the FORTRAN comparison code
990     createcodelet
991    
992     # build the mpack utility
993 edhill 1.31 if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
994 edhill 1.32 echo "skipping mpack build"
995     else
996 edhill 1.31 build_mpack
997     fi
998 edhill 1.1
999     # Create a uniquely named directory to store results
1000     MACH=`hostname`
1001 edhill 1.2 UNAMEA=`uname -a`
1002 edhill 1.1 DATE=`date +%Y%m%d`
1003 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1004 jmc 1.70 if test "x$OUTDIR" != x ; then
1005     BASE="tr_"$OUTDIR"_"$DATE"_"
1006     fi
1007 edhill 1.1 DNUM=0
1008     DRESULTS="$BASE$DNUM"
1009     while test -e $DRESULTS ; do
1010     DNUM=$(( $DNUM + 1 ))
1011     DRESULTS="$BASE$DNUM"
1012     done
1013     mkdir $DRESULTS
1014     RETVAL=$?
1015     if test "x$RETVAL" != x0 ; then
1016 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1017 edhill 1.1 exit 1
1018     fi
1019     SUMMARY="$DRESULTS/summary.txt"
1020 edhill 1.34 printf "Start time: " >> $SUMMARY
1021 edhill 1.16 start_date=`date`
1022 edhill 1.17 echo $start_date > $SUMMARY
1023 edhill 1.1
1024 edhill 1.11 of_path=
1025 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1026     if test -r $OPTFILE ; then
1027 edhill 1.11 # get the path
1028     path=${OPTFILE%/*}
1029     if test "x$path" = x ; then
1030     of_path=`pwd`
1031     else
1032     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1033     fi
1034     file=${OPTFILE##*/}
1035     OPTFILE=$of_path/$file
1036 edhill 1.21 cp $OPTFILE $DRESULTS
1037     echo >> $SUMMARY
1038     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1039 edhill 1.11 else
1040 edhill 1.21 echo | tee $SUMMARY
1041     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1042     exit 1
1043 edhill 1.10 fi
1044 edhill 1.21 else
1045     echo >> $SUMMARY
1046     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1047     echo " so the genmake default will be used." >> $SUMMARY
1048 edhill 1.10 fi
1049     echo
1050     echo >> $SUMMARY
1051 edhill 1.24 if test "x$ADM" = x ; then
1052 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1053 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1054     else line_0="default "$MATCH_CRIT ; fi
1055     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1056 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1057 edhill 1.49 line_1="G D M c m s m s m s m s"
1058     line_2="E p a R g m m e . m m e . m m e . m m e ."
1059     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1060     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1061 edhill 1.50 for ii in $PTRACERS_NUM ; do
1062 edhill 1.49 # tst=`eval 'echo $HAVE_PTR0'$ii`
1063     # if test "x$tst" = xt ; then
1064     line_0="$line_0 --PTR 0"$ii"--"
1065     line_1="$line_1 m s"
1066     line_2="$line_2 m m e ."
1067     line_3="$line_3 i a a d"
1068     line_4="$line_4 n x n ."
1069     # fi
1070     done
1071     echo "$line_0" | tee -a $SUMMARY
1072     echo "$line_1" | tee -a $SUMMARY
1073     echo "$line_2" | tee -a $SUMMARY
1074     echo "$line_3" | tee -a $SUMMARY
1075     echo "$line_4" | tee -a $SUMMARY
1076     echo " " | tee -a $SUMMARY
1077 edhill 1.24 else
1078     echo "ADJOINT=true" >> $SUMMARY
1079     echo >> $SUMMARY
1080 jmc 1.100 if [ $MATCH_CRIT -lt 10 ] ;
1081     then line_0="default "$MATCH_CRIT ;
1082     else line_0="default "$MATCH_CRIT ; fi
1083     echo "$line_0" | tee -a $SUMMARY
1084 edhill 1.24 cat << EOF | tee -a $SUMMARY
1085     G D M C G
1086     E p a R o r
1087     N n k u s a
1088     2 d e n t d
1089    
1090     EOF
1091     fi
1092 edhill 1.1
1093 edhill 1.10 # ...and each test directory...
1094     for dir in $TESTDIRS ; do
1095    
1096     # Cleanup only!
1097     if test "x$CLEANUP" = xt ; then
1098     if test -r $dir/build/Makefile ; then
1099 jmc 1.83 echo ' ------ clean dir:' $dir/build
1100 edhill 1.10 ( cd $dir/build ; make CLEAN )
1101     fi
1102 jmc 1.83 if test -d $dir/run/CVS ; then
1103     echo ' ------ clean dir:' $dir/run
1104     run_clean $dir/run
1105 edhill 1.10 fi
1106 edhill 1.62 (
1107     cd $dir
1108     rm -rf tr_run.*
1109     )
1110 edhill 1.10 continue
1111 edhill 1.1 fi
1112 edhill 1.3
1113 edhill 1.10 # Verify that the testdir exists and contains previous
1114     # results in the correct location--or skip this directory!
1115 edhill 1.24 fout=
1116     if test "x$ADM" = x ; then
1117 jmc 1.99 fout=$dir"/results/"$ref_outp
1118 edhill 1.24 else
1119 jmc 1.99 fout=$dir"/results_ad/"$ref_outp
1120 edhill 1.24 fi
1121     if test ! -r $fout ; then
1122     echo "can't read \"$fout\" -- skipping $dir"
1123 edhill 1.10 continue
1124     fi
1125 edhill 1.53 if test "x$ADM" = x ; then
1126     check_for_add_mon_output $fout
1127     fi
1128 edhill 1.7
1129 edhill 1.49 # Check for additional types of monitor output
1130    
1131 jmc 1.83 builddir="build"
1132     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1133     rundir="run"
1134     if test ! -d $dir/$rundir ; then
1135     rundir=$builddir
1136 edhill 1.10 fi
1137 jmc 1.99 CODE_DIR=$dir/$code_dir
1138 edhill 1.10 BUILD_DIR=$dir/$builddir
1139 edhill 1.28
1140 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1141 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1142     continue
1143     fi
1144 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1145     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1146     continue
1147     fi
1148    
1149     # Check whether there are "extra runs" for this testdir
1150     extra_runs=
1151     if test "x$ADM" = x ; then
1152     ex_run_dirs=`( cd $dir ; echo input.* )`
1153     #echo "ex_run_dirs='$ex_run_dirs'"
1154     for exd in $ex_run_dirs ; do
1155     name=`echo $exd | sed -e 's/input.//g'`
1156 jmc 1.85 outf="$dir/results/output.$name.txt"
1157 jmc 1.83 if test -f $outf -a -r $outf ; then
1158     if test "x$MULTI_THREAD" = "xt" ; then
1159     if test -r $dir"/"$exd"/eedata.mth" ; then
1160     extra_runs="$extra_runs $name"
1161     #else echo $dir"/"$exd"/eedata.mth: not found"
1162     fi
1163     else
1164     extra_runs="$extra_runs $name"
1165     fi
1166     fi
1167     done
1168     fi
1169 edhill 1.28
1170     echo "-------------------------------------------------------------------------------"
1171     echo
1172 jmc 1.83 if test "x$extra_runs" = "x" ; then
1173     echo "Experiment: $dir"
1174     else
1175     echo "Experiment: $dir ; extra_runs=$extra_runs"
1176     fi
1177 edhill 1.28 echo
1178     unset genmake makedepend make run
1179 jmc 1.94 results=$EMPTY_RESULTS
1180 edhill 1.10
1181     # Create an output dir for each OPTFILE/tdir combination
1182 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1183     mkdir $rel_CDIR
1184     CDIR=`pwd`"/$rel_CDIR"
1185 edhill 1.10
1186     if test "x$CLEANUP" = xt ; then
1187 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1188     makeclean $dir/$builddir \
1189     && run_clean $dir/$rundir
1190 edhill 1.10 else
1191 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1192     && makeclean $dir/$builddir \
1193 jmc 1.83 && run_clean $dir/$rundir \
1194 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1195 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1196     && makemodel $dir/$builddir && make=Y \
1197 jmc 1.83 && linkdata $dir/$rundir $input_dirs \
1198 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1199 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1200 edhill 1.10 fi
1201    
1202     echo
1203 jmc 1.100 # if test "x$ADM" = x ; then
1204 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1205 edhill 1.34 echo
1206 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1207 jmc 1.100 touch $CDIR"/summary.txt"
1208 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1209 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1210     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1211     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1212     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1213    
1214     for ex in $extra_runs ; do
1215 jmc 1.73 unset run
1216 jmc 1.94 results=$EMPTY_RESULTS
1217 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1218     rel_CDIR=$DRESULTS"/"$dir"."$ex
1219     mkdir $rel_CDIR
1220     CDIR=`pwd`"/$rel_CDIR"
1221 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1222 jmc 1.83 run_clean $dir/tr_run.$ex
1223     linkdata $dir/tr_run.$ex input.$ex input
1224 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1225 jmc 1.99 && results=`testoutput_run $dir tr_run.$ex "output.${ex}.txt"`
1226 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1227     fres="$fres.$ex"
1228     echo
1229 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1230     touch $CDIR"/summary.txt"
1231 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1232 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1233     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1234     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1235 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1236 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1237 jmc 1.100 run_clean $dir/tr_run.$ex
1238 jmc 1.83 fi
1239 edhill 1.34 done
1240 jmc 1.100 # else
1241     # fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}`
1242     # fres=$fres"$results $dir"
1243     # echo
1244     # echo "$fres" >> $SUMMARY
1245     # touch $CDIR"/summary.txt"
1246     # echo "fresults='$fres'" >> $CDIR"/summary.txt"
1247     # echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1248     # echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1249     # echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1250     # echo "tdir='$dir'" >> $CDIR"/summary.txt"
1251     # fi
1252 jmc 1.99 if test "x$ADM" = xt ; then
1253 jmc 1.100 grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1254 jmc 1.77 >> $CDIR"/summary.txt"
1255 edhill 1.24 fi
1256 edhill 1.68
1257 jmc 1.83 #postclean $dir/$builddir
1258     if test "x$POSTCLEAN" = xt ; then
1259     makeclean $dir/$builddir \
1260     && run_clean $dir/$rundir
1261     fi
1262 edhill 1.10
1263     echo "-------------------------------------------------------------------------------"
1264    
1265 edhill 1.1 done
1266    
1267 edhill 1.34 printf "Start time: " >> $SUMMARY
1268 edhill 1.16 echo $start_date >> $SUMMARY
1269 edhill 1.34 printf "End time: " >> $SUMMARY
1270 edhill 1.13 date >> $SUMMARY
1271 edhill 1.20
1272     # If addresses were supplied and mpack built successfully, then try
1273     # to send email using mpack.
1274     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1275     echo "No results email was sent."
1276     else
1277     if test "x$HAVE_MPACK" = xt ; then
1278     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1279     && gzip $DRESULTS".tar" \
1280 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1281 edhill 1.20 RETVAL=$?
1282     if test "x$RETVAL" != x0 ; then
1283     echo
1284     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1285     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1286     echo " summary of results from the directory \"$DRESULTS\"."
1287     echo
1288     else
1289     echo
1290     echo "An email containing results was sent to the following addresses:"
1291     echo " \"$ADDRESSES\""
1292     echo
1293     fi
1294     test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1295     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1296     fi
1297     fi
1298 edhill 1.13
1299 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1300 edhill 1.1
1301 edhill 1.12 if test "x$CLEANUP" != xt ; then
1302 jmc 1.94 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \.//'
1303 edhill 1.25 if test -e tr_out.txt ; then
1304     mv tr_out.txt tr_out.txt.old
1305 edhill 1.14 fi
1306 jmc 1.98 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \.//g' > tr_out.txt
1307 edhill 1.12 fi
1308 edhill 1.1
1309 edhill 1.40 if test "x$DELDIR" = xt ; then
1310     rm -rf $DRESULTS
1311     fi
1312    

  ViewVC Help
Powered by ViewVC 1.1.22