/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.119 - (hide annotations) (download)
Mon Jan 26 04:09:34 2009 UTC (15 years, 2 months ago) by jmc
Branch: MAIN
CVS Tags: checkpoint61h
Changes since 1.118: +22 -21 lines
play with "run.log" file

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.119 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.118 2008/08/28 17:12:10 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.83 echo " (-mth) run multi threaded (using eedata.mth)"
15     echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.109 echo " (-gsl) compile with \"-gsl\" flag"
19 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
20 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
21 jmc 1.108 echo " (DEF=\"\" no email is sent)"
22 jmc 1.113 echo " (-mpd|-mpackdir) DIR location of the mpack utility"
23     echo " (DEF=\"../tools/mpack-1.6\")"
24 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
25     echo " (recognized groups: basic, tutorials)"
26     echo " (DEF=\"\" which test all)"
27 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
28     echo " (DEF=\"\" which test all)"
29 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
30     echo " Bourne-compatible \"sh\" shell"
31     echo " (DEF=\"\" for \"bash\")"
32 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
33 edhill 1.20 echo " (-command) STRING command to run"
34 edhill 1.6 echo " (DEF=\"make output.txt\")"
35 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
36 edhill 1.8 echo " (DEF=\"make\")"
37 jmc 1.70 echo " (-odir) STRING used to build output directory name"
38     echo " (DEF=\"hostname\")"
39 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
40 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
41 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
42     echo " (DEF=\"12\")"
43 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
44 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
45 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
46     echo " (-nogenmake|-ng) skip the genmake stage"
47     echo " (-noclean|-nc) skip the \"make clean\" stage"
48     echo " (-nodepend|-nd) skip the \"make depend\" stage"
49     echo " (-deldir|-dd) on success, delete the output directory"
50     echo " (-ts) provide timing information per timestep"
51     echo " (-papis) provide MFlop/s per timestep using PAPI"
52     echo " (-pcls) provide MFlop/s per timestep using PCL"
53 edhill 1.6 echo
54 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
55 edhill 1.6 echo "such as:"
56 edhill 1.50 echo
57 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
58     echo " -addr='abc@123.com testing@home.org'"
59 edhill 1.1 echo
60 edhill 1.50 echo "provided that the expression is properly quoted within the current"
61     echo "shell (note the use of single quotes to protect white space)."
62     echo
63 edhill 1.1 exit 1
64     }
65    
66     # build the mpack utility
67     build_mpack()
68     {
69 edhill 1.34 printf "building the mpack utility... "
70 jmc 1.113 MPACK="$MPACKDIR/mpack"
71     if test ! -x $MPACK ; then
72 edhill 1.1 if test ! -d $MPACKDIR ; then
73 edhill 1.20 echo
74 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
75     echo " are you sure this program is being run in the correct "
76     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
77 edhill 1.20 echo
78     HAVE_MPACK=f
79 edhill 1.1 fi
80 edhill 1.26 if test "x$CC" = x ; then
81     export CC=cc
82     fi
83 jmc 1.112 printf "building mpack (using CC=$CC)... "
84 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
85 edhill 1.1 RETVAL=$?
86     if test "x$RETVAL" != x0 ; then
87     echo
88     echo "Error building the mpack tools at: $MPACK_DIR"
89 edhill 1.20 echo
90     HAVE_MPACK=f
91     else
92 edhill 1.25 rm -f tr_build_mpack.out
93 edhill 1.20 HAVE_MPACK=t
94 jmc 1.113 echo "done"
95 edhill 1.1 fi
96 edhill 1.20 else
97     HAVE_MPACK=t
98 jmc 1.113 echo "already exist"
99 edhill 1.1 fi
100     }
101    
102 jmc 1.93 testoutput_var()
103 edhill 1.1 {
104 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
105 edhill 1.1 #
106 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
107 jmc 1.99 # with same output from reference file $dir/results/$reference_output
108 jmc 1.93 # using search strings s1 and text label
109 edhill 1.1
110     if [ $debug -gt 0 ]; then
111 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
112 edhill 1.1 fi
113 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
114     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
115 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
116 jmc 1.100 if [ $lncntA -lt 2 ]; then
117 edhill 1.1 if [ $verbose -gt 0 ]; then
118     echo Not enough lines of output when searching for "$2" 1>&2
119     fi
120     return 99
121     fi
122     else
123 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
124 edhill 1.1 return 99
125     fi
126     if [ $debug -gt 0 ]; then
127 jmc 1.99 echo testoutput_var: grep "$2" $1/results/$5 1>&2
128 edhill 1.1 fi
129 jmc 1.99 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
130 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
131 jmc 1.100 if [ $lncntB -lt 2 ]; then
132 edhill 1.1 if [ $verbose -gt 0 ]; then
133     echo Not enough lines of output when searching for "$2" 1>&2
134     fi
135     return 99
136     fi
137 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
138     if [ $verbose -gt 0 ]; then
139     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
140     fi
141     return 99
142     fi
143 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
144     if [ $has_nan -gt 0 ] ; then
145 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
146 edhill 1.72 return 99
147     fi
148     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
149     if [ $has_inf -gt 0 ] ; then
150 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
151 jmc 1.71 return 99
152     fi
153 edhill 1.1 if [ $debug -gt 0 ]; then
154 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
155 edhill 1.1 fi
156     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
157     if [ $debug -gt 0 ]; then
158 jmc 1.93 echo testoutput_var: compare_lines 1>&2
159 edhill 1.1 fi
160 edhill 1.22 if [ $verbose -gt 1 ]; then
161     cat tmp3.txt 1>&2
162     fi
163     echo "-1" >> tmp3.txt
164 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
165     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
166 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
167 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
168     if [ $verbose -gt 0 ]; then
169 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
170 edhill 1.1 fi
171     digits_of_similarity=99
172     else
173     if [ $verbose -gt 0 ]; then
174 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
175 edhill 1.1 fi
176     fi
177 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
178 edhill 1.1
179     return $digits_of_similarity
180     }
181    
182 jmc 1.93 testoutput_run()
183 edhill 1.1 {
184 jmc 1.99 # testoutput_run directory subdir reference_output
185 edhill 1.1 #
186 jmc 1.93 # test output from 1 run in "directory"
187 jmc 1.103 # --> same processing for adjoint & forward test
188 jmc 1.94 # default list of output variables to be checked:
189     # 1rst : main variable used to decide if it pass or FAIL
190     # others : number of matching digits to be printed in summary.txt
191     listChk=$DEF_CHECK_LIST
192     # load experiment-specific list from file "tr_checklist" (if it exist)
193     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
194     sVar=`echo $listChk | awk '{print $1}'`
195     # remove 1rst var and expand the list: + => min max mean s.d
196 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
197     | sed 's/+//g' | sed "s/^$sVar//"`
198 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
199 jmc 1.117 # check for ptracer output in reference_output file :
200     outpref=$1/results/$3
201     ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
202     ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
203     for ii in $PTRACERS_NUM ; do
204     ptrfound=0
205     for jj in $ptr_mon ; do
206     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
207     tst=`grep $name $outpref | wc -l | awk '{print $1}'`
208     if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
209     done
210     if test $ptrfound = '1' ; then
211     eval "HAVE_PTR0"$ii"=t"
212     else
213     eval "HAVE_PTR0"$ii"=f"
214     if test "x$ADM" = x ; then
215     # remove this ptr from the list of output variable to check
216     # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
217     listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
218     fi
219     fi
220     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
221     done
222 mlosch 1.116 tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
223 jmc 1.94 if test $tst != 1 ; then
224     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
225     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
226     echo "==> WARNING: in checked list:" $listVar 1>&2
227     #- put it back once:
228     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
229     fi
230     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
231     echo "listVar='$listVar'" > $CDIR"/summary.txt"
232     allargs=""
233     for xx in $listVar
234     do
235     case $xx in
236     'PS') if [ $debug -gt 0 ]
237     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
238 jmc 1.99 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
239 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
240 jmc 1.101 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
241     'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
242 jmc 1.99 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
243     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
244     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
245     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
246     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
247     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
248     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
249     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
250 jmc 1.94 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
251     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
252     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
253 jmc 1.99 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
254 jmc 1.94 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
255     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
256     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
257 jmc 1.99 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
258 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
259     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
260     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
261     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
262     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
263     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
264     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
265 jmc 1.99 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
266 jmc 1.101 'Qntmn') testoutput_var $1 "extforcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;;
267     'Qntmx') testoutput_var $1 "extforcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;;
268     'Qntav') testoutput_var $1 "extforcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;;
269     'Qntsd') testoutput_var $1 "extforcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;;
270     'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;;
271     'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;;
272     'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;;
273     'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;;
274     'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;;
275     'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Hell max" $2 $3; yy=$? ;;
276     'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Hell mean" $2 $3; yy=$? ;;
277     'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Hell StDv" $2 $3; yy=$? ;;
278     'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
279     'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;;
280     'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;;
281     'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;;
282     'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;;
283     'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;;
284 jmc 1.94 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
285     esac
286     if test $xx = $sVar
287     then allargs="$allargs > $yy <"
288     else allargs="$allargs $yy"
289     fi
290     done
291    
292     nbVar=`echo $listVar | awk '{print NF}'`
293 jmc 1.97 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
294 jmc 1.94 #-- fill line (up to standard length) with dot:
295 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
296 jmc 1.94 echo $allargs $adNul
297     else
298     echo $allargs
299     fi
300 jmc 1.103 # <-- same processing for adjoint & forward test
301 edhill 1.1 }
302    
303     genmakemodel()
304     {
305     # genmakemodel directory
306 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
307     echo "genmake skipped!"
308     else
309 edhill 1.34 if test "x$BASH" = x ; then
310     GENMAKE2="../../../tools/genmake2"
311     else
312     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
313     fi
314 edhill 1.10 (
315     cd $1;
316 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
317     if test "x$ADM" = x ; then
318     command="$command --mods=../code"
319     else
320     command="$command --mods=../code_ad"
321     fi
322 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
323     command="$command --optfile=$OPTFILE"
324     fi
325     if test "x$IEEE" != x ; then
326     command="$command -ieee"
327     fi
328 jmc 1.109 if test "x$GSL" = xt ; then
329     command="$command -gsl"
330     fi
331 edhill 1.66 if test "x$MPI" = xt ; then
332     command="$command -mpi"
333     fi
334 ce107 1.90 if test "x$TS" = xt ; then
335     command="$command -ts"
336     fi
337     if test "x$PAPIS" = xt ; then
338     command="$command -papis"
339     else
340     if test "x$PCLS" = xt ; then
341     command="$command -pcls"
342     fi
343     fi
344 edhill 1.10 printf 'genmake ... ' 1>&2
345     $command > make.log 2>&1
346     RETVAL=$?
347 edhill 1.44 # Reduce the size of the testing emails!
348 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
349 edhill 1.10 if test "x$RETVAL" != x0 ; then
350     tail make.log
351     echo "genmakemodel: genmake failed" 1>&2
352 edhill 1.21 cp genmake_* make.log $CDIR
353 edhill 1.10 return 1
354     else
355 edhill 1.20 echo "successful" 1>&2
356 edhill 1.1 fi
357 edhill 1.10 )
358     fi
359 edhill 1.1 }
360    
361     makeclean()
362     {
363     # makeclean directory
364 edhill 1.10 if test "x$NOCLEAN" = xt ; then
365 jmc 1.83 echo "make Clean skipped!"
366 edhill 1.10 else
367     (
368     cd $1;
369 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
370 edhill 1.10 if test -r Makefile ; then
371 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
372     $MAKE Clean >> make.log 2>&1
373 edhill 1.10 RETVAL=$?
374     if test "x$RETVAL" != x0 ; then
375     tail make.log
376 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
377 edhill 1.10 cp make.log $CDIR"/make.log"
378     return 1
379     fi
380     fi
381 edhill 1.20 echo successful 1>&2
382 edhill 1.10 exit 0
383     )
384     fi
385     }
386    
387 jmc 1.83 run_clean()
388 edhill 1.68 {
389 jmc 1.83 # run_clean directory
390     if test "x$NOCLEAN" = xt ; then
391     echo "run_clean skipped!"
392     else
393 edhill 1.68 (
394     cd $1;
395 jmc 1.83 printf 'clean run-dir ... ' 2>&1
396     # part of what is done after "make clean" when doing "make CLEAN"
397     find . -name "*.meta" -exec rm {} \;
398     find . -name "*.data" -exec rm {} \;
399     find . -name "fort.*" -exec rm {} \;
400     find . -type l -exec rm {} \;
401     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
402     rm -rf mnc_test_*
403 jmc 1.107 rm -f *_MIT_CE_000.opt0000 costfunction*0000
404 jmc 1.83 echo successful 1>&2
405 edhill 1.68 exit 0
406     )
407     fi
408     }
409    
410 edhill 1.10 makedependmodel()
411     {
412     # makedependmodel directory
413     if test "x$NODEPEND" = xt ; then
414     echo "make depend skipped!"
415     else
416     (
417     cd $1;
418     printf 'make depend ... ' 1>&2
419     $MAKE depend >> make.log 2>&1
420 edhill 1.1 RETVAL=$?
421     if test "x$RETVAL" != x0 ; then
422     tail make.log
423 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
424 edhill 1.1 cp make.log $CDIR"/make.log"
425     return 1
426 edhill 1.10 else
427 edhill 1.20 echo successful 1>&2
428 edhill 1.1 fi
429 edhill 1.10 )
430     fi
431 edhill 1.1 }
432    
433     makemodel()
434     {
435     # makemodel directory
436     (
437     cd $1;
438     if test -r Makefile ; then
439     printf 'make ... ' 1>&2
440 edhill 1.24 if test "x$ADM" = x ; then
441 edhill 1.43 if test "x$JOBS" = x ; then
442     $MAKE >> make.log 2>&1
443     else
444     $MAKE -j $JOBS >> make.log 2>&1
445     fi
446 edhill 1.24 else
447     $MAKE adall >> make.log 2>&1
448     fi
449 edhill 1.1 RETVAL=$?
450     if test "x$RETVAL" != x0 ; then
451     tail make.log
452     echo failed 1>&2
453     cp make.log $CDIR"/make.log"
454     return 1
455     else
456 edhill 1.20 echo successful 1>&2
457 edhill 1.1 fi
458     fi
459     )
460     }
461    
462 edhill 1.27 symlink_mpifiles()
463     {
464     # Put special links so that MPI specific files are used
465     # This MUST be invoked between makeclean and makelinks because
466     # the Makefile will link to non-mpi files by default
467    
468     dir=$1
469     code_dir=$2
470     BUILD_DIR=$dir/$3
471     CODE_DIR=$dir/$code_dir
472 edhill 1.45
473 edhill 1.27 # These are files that should replace their counter-part when using -mpi
474 mlosch 1.111 MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`
475 edhill 1.27
476     # Is this an MPI run?
477     if test "x$MPI" = xt ; then
478     # YES: We symbolically link these files to the build
479     # dir so long as there is no real file in place
480     for ii in $MPI_FILES ; do
481     i=`echo $ii | sed 's:^\./::'`
482     name=`echo $i | sed 's:_mpi::' `
483     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
484     RETVAL=$?
485     if test "x$RETVAL" != x0 ; then
486     if ! test -f $BUILD_DIR/$i ; then
487 edhill 1.45 #echo Linking $name to $i
488 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
489     fi
490     fi
491     done
492     else
493 edhill 1.45 # NO: We undo any _mpi symbolically linked files
494 edhill 1.27 for ii in $MPI_FILES ; do
495     i=`echo $ii | sed 's:^\./::'`
496     name=`echo $i | sed 's:_mpi::' `
497     if test -L $BUILD_DIR/$name ; then
498 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
499     RETVAL=$?
500     if test "x$RETVAL" = x0 ; then
501     #echo Un-linking $name from $linktarg
502 edhill 1.27 rm -f $BUILD_DIR/$name
503     fi
504     fi
505     done
506     fi
507    
508     }
509    
510 edhill 1.1 linkdata()
511     {
512 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
513 edhill 1.1 #
514     # symbolically link data files to run directory
515 jmc 1.83 if test -d $1 ; then
516 edhill 1.12 (
517 jmc 1.83 cd $1 ; shift
518     if test -r "../"$1"/eedata.mth" ; then
519     # found eedata.mth in 1rst input dir and it is readable
520     if test "x$MULTI_THREAD" = "xt" ; then
521     # multi-threaded test: remove symbolic link & link eedata.mth
522     if test -h eedata ; then rm -f eedata ; fi
523     if test ! -r eedata ; then
524     ln -sf "../"$1"/eedata.mth" eedata ;
525     printf 'eedata.mth ' 1>&2
526 edhill 1.24 fi
527 jmc 1.83 else
528     # not multi-threaded test: remove eedata symbolic link
529     if test -h eedata ; then rm -f eedata ; fi
530     fi
531     fi
532 jmc 1.108 prevDir='NONE'
533 jmc 1.83 for ldir in $* ; do
534 jmc 1.108 if test -d "../"$ldir -a $ldir != $prevDir ; then
535 jmc 1.83 printf 'ldir='${ldir} 1>&2
536     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
537     for i in $files ; do
538     if test ! -d "../"$ldir/$i ; then
539     if test ! -r $i ; then
540     printf ' '$i 1>&2
541     ln -sf "../"$ldir"/"$i $i
542     fi
543     fi
544     done
545     if test -x "../"$ldir"/"prepare_run ; then
546     "../"$ldir"/"prepare_run
547 edhill 1.24 fi
548 jmc 1.83 printf ' ; ' 1>&2
549 jmc 1.82 fi
550 jmc 1.108 prevDir=$ldir
551 jmc 1.83 done
552 edhill 1.12 )
553 edhill 1.1 fi
554     }
555    
556     runmodel()
557     {
558 edhill 1.6 # runmodel directory
559 edhill 1.1 #
560 edhill 1.24 # runs "$COMMAND in "directory"
561 edhill 1.6 # (where "$COMMAND" is relative to "directory")
562 edhill 1.1 (
563     cd $1
564 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
565 edhill 1.6 # make output.txt
566 edhill 1.38 echo
567 jmc 1.106 if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
568     diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
569 jmc 1.119 outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi
570 jmc 1.106 fi
571 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
572 jmc 1.119 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
573 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
574     fi
575     if test ! -x $EXECUTABLE ; then
576 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
577     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
578     echo " no executable:" $EXECUTABLE >> $RUNLOG
579 jmc 1.83 RETVAL=8
580 jmc 1.88 ENDVAL=-1
581 jmc 1.75 else
582 jmc 1.98 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
583     # output do not exist or is older than executable:
584 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
585     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
586     ( eval $COMMAND ) >> $RUNLOG 2>&1
587 jmc 1.83 RETVAL=$?
588     else
589     RETVAL=0
590 jmc 1.119 if test -f $RUNLOG ; then
591     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
592     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
593 jmc 1.88 else
594 jmc 1.119 touch $RUNLOG
595     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
596     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
597     echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1
598 jmc 1.88 fi
599 jmc 1.83 fi
600 jmc 1.119 ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
601 jmc 1.75 fi
602 jmc 1.119 rm -f run.log_tmp
603 jmc 1.88 #if test "x$RETVAL" = x0 ; then
604     if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
605 jmc 1.119 tail $RUNLOG
606 edhill 1.20 echo successful 1>&2
607 edhill 1.44 # === Reduce the size of the testing emails!
608 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
609 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
610 edhill 1.6 return 0
611     else
612 jmc 1.119 tail $RUNLOG
613 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
614 jmc 1.119 cp $RUNLOG $CDIR"/"$RUNLOG
615 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
616 edhill 1.6 return 1
617 edhill 1.1 fi
618     )
619     }
620    
621     createcodelet()
622     {
623     # create codelet for comparing model output
624    
625 jmc 1.112 printf "creating the comparison code (using CC=$CC)... "
626 jmc 1.91 cat > tr_cmpnum.c <<EOF
627 edhill 1.22 #include <stdio.h>
628     #include <math.h>
629     int main( int argc, char** argv ) {
630 jmc 1.91 int linnum,cmplin,best,lncnt;
631 edhill 1.52 double a,b,abave,relerr;
632 jmc 1.57 best = -22;
633 adcroft 1.33 lncnt = 0;
634 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
635 edhill 1.22 scanf("%d", &linnum);
636     if (linnum == -1) break;
637     scanf("%lf", &a); scanf("%lf", &b);
638 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
639 jmc 1.91 if ( abave == abave ) {
640     if (abave > 0.0) {
641     relerr=fabs(a-b)/abave;
642     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
643     else { cmplin = -16 ; }
644     best = (best > cmplin) ? best : cmplin; }
645     else { cmplin = -22 ; }
646     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
647     }
648     else {
649     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
650     break; }
651 edhill 1.22 }
652 adcroft 1.33 if (lncnt == 999) best=-29;
653 jmc 1.91 if (linnum != -1) best=-99;
654 edhill 1.22 printf("%d\n", -best);
655     return 0;
656     }
657     EOF
658 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
659 edhill 1.1
660 jmc 1.91 if [ -x ./tr_cmpnum ]; then
661 edhill 1.1 echo "OK"
662     return 0
663     else
664     echo
665 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
666     echo " a C compiler using the CC environment variable."
667 edhill 1.1 exit 1
668     fi
669     }
670    
671     formatresults()
672     {
673     # formatresults expt genmake depend make run results*
674    
675     nm=$1
676     printf '%s %s %s %s' $2 $3 $4 $5
677     shift; shift; shift; shift; shift;
678 jmc 1.94 listPrt=$@
679     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
680     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
681 jmc 1.97 printf '%3s' $listPrt
682 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
683 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
684 edhill 1.1
685 jmc 1.94 if [ $xx = '..' ]; then
686     printf ' N/O '
687     elif [ $xx = '--' ]; then
688     printf ' N/O '
689     elif [ $xx = 99 ]; then
690 edhill 1.1 printf ' N/O '
691     else
692 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
693 edhill 1.1 printf ' pass'
694     else
695     printf ' FAIL'
696     fi
697     fi
698     printf ' %s' $nm
699     printf '\n'
700    
701     }
702    
703     scandirs()
704     {
705 jmc 1.81 if [ $# -eq 1 ]; then
706     for arg in * ; do
707 jmc 1.99 test -f $arg/$1 && echo $arg
708 jmc 1.81 done
709 edhill 1.1 else
710 jmc 1.81 echo $*
711 edhill 1.1 fi
712     }
713    
714    
715     ###############################################################################
716    
717    
718     # Default properties
719     debug=0
720     verbose=1
721     clean=0
722 edhill 1.10
723 edhill 1.40 IEEE=true
724 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
725     IEEE=$MITGCM_IEEE
726     fi
727 jmc 1.109 GSL=f
728 edhill 1.10
729     CLEANUP=f
730     QUICK=f
731     NOGENMAKE=f
732     NOCLEAN=f
733     NODEPEND=f
734 edhill 1.68 POSTCLEAN=f
735 edhill 1.1
736 edhill 1.4 BASH=
737 edhill 1.10 OPTFILE=NONE
738     ADDRESSES=
739 edhill 1.1 TESTDIRS=
740 jmc 1.89 SKIPDIRS=
741 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
742 edhill 1.20 HAVE_MPACK=
743 jmc 1.113 MPACK=
744 edhill 1.24 COMMAND=
745 edhill 1.59 if test "x$MAKE" = x ; then
746     MAKE=make
747     fi
748     if test "x$CC" = x ; then
749     CC=cc
750     fi
751 edhill 1.43 JOBS=
752 edhill 1.7 MPI=f
753 jmc 1.83 MULTI_THREAD=f
754 jmc 1.70 OUTDIR=
755 edhill 1.40 DELDIR=
756 edhill 1.1
757 edhill 1.24 ADM=
758    
759 jmc 1.117 # list of pTracers to check for monitor output
760 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
761 edhill 1.49
762 jmc 1.92 MATCH_CRIT=13
763    
764 edhill 1.34 printf "parsing options... "
765 edhill 1.1
766     ac_prev=
767     for ac_option ; do
768    
769     # If the previous option needs an argument, assign it.
770     if test -n "$ac_prev"; then
771     eval "$ac_prev=\$ac_option"
772     ac_prev=
773     continue
774     fi
775    
776     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
777    
778     case $ac_option in
779    
780     -help | --help | -h | --h)
781     usage ;;
782    
783 edhill 1.2 -optfile | --optfile | -of | --of)
784 edhill 1.10 ac_prev=OPTFILE ;;
785 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
786 edhill 1.10 OPTFILE=$ac_optarg ;;
787 edhill 1.1
788     -addr | --addr | -a | --a)
789     ac_prev=ADDRESSES ;;
790     -addr=* | --addr=*)
791     ADDRESSES=$ac_optarg ;;
792 jmc 1.113 -mpackdir | --mpackdir | -mpd | --mpd)
793     ac_prev=MPACKDIR ;;
794     -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
795     MPACKDIR=$ac_optarg ;;
796 edhill 1.1
797     -tdir | --tdir | -t | --t)
798     ac_prev=TESTDIRS ;;
799     -tdir=* | --tdir=*)
800     TESTDIRS=$ac_optarg ;;
801 edhill 1.4
802 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
803     ac_prev=SKIPDIRS ;;
804     -skipdir=* | --skipdir=*)
805     SKIPDIRS=$ac_optarg ;;
806    
807 edhill 1.4 -bash | --bash | -b | --b)
808     ac_prev=BASH ;;
809     -bash=* | --bash=*)
810     BASH=$ac_optarg ;;
811 edhill 1.5
812 edhill 1.6 -command | --command | -c | --c)
813     ac_prev=COMMAND ;;
814     -command=* | --command=*)
815     COMMAND=$ac_optarg ;;
816 edhill 1.8
817     -make | --make | -m | --m)
818     ac_prev=MAKE ;;
819     -make=* | --make=*)
820     MAKE=$ac_optarg ;;
821 edhill 1.1
822 jmc 1.70 -odir | --odir)
823     ac_prev=OUTDIR ;;
824     -odir=* | --odir=*)
825     OUTDIR=$ac_optarg ;;
826    
827 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
828     ac_prev=PTRACERS_NUM ;;
829     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
830     PTRACERS_NUM=$ac_optarg ;;
831    
832 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
833     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
834    
835 edhill 1.43 -j) ac_prev=JOBS ;;
836     -j=*) JOBS=$ac_optarg ;;
837    
838 edhill 1.10 -clean | --clean)
839 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
840 edhill 1.10
841     -quick | --quick | -q | --q)
842     QUICK=t ;;
843     -nogenmake | --nogenmake | -ng | --ng)
844     NOGENMAKE=t ;;
845     -noclean | --noclean | -nc | --nc)
846     NOCLEAN=t ;;
847     -nodepend | --nodepend | -nd | --nd)
848     NODEPEND=t ;;
849    
850 edhill 1.68 -postclean | --postclean | -pc | --pc)
851     POSTCLEAN=t ;;
852    
853 edhill 1.18 -mpi) MPI=t ;;
854 edhill 1.10
855 jmc 1.83 -mth) MULTI_THREAD=t ;;
856    
857 edhill 1.24 -adm | -ad) ADM=t ;;
858    
859 edhill 1.10 -ieee) IEEE=true ;;
860     -noieee) IEEE= ;;
861 jmc 1.109 -gsl) GSL=t ;;
862 edhill 1.10
863 edhill 1.1 -verbose) verbose=2 ;;
864     -debug) debug=1 ;;
865     -quiet) verbose=0 ;;
866    
867 edhill 1.40 -deldir | -dd) DELDIR=t ;;
868    
869 ce107 1.90 -ts) TS=t;;
870    
871     -papis) PAPIS=t;;
872    
873     -pcls) PCL=t;;
874    
875 edhill 1.1 -*)
876     echo "Error: unrecognized option: "$ac_option
877     usage
878     ;;
879    
880     *)
881     echo "Error: unrecognized argument: "$ac_option
882     usage
883     ;;
884    
885     esac
886    
887     done
888    
889 edhill 1.10 if test "x$QUICK" = xt ; then
890     NOGENMAKE=t
891     NOCLEAN=t
892     NODEPEND=t
893     fi
894    
895 jmc 1.99 #- setting for forward or ADM testing
896     if test "x$ADM" = xt ; then
897     code_dir=code_ad
898 jmc 1.108 inputdir=input_ad
899 jmc 1.99 ref_outp="output_adm.txt"
900     EXECUTABLE="mitgcmuv_ad"
901     else
902     code_dir=code
903 jmc 1.108 inputdir=input
904 jmc 1.99 ref_outp="output.txt"
905     EXECUTABLE="mitgcmuv"
906     fi
907    
908 edhill 1.1 if test "x$TESTDIRS" = x ; then
909 jmc 1.103 LIST=`scandirs results/$ref_outp`
910 jmc 1.84 else
911     #- expand group of experiments:
912     LIST=" "
913     for xx in $TESTDIRS
914     do
915     case $xx in
916 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
917     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
918     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
919 jmc 1.84 ;;
920     'tutorials')
921     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
922     *) LIST=${LIST}" "$xx ;;
923     esac
924     done
925 jmc 1.89 fi
926     #echo 'LIST='${LIST}'<'
927     #- skip dirs, remove duplicate and non-directory:
928     TESTDIRS=" "
929     count=0
930     for xx in $LIST
931     do
932     yy=`echo $SKIPDIRS | grep -c $xx`
933     if test $yy = 0 ; then
934 jmc 1.84 if test -d $xx ; then
935     yy=`echo $TESTDIRS | grep -c $xx`
936     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
937 jmc 1.89 else count=1 ;
938     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
939     fi
940     else
941     if test $count = 1 ; then echo -n ", \"$xx\""
942     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
943 jmc 1.84 fi
944 jmc 1.89 fi
945     done
946     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
947 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
948 edhill 1.1
949 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
950     OPTFILE=$MITGCM_OF
951     fi
952    
953 jmc 1.119 RUNLOG="run.log"
954 jmc 1.99 OUTPUTFILE=$ref_outp
955 jmc 1.87 if test "x$COMMAND" = x ; then
956 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
957 edhill 1.24 fi
958 jmc 1.87 if test "x$MPI" = xt ; then
959     OUTPUTFILE="STDOUT.0000"
960 edhill 1.24 fi
961    
962 jmc 1.76 echo "OK (COMMAND= $COMMAND )"
963 edhill 1.1
964 jmc 1.94 # set the Default List of output variables to be checked:
965     # (use default or load experiment-specific list from file "tr_checklist")
966     # content : 1rst = main variable used to decide if it pass or FAIL
967     # others = number of matching digits to be printed in summary.txt
968 jmc 1.99 if test "x$ADM" = x ; then
969 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
970     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
971 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
972 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
973 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
974 jmc 1.99 else
975 jmc 1.103 DEF_CHECK_LIST='Grad Cost Grad'
976 jmc 1.99 EMPTY_RESULTS='.. ..'
977     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
978 jmc 1.94 fi
979    
980 edhill 1.1 # create the FORTRAN comparison code
981     createcodelet
982    
983 jmc 1.112 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
984     if test "x$ADDRESSES" = x ; then
985 edhill 1.32 echo "skipping mpack build"
986     else
987 edhill 1.31 build_mpack
988     fi
989 edhill 1.1
990     # Create a uniquely named directory to store results
991 jmc 1.110 CMDLINE=$0
992 jmc 1.118 for xx in "$@" ; do nw=`echo $xx | wc -w`
993     if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
994     else CMDLINE="$CMDLINE '$xx'" ; fi
995     done
996     #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
997 edhill 1.1 MACH=`hostname`
998 edhill 1.2 UNAMEA=`uname -a`
999 edhill 1.1 DATE=`date +%Y%m%d`
1000 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1001 jmc 1.70 if test "x$OUTDIR" != x ; then
1002     BASE="tr_"$OUTDIR"_"$DATE"_"
1003     fi
1004 edhill 1.1 DNUM=0
1005     DRESULTS="$BASE$DNUM"
1006     while test -e $DRESULTS ; do
1007     DNUM=$(( $DNUM + 1 ))
1008     DRESULTS="$BASE$DNUM"
1009     done
1010     mkdir $DRESULTS
1011     RETVAL=$?
1012     if test "x$RETVAL" != x0 ; then
1013 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1014 edhill 1.1 exit 1
1015     fi
1016     SUMMARY="$DRESULTS/summary.txt"
1017 edhill 1.16 start_date=`date`
1018 edhill 1.17 echo $start_date > $SUMMARY
1019 jmc 1.110 echo 'run:' $CMDLINE >> $SUMMARY
1020     echo 'on :' $UNAMEA >> $SUMMARY
1021 edhill 1.1
1022 edhill 1.11 of_path=
1023 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1024     if test -r $OPTFILE ; then
1025 edhill 1.11 # get the path
1026     path=${OPTFILE%/*}
1027     if test "x$path" = x ; then
1028     of_path=`pwd`
1029     else
1030     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1031     fi
1032     file=${OPTFILE##*/}
1033     OPTFILE=$of_path/$file
1034 edhill 1.21 cp $OPTFILE $DRESULTS
1035     echo >> $SUMMARY
1036     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1037 edhill 1.11 else
1038 edhill 1.21 echo | tee $SUMMARY
1039     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1040     exit 1
1041 edhill 1.10 fi
1042 edhill 1.21 else
1043     echo >> $SUMMARY
1044     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1045     echo " so the genmake default will be used." >> $SUMMARY
1046 edhill 1.10 fi
1047     echo
1048     echo >> $SUMMARY
1049 edhill 1.24 if test "x$ADM" = x ; then
1050 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1051 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1052     else line_0="default "$MATCH_CRIT ; fi
1053     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1054 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1055 edhill 1.49 line_1="G D M c m s m s m s m s"
1056     line_2="E p a R g m m e . m m e . m m e . m m e ."
1057     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1058     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1059 edhill 1.50 for ii in $PTRACERS_NUM ; do
1060 edhill 1.49 line_0="$line_0 --PTR 0"$ii"--"
1061     line_1="$line_1 m s"
1062     line_2="$line_2 m m e ."
1063     line_3="$line_3 i a a d"
1064     line_4="$line_4 n x n ."
1065     done
1066     echo "$line_0" | tee -a $SUMMARY
1067     echo "$line_1" | tee -a $SUMMARY
1068     echo "$line_2" | tee -a $SUMMARY
1069     echo "$line_3" | tee -a $SUMMARY
1070     echo "$line_4" | tee -a $SUMMARY
1071     echo " " | tee -a $SUMMARY
1072 edhill 1.24 else
1073     echo "ADJOINT=true" >> $SUMMARY
1074     echo >> $SUMMARY
1075 jmc 1.100 if [ $MATCH_CRIT -lt 10 ] ;
1076 jmc 1.103 then line_0="default "$MATCH_CRIT ;
1077     else line_0="default "$MATCH_CRIT ; fi
1078 jmc 1.100 echo "$line_0" | tee -a $SUMMARY
1079 edhill 1.24 cat << EOF | tee -a $SUMMARY
1080     G D M C G
1081     E p a R o r
1082     N n k u s a
1083     2 d e n t d
1084    
1085     EOF
1086     fi
1087 edhill 1.1
1088 edhill 1.10 # ...and each test directory...
1089     for dir in $TESTDIRS ; do
1090    
1091 jmc 1.106 # set builddir & rundir:
1092     builddir="build"
1093     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1094     rundir="run"
1095     if test ! -d $dir/$rundir ; then
1096     rundir=$builddir
1097     fi
1098     CODE_DIR=$dir/$code_dir
1099     BUILD_DIR=$dir/$builddir
1100    
1101 edhill 1.10 # Cleanup only!
1102     if test "x$CLEANUP" = xt ; then
1103 jmc 1.106 if test -r $BUILD_DIR/Makefile ; then
1104 jmc 1.115 echo ' ------ clean dir:' $dir/$builddir
1105 jmc 1.106 ( cd $BUILD_DIR ; make CLEAN )
1106 edhill 1.10 fi
1107 jmc 1.106 if test -d $dir/$rundir/CVS ; then
1108     echo ' ------ clean dir:' $dir/$rundir
1109     run_clean $dir/$rundir
1110 edhill 1.10 fi
1111 edhill 1.62 (
1112     cd $dir
1113     rm -rf tr_run.*
1114     )
1115 edhill 1.10 continue
1116 edhill 1.1 fi
1117 edhill 1.3
1118 edhill 1.10 # Verify that the testdir exists and contains previous
1119     # results in the correct location--or skip this directory!
1120 jmc 1.104 fout=$dir"/results/"$ref_outp
1121 edhill 1.24 if test ! -r $fout ; then
1122     echo "can't read \"$fout\" -- skipping $dir"
1123 edhill 1.10 continue
1124     fi
1125 edhill 1.7
1126 jmc 1.117 # Check for specific files for particular type of run
1127 edhill 1.49
1128 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1129 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1130     continue
1131     fi
1132 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1133     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1134     continue
1135     fi
1136    
1137     # Check whether there are "extra runs" for this testdir
1138     extra_runs=
1139 jmc 1.108 ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
1140     #echo "ex_run_dirs='$ex_run_dirs'"
1141     for exd in $ex_run_dirs ; do
1142     name=`echo $exd | sed -e "s/$inputdir\.//"`
1143     refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1144     outf="$dir/results/$refExOut"
1145     if test -f $outf -a -r $outf ; then
1146     if test "x$MULTI_THREAD" = "xt" ; then
1147     if test -r $dir"/"$exd"/eedata.mth" ; then
1148 jmc 1.83 extra_runs="$extra_runs $name"
1149 jmc 1.108 #else echo $dir"/"$exd"/eedata.mth: not found"
1150 jmc 1.83 fi
1151 jmc 1.108 else
1152     extra_runs="$extra_runs $name"
1153 jmc 1.83 fi
1154 jmc 1.108 fi
1155     done
1156 edhill 1.28
1157     echo "-------------------------------------------------------------------------------"
1158     echo
1159 jmc 1.83 if test "x$extra_runs" = "x" ; then
1160     echo "Experiment: $dir"
1161     else
1162     echo "Experiment: $dir ; extra_runs=$extra_runs"
1163     fi
1164 edhill 1.28 echo
1165     unset genmake makedepend make run
1166 jmc 1.94 results=$EMPTY_RESULTS
1167 edhill 1.10
1168     # Create an output dir for each OPTFILE/tdir combination
1169 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1170     mkdir $rel_CDIR
1171     CDIR=`pwd`"/$rel_CDIR"
1172 edhill 1.10
1173     if test "x$CLEANUP" = xt ; then
1174 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1175     makeclean $dir/$builddir \
1176     && run_clean $dir/$rundir
1177 edhill 1.10 else
1178 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1179     && makeclean $dir/$builddir \
1180 jmc 1.83 && run_clean $dir/$rundir \
1181 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1182 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1183     && makemodel $dir/$builddir && make=Y \
1184 jmc 1.108 && linkdata $dir/$rundir $inputdir input \
1185 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1186 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1187 edhill 1.10 fi
1188    
1189     echo
1190 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1191 edhill 1.34 echo
1192 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1193 jmc 1.100 touch $CDIR"/summary.txt"
1194 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1195 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1196     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1197     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1198     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1199 jmc 1.108 if test "x$ADM" = xt ; then
1200     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1201     grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1202     >> $CDIR"/summary.txt"
1203     fi
1204 edhill 1.34
1205     for ex in $extra_runs ; do
1206 jmc 1.73 unset run
1207 jmc 1.94 results=$EMPTY_RESULTS
1208 jmc 1.106 # reference output file
1209     refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1210 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1211     rel_CDIR=$DRESULTS"/"$dir"."$ex
1212     mkdir $rel_CDIR
1213     CDIR=`pwd`"/$rel_CDIR"
1214 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1215 jmc 1.106 run_clean $dir/tr_run.$ex
1216 jmc 1.108 linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
1217 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1218 jmc 1.106 && results=`testoutput_run $dir tr_run.$ex $refExOut`
1219 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1220     fres="$fres.$ex"
1221     echo
1222 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1223     touch $CDIR"/summary.txt"
1224 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1225 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1226     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1227     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1228 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1229 jmc 1.108 if test "x$ADM" = xt ; then
1230     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1231     grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
1232     >> $CDIR"/summary.txt"
1233     fi
1234 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1235 jmc 1.100 run_clean $dir/tr_run.$ex
1236 jmc 1.83 fi
1237 edhill 1.34 done
1238 edhill 1.68
1239 jmc 1.83 #postclean $dir/$builddir
1240     if test "x$POSTCLEAN" = xt ; then
1241     makeclean $dir/$builddir \
1242     && run_clean $dir/$rundir
1243     fi
1244 edhill 1.10
1245     echo "-------------------------------------------------------------------------------"
1246    
1247 edhill 1.1 done
1248    
1249 edhill 1.34 printf "Start time: " >> $SUMMARY
1250 jmc 1.114 echo "$start_date" >> $SUMMARY
1251 edhill 1.34 printf "End time: " >> $SUMMARY
1252 edhill 1.13 date >> $SUMMARY
1253 edhill 1.20
1254     # If addresses were supplied and mpack built successfully, then try
1255     # to send email using mpack.
1256     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1257     echo "No results email was sent."
1258     else
1259     if test "x$HAVE_MPACK" = xt ; then
1260     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1261     && gzip $DRESULTS".tar" \
1262 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1263 edhill 1.20 RETVAL=$?
1264     if test "x$RETVAL" != x0 ; then
1265     echo
1266     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1267     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1268     echo " summary of results from the directory \"$DRESULTS\"."
1269     echo
1270     else
1271     echo
1272     echo "An email containing results was sent to the following addresses:"
1273     echo " \"$ADDRESSES\""
1274     echo
1275     fi
1276     test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1277     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1278     fi
1279     fi
1280 edhill 1.13
1281 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1282 edhill 1.1
1283 edhill 1.12 if test "x$CLEANUP" != xt ; then
1284 jmc 1.102 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1285 edhill 1.25 if test -e tr_out.txt ; then
1286     mv tr_out.txt tr_out.txt.old
1287 edhill 1.14 fi
1288 jmc 1.103 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1289 edhill 1.12 fi
1290 edhill 1.1
1291 edhill 1.40 if test "x$DELDIR" = xt ; then
1292     rm -rf $DRESULTS
1293     fi
1294    

  ViewVC Help
Powered by ViewVC 1.1.22