/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.132 - (hide annotations) (download)
Thu Jan 14 23:17:42 2010 UTC (14 years, 2 months ago) by jmc
Branch: MAIN
CVS Tags: checkpoint62a
Changes since 1.131: +6 -2 lines
a hack for weird situation (finish the run but with error code)

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.132 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.131 2010/01/02 01:53:49 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.122 echo " (-mth) run multi-threaded (using eedata.mth)"
15 jmc 1.83 echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.109 echo " (-gsl) compile with \"-gsl\" flag"
19 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
20 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
21 jmc 1.108 echo " (DEF=\"\" no email is sent)"
22 jmc 1.113 echo " (-mpd|-mpackdir) DIR location of the mpack utility"
23     echo " (DEF=\"../tools/mpack-1.6\")"
24 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
25     echo " (recognized groups: basic, tutorials)"
26     echo " (DEF=\"\" which test all)"
27 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
28     echo " (DEF=\"\" which test all)"
29 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
30     echo " Bourne-compatible \"sh\" shell"
31     echo " (DEF=\"\" for \"bash\")"
32 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
33 utke 1.120 echo " (-oad) perform an OpenAD adjoint run"
34 edhill 1.20 echo " (-command) STRING command to run"
35 edhill 1.6 echo " (DEF=\"make output.txt\")"
36 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
37 edhill 1.8 echo " (DEF=\"make\")"
38 jmc 1.70 echo " (-odir) STRING used to build output directory name"
39     echo " (DEF=\"hostname\")"
40 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
41 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
42 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
43     echo " (DEF=\"12\")"
44 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
45 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
46 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
47     echo " (-nogenmake|-ng) skip the genmake stage"
48     echo " (-noclean|-nc) skip the \"make clean\" stage"
49     echo " (-nodepend|-nd) skip the \"make depend\" stage"
50     echo " (-deldir|-dd) on success, delete the output directory"
51     echo " (-ts) provide timing information per timestep"
52     echo " (-papis) provide MFlop/s per timestep using PAPI"
53     echo " (-pcls) provide MFlop/s per timestep using PCL"
54 edhill 1.6 echo
55 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
56 edhill 1.6 echo "such as:"
57 edhill 1.50 echo
58 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
59     echo " -addr='abc@123.com testing@home.org'"
60 edhill 1.1 echo
61 edhill 1.50 echo "provided that the expression is properly quoted within the current"
62     echo "shell (note the use of single quotes to protect white space)."
63     echo
64 edhill 1.1 exit 1
65     }
66    
67     # build the mpack utility
68     build_mpack()
69     {
70 edhill 1.34 printf "building the mpack utility... "
71 jmc 1.113 MPACK="$MPACKDIR/mpack"
72     if test ! -x $MPACK ; then
73 edhill 1.1 if test ! -d $MPACKDIR ; then
74 edhill 1.20 echo
75 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
76     echo " are you sure this program is being run in the correct "
77     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
78 edhill 1.20 echo
79     HAVE_MPACK=f
80 edhill 1.1 fi
81 edhill 1.26 if test "x$CC" = x ; then
82     export CC=cc
83     fi
84 jmc 1.112 printf "building mpack (using CC=$CC)... "
85 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
86 edhill 1.1 RETVAL=$?
87     if test "x$RETVAL" != x0 ; then
88     echo
89     echo "Error building the mpack tools at: $MPACK_DIR"
90 edhill 1.20 echo
91     HAVE_MPACK=f
92     else
93 edhill 1.25 rm -f tr_build_mpack.out
94 edhill 1.20 HAVE_MPACK=t
95 jmc 1.113 echo "done"
96 edhill 1.1 fi
97 edhill 1.20 else
98     HAVE_MPACK=t
99 jmc 1.113 echo "already exist"
100 edhill 1.1 fi
101     }
102    
103 jmc 1.93 testoutput_var()
104 edhill 1.1 {
105 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
106 edhill 1.1 #
107 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
108 jmc 1.99 # with same output from reference file $dir/results/$reference_output
109 jmc 1.93 # using search strings s1 and text label
110 edhill 1.1
111     if [ $debug -gt 0 ]; then
112 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
113 edhill 1.1 fi
114 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
115     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
116 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
117 jmc 1.100 if [ $lncntA -lt 2 ]; then
118 edhill 1.1 if [ $verbose -gt 0 ]; then
119     echo Not enough lines of output when searching for "$2" 1>&2
120     fi
121     return 99
122     fi
123     else
124 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
125 edhill 1.1 return 99
126     fi
127     if [ $debug -gt 0 ]; then
128 jmc 1.99 echo testoutput_var: grep "$2" $1/results/$5 1>&2
129 edhill 1.1 fi
130 jmc 1.99 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
131 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
132 jmc 1.100 if [ $lncntB -lt 2 ]; then
133 edhill 1.1 if [ $verbose -gt 0 ]; then
134     echo Not enough lines of output when searching for "$2" 1>&2
135     fi
136     return 99
137     fi
138 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
139     if [ $verbose -gt 0 ]; then
140     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
141     fi
142     return 99
143     fi
144 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
145     if [ $has_nan -gt 0 ] ; then
146 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
147 edhill 1.72 return 99
148     fi
149     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
150     if [ $has_inf -gt 0 ] ; then
151 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
152 jmc 1.71 return 99
153     fi
154 edhill 1.1 if [ $debug -gt 0 ]; then
155 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
156 edhill 1.1 fi
157     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
158     if [ $debug -gt 0 ]; then
159 jmc 1.93 echo testoutput_var: compare_lines 1>&2
160 edhill 1.1 fi
161 edhill 1.22 if [ $verbose -gt 1 ]; then
162     cat tmp3.txt 1>&2
163     fi
164     echo "-1" >> tmp3.txt
165 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
166     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
167 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
168 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
169     if [ $verbose -gt 0 ]; then
170 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
171 edhill 1.1 fi
172     digits_of_similarity=99
173     else
174     if [ $verbose -gt 0 ]; then
175 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
176 edhill 1.1 fi
177     fi
178 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
179 edhill 1.1
180     return $digits_of_similarity
181     }
182    
183 jmc 1.93 testoutput_run()
184 edhill 1.1 {
185 jmc 1.99 # testoutput_run directory subdir reference_output
186 edhill 1.1 #
187 jmc 1.93 # test output from 1 run in "directory"
188 jmc 1.103 # --> same processing for adjoint & forward test
189 jmc 1.94 # default list of output variables to be checked:
190     # 1rst : main variable used to decide if it pass or FAIL
191     # others : number of matching digits to be printed in summary.txt
192     listChk=$DEF_CHECK_LIST
193     # load experiment-specific list from file "tr_checklist" (if it exist)
194     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
195     sVar=`echo $listChk | awk '{print $1}'`
196     # remove 1rst var and expand the list: + => min max mean s.d
197 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
198     | sed 's/+//g' | sed "s/^$sVar//"`
199 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
200 jmc 1.117 # check for ptracer output in reference_output file :
201     outpref=$1/results/$3
202     ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
203     ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
204     for ii in $PTRACERS_NUM ; do
205     ptrfound=0
206     for jj in $ptr_mon ; do
207     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
208     tst=`grep $name $outpref | wc -l | awk '{print $1}'`
209     if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
210     done
211     if test $ptrfound = '1' ; then
212     eval "HAVE_PTR0"$ii"=t"
213     else
214     eval "HAVE_PTR0"$ii"=f"
215 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
216 jmc 1.117 # remove this ptr from the list of output variable to check
217     # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
218     listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
219     fi
220     fi
221     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
222     done
223 mlosch 1.116 tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
224 jmc 1.94 if test $tst != 1 ; then
225     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
226     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
227     echo "==> WARNING: in checked list:" $listVar 1>&2
228     #- put it back once:
229     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
230     fi
231     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
232     echo "listVar='$listVar'" > $CDIR"/summary.txt"
233     allargs=""
234     for xx in $listVar
235     do
236     case $xx in
237     'PS') if [ $debug -gt 0 ]
238     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
239 jmc 1.99 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
240 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
241 jmc 1.101 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
242     'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
243 jmc 1.99 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
244     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
245     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
246     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
247     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
248     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
249     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
250     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
251 jmc 1.94 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
252     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
253     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
254 jmc 1.99 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
255 jmc 1.94 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
256     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
257     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
258 jmc 1.99 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
259 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
260     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
261     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
262     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
263     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
264     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
265     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
266 jmc 1.99 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
267 jmc 1.130 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;;
268     'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;;
269     'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;;
270     'Qntsd') testoutput_var $1 "forcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;;
271 jmc 1.101 'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;;
272     'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;;
273     'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;;
274     'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;;
275     'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;;
276 jmc 1.126 'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Heff max" $2 $3; yy=$? ;;
277     'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Heff mean" $2 $3; yy=$? ;;
278     'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Heff StDv" $2 $3; yy=$? ;;
279 jmc 1.101 'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
280     'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;;
281     'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;;
282     'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;;
283     'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;;
284     'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;;
285 jmc 1.94 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
286     esac
287     if test $xx = $sVar
288     then allargs="$allargs > $yy <"
289     else allargs="$allargs $yy"
290     fi
291     done
292    
293     nbVar=`echo $listVar | awk '{print NF}'`
294 jmc 1.97 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
295 jmc 1.94 #-- fill line (up to standard length) with dot:
296 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
297 jmc 1.94 echo $allargs $adNul
298     else
299     echo $allargs
300     fi
301 jmc 1.103 # <-- same processing for adjoint & forward test
302 edhill 1.1 }
303    
304     genmakemodel()
305     {
306     # genmakemodel directory
307 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
308     echo "genmake skipped!"
309     else
310 edhill 1.34 if test "x$BASH" = x ; then
311     GENMAKE2="../../../tools/genmake2"
312     else
313     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
314     fi
315 edhill 1.10 (
316     cd $1;
317 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
318 utke 1.120 if test "x$ADM" = xt ; then
319     command="$command --mods=../code_ad"
320     elif test "x$OADM" = xt ; then
321     command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
322     else
323 jmc 1.124 command="$command -mods=../code"
324 edhill 1.24 fi
325 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
326 jmc 1.124 command="$command -optfile=$OPTFILE"
327 edhill 1.10 fi
328     if test "x$IEEE" != x ; then
329     command="$command -ieee"
330     fi
331 jmc 1.109 if test "x$GSL" = xt ; then
332     command="$command -gsl"
333     fi
334 edhill 1.66 if test "x$MPI" = xt ; then
335     command="$command -mpi"
336     fi
337 jmc 1.122 if test "x$MULTI_THREAD" = xt ; then
338     #- run multi-threaded using OpenMP:
339     command="$command -omp"
340     fi
341 ce107 1.90 if test "x$TS" = xt ; then
342     command="$command -ts"
343     fi
344     if test "x$PAPIS" = xt ; then
345     command="$command -papis"
346     else
347     if test "x$PCLS" = xt ; then
348     command="$command -pcls"
349     fi
350     fi
351 edhill 1.10 printf 'genmake ... ' 1>&2
352 utke 1.120 # $command > make.log 2>&1
353     eval $command > make.log 2>&1
354 edhill 1.10 RETVAL=$?
355 edhill 1.44 # Reduce the size of the testing emails!
356 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
357 edhill 1.10 if test "x$RETVAL" != x0 ; then
358     tail make.log
359     echo "genmakemodel: genmake failed" 1>&2
360 edhill 1.21 cp genmake_* make.log $CDIR
361 edhill 1.10 return 1
362     else
363 edhill 1.20 echo "successful" 1>&2
364 edhill 1.1 fi
365 edhill 1.10 )
366     fi
367 edhill 1.1 }
368    
369     makeclean()
370     {
371     # makeclean directory
372 edhill 1.10 if test "x$NOCLEAN" = xt ; then
373 jmc 1.83 echo "make Clean skipped!"
374 edhill 1.10 else
375     (
376     cd $1;
377 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
378 edhill 1.10 if test -r Makefile ; then
379 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
380     $MAKE Clean >> make.log 2>&1
381 edhill 1.10 RETVAL=$?
382     if test "x$RETVAL" != x0 ; then
383     tail make.log
384 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
385 edhill 1.10 cp make.log $CDIR"/make.log"
386     return 1
387     fi
388 jmc 1.127 echo successful 1>&2
389     else
390     echo ''
391 edhill 1.10 fi
392     exit 0
393     )
394     fi
395     }
396    
397 jmc 1.83 run_clean()
398 edhill 1.68 {
399 jmc 1.83 # run_clean directory
400     if test "x$NOCLEAN" = xt ; then
401     echo "run_clean skipped!"
402     else
403 edhill 1.68 (
404     cd $1;
405 jmc 1.83 printf 'clean run-dir ... ' 2>&1
406     # part of what is done after "make clean" when doing "make CLEAN"
407     find . -name "*.meta" -exec rm {} \;
408     find . -name "*.data" -exec rm {} \;
409     find . -name "fort.*" -exec rm {} \;
410     find . -type l -exec rm {} \;
411     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
412     rm -rf mnc_test_*
413 jmc 1.107 rm -f *_MIT_CE_000.opt0000 costfunction*0000
414 jmc 1.83 echo successful 1>&2
415 edhill 1.68 exit 0
416     )
417     fi
418     }
419    
420 edhill 1.10 makedependmodel()
421     {
422     # makedependmodel directory
423     if test "x$NODEPEND" = xt ; then
424     echo "make depend skipped!"
425     else
426     (
427     cd $1;
428     printf 'make depend ... ' 1>&2
429     $MAKE depend >> make.log 2>&1
430 edhill 1.1 RETVAL=$?
431     if test "x$RETVAL" != x0 ; then
432     tail make.log
433 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
434 edhill 1.1 cp make.log $CDIR"/make.log"
435     return 1
436 edhill 1.10 else
437 edhill 1.20 echo successful 1>&2
438 edhill 1.1 fi
439 edhill 1.10 )
440     fi
441 edhill 1.1 }
442    
443     makemodel()
444     {
445     # makemodel directory
446     (
447     cd $1;
448     if test -r Makefile ; then
449     printf 'make ... ' 1>&2
450 utke 1.120 if test "x$ADM" = xt ; then
451     $MAKE adall >> make.log 2>&1
452     elif test "x$OADM" = xt ; then
453     $MAKE adAll >> make.log 2>&1
454     else
455 edhill 1.43 if test "x$JOBS" = x ; then
456     $MAKE >> make.log 2>&1
457     else
458     $MAKE -j $JOBS >> make.log 2>&1
459     fi
460 edhill 1.24 fi
461 edhill 1.1 RETVAL=$?
462     if test "x$RETVAL" != x0 ; then
463     tail make.log
464     echo failed 1>&2
465     cp make.log $CDIR"/make.log"
466 jmc 1.131 rm -f $EXECUTABLE
467 edhill 1.1 return 1
468     else
469 edhill 1.20 echo successful 1>&2
470 edhill 1.1 fi
471     fi
472     )
473     }
474    
475 edhill 1.27 symlink_mpifiles()
476     {
477     # Put special links so that MPI specific files are used
478     # This MUST be invoked between makeclean and makelinks because
479     # the Makefile will link to non-mpi files by default
480    
481     dir=$1
482     code_dir=$2
483     BUILD_DIR=$dir/$3
484     CODE_DIR=$dir/$code_dir
485 edhill 1.45
486 edhill 1.27 # These are files that should replace their counter-part when using -mpi
487 mlosch 1.111 MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`
488 edhill 1.27
489     # Is this an MPI run?
490     if test "x$MPI" = xt ; then
491     # YES: We symbolically link these files to the build
492     # dir so long as there is no real file in place
493     for ii in $MPI_FILES ; do
494     i=`echo $ii | sed 's:^\./::'`
495     name=`echo $i | sed 's:_mpi::' `
496     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
497     RETVAL=$?
498     if test "x$RETVAL" != x0 ; then
499     if ! test -f $BUILD_DIR/$i ; then
500 edhill 1.45 #echo Linking $name to $i
501 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
502     fi
503     fi
504     done
505     else
506 edhill 1.45 # NO: We undo any _mpi symbolically linked files
507 edhill 1.27 for ii in $MPI_FILES ; do
508     i=`echo $ii | sed 's:^\./::'`
509     name=`echo $i | sed 's:_mpi::' `
510     if test -L $BUILD_DIR/$name ; then
511 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
512     RETVAL=$?
513     if test "x$RETVAL" = x0 ; then
514     #echo Un-linking $name from $linktarg
515 edhill 1.27 rm -f $BUILD_DIR/$name
516     fi
517     fi
518     done
519     fi
520    
521     }
522    
523 edhill 1.1 linkdata()
524     {
525 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
526 edhill 1.1 #
527     # symbolically link data files to run directory
528 jmc 1.83 if test -d $1 ; then
529 edhill 1.12 (
530 jmc 1.83 cd $1 ; shift
531 jmc 1.125 inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
532     for xx in $inpMPI ; do
533     if test -r "../"$1"/"$xx ; then
534     # found 1 _mpi sfx file in 1rst input dir and it is readable
535     yy=`echo $xx | sed 's:\.mpi$::'`
536     if test "x$MPI" = "xt" ; then
537     # mpi test: remove symbolic link & link _mpi sfx file
538     if test -h $yy ; then rm -f $yy ; fi
539     if test ! -r $yy ; then
540     ln -sf "../"$1"/"$xx $yy ;
541     printf "$xx " 1>&2
542     fi
543     else
544     # not mpi test: remove symbolic link
545     if test -h $yy ; then rm -f $yy ; fi
546     fi
547     fi
548     done
549 jmc 1.83 if test -r "../"$1"/eedata.mth" ; then
550     # found eedata.mth in 1rst input dir and it is readable
551     if test "x$MULTI_THREAD" = "xt" ; then
552     # multi-threaded test: remove symbolic link & link eedata.mth
553     if test -h eedata ; then rm -f eedata ; fi
554     if test ! -r eedata ; then
555     ln -sf "../"$1"/eedata.mth" eedata ;
556     printf 'eedata.mth ' 1>&2
557 edhill 1.24 fi
558 jmc 1.83 else
559     # not multi-threaded test: remove eedata symbolic link
560     if test -h eedata ; then rm -f eedata ; fi
561     fi
562     fi
563 jmc 1.108 prevDir='NONE'
564 jmc 1.83 for ldir in $* ; do
565 jmc 1.108 if test -d "../"$ldir -a $ldir != $prevDir ; then
566 jmc 1.83 printf 'ldir='${ldir} 1>&2
567     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
568     for i in $files ; do
569     if test ! -d "../"$ldir/$i ; then
570     if test ! -r $i ; then
571     printf ' '$i 1>&2
572     ln -sf "../"$ldir"/"$i $i
573     fi
574     fi
575     done
576     if test -x "../"$ldir"/"prepare_run ; then
577     "../"$ldir"/"prepare_run
578 edhill 1.24 fi
579 jmc 1.83 printf ' ; ' 1>&2
580 jmc 1.82 fi
581 jmc 1.108 prevDir=$ldir
582 jmc 1.83 done
583 edhill 1.12 )
584 edhill 1.1 fi
585     }
586    
587     runmodel()
588     {
589 edhill 1.6 # runmodel directory
590 edhill 1.1 #
591 edhill 1.24 # runs "$COMMAND in "directory"
592 edhill 1.6 # (where "$COMMAND" is relative to "directory")
593 edhill 1.1 (
594     cd $1
595 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
596 edhill 1.6 # make output.txt
597 edhill 1.38 echo
598 jmc 1.106 if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
599     diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
600 jmc 1.119 outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi
601 jmc 1.106 fi
602 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
603 jmc 1.119 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
604 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
605     fi
606     if test ! -x $EXECUTABLE ; then
607 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
608     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
609     echo " no executable:" $EXECUTABLE >> $RUNLOG
610 jmc 1.83 RETVAL=8
611 jmc 1.88 ENDVAL=-1
612 jmc 1.75 else
613 jmc 1.98 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
614     # output do not exist or is older than executable:
615 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
616     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
617     ( eval $COMMAND ) >> $RUNLOG 2>&1
618 jmc 1.83 RETVAL=$?
619     else
620     RETVAL=0
621 jmc 1.119 if test -f $RUNLOG ; then
622     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
623     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
624 jmc 1.88 else
625 jmc 1.119 touch $RUNLOG
626     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
627     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
628     echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1
629 jmc 1.88 fi
630 jmc 1.83 fi
631 jmc 1.119 ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
632 jmc 1.75 fi
633 jmc 1.119 rm -f run.log_tmp
634 jmc 1.88 if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
635 jmc 1.119 tail $RUNLOG
636 edhill 1.20 echo successful 1>&2
637 edhill 1.44 # === Reduce the size of the testing emails!
638 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
639 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
640 edhill 1.6 return 0
641 jmc 1.132 elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then
642     #-- for some weird cases (run is finihed but with error code)
643     tail $RUNLOG
644     echo finished with error '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
645     return 0
646 edhill 1.6 else
647 jmc 1.119 tail $RUNLOG
648 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
649 jmc 1.119 cp $RUNLOG $CDIR"/"$RUNLOG
650 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
651 edhill 1.6 return 1
652 edhill 1.1 fi
653     )
654     }
655    
656     createcodelet()
657     {
658     # create codelet for comparing model output
659    
660 jmc 1.112 printf "creating the comparison code (using CC=$CC)... "
661 jmc 1.91 cat > tr_cmpnum.c <<EOF
662 edhill 1.22 #include <stdio.h>
663     #include <math.h>
664     int main( int argc, char** argv ) {
665 jmc 1.91 int linnum,cmplin,best,lncnt;
666 edhill 1.52 double a,b,abave,relerr;
667 jmc 1.57 best = -22;
668 adcroft 1.33 lncnt = 0;
669 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
670 edhill 1.22 scanf("%d", &linnum);
671     if (linnum == -1) break;
672     scanf("%lf", &a); scanf("%lf", &b);
673 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
674 jmc 1.91 if ( abave == abave ) {
675     if (abave > 0.0) {
676     relerr=fabs(a-b)/abave;
677     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
678     else { cmplin = -16 ; }
679     best = (best > cmplin) ? best : cmplin; }
680     else { cmplin = -22 ; }
681     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
682     }
683     else {
684     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
685     break; }
686 edhill 1.22 }
687 adcroft 1.33 if (lncnt == 999) best=-29;
688 jmc 1.91 if (linnum != -1) best=-99;
689 edhill 1.22 printf("%d\n", -best);
690     return 0;
691     }
692     EOF
693 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
694 edhill 1.1
695 jmc 1.91 if [ -x ./tr_cmpnum ]; then
696 edhill 1.1 echo "OK"
697     return 0
698     else
699     echo
700 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
701     echo " a C compiler using the CC environment variable."
702 edhill 1.1 exit 1
703     fi
704     }
705    
706     formatresults()
707     {
708     # formatresults expt genmake depend make run results*
709    
710     nm=$1
711     printf '%s %s %s %s' $2 $3 $4 $5
712     shift; shift; shift; shift; shift;
713 jmc 1.94 listPrt=$@
714     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
715     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
716 jmc 1.97 printf '%3s' $listPrt
717 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
718 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
719 edhill 1.1
720 jmc 1.94 if [ $xx = '..' ]; then
721     printf ' N/O '
722     elif [ $xx = '--' ]; then
723     printf ' N/O '
724     elif [ $xx = 99 ]; then
725 edhill 1.1 printf ' N/O '
726     else
727 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
728 edhill 1.1 printf ' pass'
729     else
730     printf ' FAIL'
731     fi
732     fi
733     printf ' %s' $nm
734     printf '\n'
735    
736     }
737    
738     scandirs()
739     {
740 jmc 1.81 if [ $# -eq 1 ]; then
741     for arg in * ; do
742 jmc 1.99 test -f $arg/$1 && echo $arg
743 jmc 1.81 done
744 edhill 1.1 else
745 jmc 1.81 echo $*
746 edhill 1.1 fi
747     }
748    
749    
750 jmc 1.123 check_eedata()
751     {
752     # check_eedata size.h eedata
753     if [ $# -eq 2 ] ; then
754     if test -f $1 -a -f $2 ; then
755     sx=`grep "^ & *nSx *=" $1 | sed "s/^ & *nSx *=//" | sed 's/, *$//'`
756     nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
757     if test "x$nx" = x ; then
758     rx=10
759     else
760     rx=`expr $sx % $nx`
761     fi
762     sy=`grep "^ & *nSy *=" $1 | sed "s/^ & *nSy *=//" | sed 's/, *$//'`
763     ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
764     if test "x$ny" = x ; then
765     ry=20
766     else
767     ry=`expr $sy % $ny`
768     fi
769     echo `expr $rx + $ry`
770     else
771     echo '-1'
772     fi
773     else
774     echo '-2'
775     fi
776    
777     }
778    
779 edhill 1.1 ###############################################################################
780    
781    
782     # Default properties
783     debug=0
784     verbose=1
785     clean=0
786 edhill 1.10
787 edhill 1.40 IEEE=true
788 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
789     IEEE=$MITGCM_IEEE
790     fi
791 jmc 1.109 GSL=f
792 edhill 1.10
793     CLEANUP=f
794     QUICK=f
795     NOGENMAKE=f
796     NOCLEAN=f
797     NODEPEND=f
798 edhill 1.68 POSTCLEAN=f
799 edhill 1.1
800 edhill 1.4 BASH=
801 edhill 1.10 OPTFILE=NONE
802     ADDRESSES=
803 edhill 1.1 TESTDIRS=
804 jmc 1.89 SKIPDIRS=
805 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
806 edhill 1.20 HAVE_MPACK=
807 jmc 1.113 MPACK=
808 edhill 1.24 COMMAND=
809 edhill 1.59 if test "x$MAKE" = x ; then
810     MAKE=make
811     fi
812     if test "x$CC" = x ; then
813     CC=cc
814     fi
815 edhill 1.43 JOBS=
816 edhill 1.7 MPI=f
817 jmc 1.83 MULTI_THREAD=f
818 jmc 1.70 OUTDIR=
819 edhill 1.40 DELDIR=
820 edhill 1.1
821 edhill 1.24 ADM=
822 utke 1.120 OADM=
823 edhill 1.24
824 jmc 1.117 # list of pTracers to check for monitor output
825 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
826 edhill 1.49
827 jmc 1.92 MATCH_CRIT=13
828    
829 edhill 1.34 printf "parsing options... "
830 edhill 1.1
831     ac_prev=
832     for ac_option ; do
833    
834     # If the previous option needs an argument, assign it.
835     if test -n "$ac_prev"; then
836     eval "$ac_prev=\$ac_option"
837     ac_prev=
838     continue
839     fi
840    
841     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
842    
843     case $ac_option in
844    
845     -help | --help | -h | --h)
846     usage ;;
847    
848 edhill 1.2 -optfile | --optfile | -of | --of)
849 edhill 1.10 ac_prev=OPTFILE ;;
850 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
851 edhill 1.10 OPTFILE=$ac_optarg ;;
852 edhill 1.1
853     -addr | --addr | -a | --a)
854     ac_prev=ADDRESSES ;;
855     -addr=* | --addr=*)
856     ADDRESSES=$ac_optarg ;;
857 jmc 1.113 -mpackdir | --mpackdir | -mpd | --mpd)
858     ac_prev=MPACKDIR ;;
859     -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
860     MPACKDIR=$ac_optarg ;;
861 edhill 1.1
862     -tdir | --tdir | -t | --t)
863     ac_prev=TESTDIRS ;;
864     -tdir=* | --tdir=*)
865     TESTDIRS=$ac_optarg ;;
866 edhill 1.4
867 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
868     ac_prev=SKIPDIRS ;;
869     -skipdir=* | --skipdir=*)
870     SKIPDIRS=$ac_optarg ;;
871    
872 edhill 1.4 -bash | --bash | -b | --b)
873     ac_prev=BASH ;;
874     -bash=* | --bash=*)
875     BASH=$ac_optarg ;;
876 edhill 1.5
877 edhill 1.6 -command | --command | -c | --c)
878     ac_prev=COMMAND ;;
879     -command=* | --command=*)
880     COMMAND=$ac_optarg ;;
881 edhill 1.8
882     -make | --make | -m | --m)
883     ac_prev=MAKE ;;
884     -make=* | --make=*)
885     MAKE=$ac_optarg ;;
886 edhill 1.1
887 jmc 1.70 -odir | --odir)
888     ac_prev=OUTDIR ;;
889     -odir=* | --odir=*)
890     OUTDIR=$ac_optarg ;;
891    
892 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
893     ac_prev=PTRACERS_NUM ;;
894     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
895     PTRACERS_NUM=$ac_optarg ;;
896    
897 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
898     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
899    
900 edhill 1.43 -j) ac_prev=JOBS ;;
901     -j=*) JOBS=$ac_optarg ;;
902    
903 edhill 1.10 -clean | --clean)
904 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
905 edhill 1.10
906     -quick | --quick | -q | --q)
907     QUICK=t ;;
908     -nogenmake | --nogenmake | -ng | --ng)
909     NOGENMAKE=t ;;
910     -noclean | --noclean | -nc | --nc)
911     NOCLEAN=t ;;
912     -nodepend | --nodepend | -nd | --nd)
913     NODEPEND=t ;;
914    
915 edhill 1.68 -postclean | --postclean | -pc | --pc)
916     POSTCLEAN=t ;;
917    
918 edhill 1.18 -mpi) MPI=t ;;
919 edhill 1.10
920 jmc 1.83 -mth) MULTI_THREAD=t ;;
921    
922 edhill 1.24 -adm | -ad) ADM=t ;;
923    
924 utke 1.120 -oad) OADM=t; NODEPEND=t ;;
925    
926 edhill 1.10 -ieee) IEEE=true ;;
927     -noieee) IEEE= ;;
928 jmc 1.109 -gsl) GSL=t ;;
929 edhill 1.10
930 edhill 1.1 -verbose) verbose=2 ;;
931     -debug) debug=1 ;;
932     -quiet) verbose=0 ;;
933    
934 edhill 1.40 -deldir | -dd) DELDIR=t ;;
935    
936 ce107 1.90 -ts) TS=t;;
937    
938     -papis) PAPIS=t;;
939    
940     -pcls) PCL=t;;
941    
942 edhill 1.1 -*)
943     echo "Error: unrecognized option: "$ac_option
944     usage
945     ;;
946    
947     *)
948     echo "Error: unrecognized argument: "$ac_option
949     usage
950     ;;
951    
952     esac
953    
954     done
955    
956 edhill 1.10 if test "x$QUICK" = xt ; then
957     NOGENMAKE=t
958     NOCLEAN=t
959     NODEPEND=t
960     fi
961    
962 jmc 1.99 #- setting for forward or ADM testing
963     if test "x$ADM" = xt ; then
964     code_dir=code_ad
965 jmc 1.108 inputdir=input_ad
966 jmc 1.99 ref_outp="output_adm.txt"
967     EXECUTABLE="mitgcmuv_ad"
968 utke 1.120 elif test "x$OADM" = xt ; then
969     code_dir=code_oad
970     inputdir=input_oad
971     ref_outp="output_oadm.txt"
972     EXECUTABLE="mitgcmuv_ad"
973 jmc 1.99 else
974     code_dir=code
975 jmc 1.108 inputdir=input
976 jmc 1.99 ref_outp="output.txt"
977     EXECUTABLE="mitgcmuv"
978     fi
979    
980 edhill 1.1 if test "x$TESTDIRS" = x ; then
981 jmc 1.103 LIST=`scandirs results/$ref_outp`
982 jmc 1.84 else
983     #- expand group of experiments:
984     LIST=" "
985     for xx in $TESTDIRS
986     do
987     case $xx in
988 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
989     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
990     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
991 jmc 1.84 ;;
992     'tutorials')
993     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
994     *) LIST=${LIST}" "$xx ;;
995     esac
996     done
997 jmc 1.89 fi
998     #echo 'LIST='${LIST}'<'
999     #- skip dirs, remove duplicate and non-directory:
1000     TESTDIRS=" "
1001     count=0
1002     for xx in $LIST
1003     do
1004     yy=`echo $SKIPDIRS | grep -c $xx`
1005     if test $yy = 0 ; then
1006 jmc 1.84 if test -d $xx ; then
1007     yy=`echo $TESTDIRS | grep -c $xx`
1008     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
1009 jmc 1.89 else count=1 ;
1010     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
1011     fi
1012     else
1013     if test $count = 1 ; then echo -n ", \"$xx\""
1014     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
1015 jmc 1.84 fi
1016 jmc 1.89 fi
1017     done
1018     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
1019 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
1020 edhill 1.1
1021 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
1022     OPTFILE=$MITGCM_OF
1023     fi
1024    
1025 jmc 1.119 RUNLOG="run.log"
1026 jmc 1.99 OUTPUTFILE=$ref_outp
1027 jmc 1.87 if test "x$COMMAND" = x ; then
1028 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
1029 edhill 1.24 fi
1030 jmc 1.87 if test "x$MPI" = xt ; then
1031     OUTPUTFILE="STDOUT.0000"
1032 edhill 1.24 fi
1033    
1034 jmc 1.76 echo "OK (COMMAND= $COMMAND )"
1035 edhill 1.1
1036 jmc 1.94 # set the Default List of output variables to be checked:
1037     # (use default or load experiment-specific list from file "tr_checklist")
1038     # content : 1rst = main variable used to decide if it pass or FAIL
1039     # others = number of matching digits to be printed in summary.txt
1040 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1041 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
1042     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
1043 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1044 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
1045 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
1046 jmc 1.99 else
1047 jmc 1.103 DEF_CHECK_LIST='Grad Cost Grad'
1048 jmc 1.99 EMPTY_RESULTS='.. ..'
1049     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1050 jmc 1.94 fi
1051    
1052 edhill 1.1 # create the FORTRAN comparison code
1053     createcodelet
1054    
1055 jmc 1.112 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
1056     if test "x$ADDRESSES" = x ; then
1057 edhill 1.32 echo "skipping mpack build"
1058     else
1059 edhill 1.31 build_mpack
1060     fi
1061 edhill 1.1
1062     # Create a uniquely named directory to store results
1063 jmc 1.110 CMDLINE=$0
1064 jmc 1.118 for xx in "$@" ; do nw=`echo $xx | wc -w`
1065     if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
1066     else CMDLINE="$CMDLINE '$xx'" ; fi
1067     done
1068     #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
1069 edhill 1.1 MACH=`hostname`
1070 edhill 1.2 UNAMEA=`uname -a`
1071 edhill 1.1 DATE=`date +%Y%m%d`
1072 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1073 jmc 1.70 if test "x$OUTDIR" != x ; then
1074     BASE="tr_"$OUTDIR"_"$DATE"_"
1075 jmc 1.121 else
1076     short_name=`hostname | sed 's/\..*$//'`
1077     BASE="tr_"$short_name"_"$DATE"_"
1078 jmc 1.70 fi
1079 edhill 1.1 DNUM=0
1080     DRESULTS="$BASE$DNUM"
1081     while test -e $DRESULTS ; do
1082     DNUM=$(( $DNUM + 1 ))
1083     DRESULTS="$BASE$DNUM"
1084     done
1085     mkdir $DRESULTS
1086     RETVAL=$?
1087     if test "x$RETVAL" != x0 ; then
1088 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1089 edhill 1.1 exit 1
1090     fi
1091     SUMMARY="$DRESULTS/summary.txt"
1092 edhill 1.16 start_date=`date`
1093 edhill 1.17 echo $start_date > $SUMMARY
1094 jmc 1.110 echo 'run:' $CMDLINE >> $SUMMARY
1095     echo 'on :' $UNAMEA >> $SUMMARY
1096 edhill 1.1
1097 edhill 1.11 of_path=
1098 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1099     if test -r $OPTFILE ; then
1100 edhill 1.11 # get the path
1101     path=${OPTFILE%/*}
1102     if test "x$path" = x ; then
1103     of_path=`pwd`
1104     else
1105     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1106     fi
1107     file=${OPTFILE##*/}
1108     OPTFILE=$of_path/$file
1109 edhill 1.21 cp $OPTFILE $DRESULTS
1110     echo >> $SUMMARY
1111     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1112 edhill 1.11 else
1113 edhill 1.21 echo | tee $SUMMARY
1114     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1115     exit 1
1116 edhill 1.10 fi
1117 edhill 1.21 else
1118     echo >> $SUMMARY
1119     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1120     echo " so the genmake default will be used." >> $SUMMARY
1121 edhill 1.10 fi
1122     echo
1123     echo >> $SUMMARY
1124 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1125 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1126 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1127     else line_0="default "$MATCH_CRIT ; fi
1128     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1129 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1130 edhill 1.49 line_1="G D M c m s m s m s m s"
1131     line_2="E p a R g m m e . m m e . m m e . m m e ."
1132     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1133     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1134 edhill 1.50 for ii in $PTRACERS_NUM ; do
1135 edhill 1.49 line_0="$line_0 --PTR 0"$ii"--"
1136     line_1="$line_1 m s"
1137     line_2="$line_2 m m e ."
1138     line_3="$line_3 i a a d"
1139     line_4="$line_4 n x n ."
1140     done
1141     echo "$line_0" | tee -a $SUMMARY
1142     echo "$line_1" | tee -a $SUMMARY
1143     echo "$line_2" | tee -a $SUMMARY
1144     echo "$line_3" | tee -a $SUMMARY
1145     echo "$line_4" | tee -a $SUMMARY
1146     echo " " | tee -a $SUMMARY
1147 edhill 1.24 else
1148     echo "ADJOINT=true" >> $SUMMARY
1149     echo >> $SUMMARY
1150 jmc 1.100 if [ $MATCH_CRIT -lt 10 ] ;
1151 jmc 1.103 then line_0="default "$MATCH_CRIT ;
1152     else line_0="default "$MATCH_CRIT ; fi
1153 jmc 1.100 echo "$line_0" | tee -a $SUMMARY
1154 edhill 1.24 cat << EOF | tee -a $SUMMARY
1155     G D M C G
1156     E p a R o r
1157     N n k u s a
1158     2 d e n t d
1159    
1160     EOF
1161     fi
1162 edhill 1.1
1163 edhill 1.10 # ...and each test directory...
1164     for dir in $TESTDIRS ; do
1165    
1166 jmc 1.106 # set builddir & rundir:
1167     builddir="build"
1168     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1169     rundir="run"
1170     if test ! -d $dir/$rundir ; then
1171     rundir=$builddir
1172     fi
1173     CODE_DIR=$dir/$code_dir
1174     BUILD_DIR=$dir/$builddir
1175    
1176 edhill 1.10 # Cleanup only!
1177     if test "x$CLEANUP" = xt ; then
1178 jmc 1.127 echo -n ' --- dir:' $BUILD_DIR ': '
1179     makeclean $BUILD_DIR
1180 jmc 1.128 rm -f $BUILD_DIR/$EXECUTABLE
1181 jmc 1.106 if test -d $dir/$rundir/CVS ; then
1182 jmc 1.127 echo -n ' --- dir:' $dir/$rundir ': '
1183 jmc 1.106 run_clean $dir/$rundir
1184 edhill 1.10 fi
1185 jmc 1.127 trdir=`( cd $dir ; find . -type d -name "tr_run.*" -print | sed 's/^.\///')`
1186     ttd=`echo $trdir | wc -w`
1187     if test $ttd != 0 ; then
1188     echo ' --- rm dir:' $trdir
1189     ( cd $dir ; rm -rf $trdir )
1190     fi
1191 edhill 1.10 continue
1192 edhill 1.1 fi
1193 edhill 1.3
1194 edhill 1.10 # Verify that the testdir exists and contains previous
1195     # results in the correct location--or skip this directory!
1196 jmc 1.104 fout=$dir"/results/"$ref_outp
1197 edhill 1.24 if test ! -r $fout ; then
1198     echo "can't read \"$fout\" -- skipping $dir"
1199 edhill 1.10 continue
1200     fi
1201 edhill 1.7
1202 jmc 1.117 # Check for specific files for particular type of run
1203 edhill 1.49
1204 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1205 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1206     continue
1207     fi
1208 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1209     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1210     continue
1211     fi
1212 jmc 1.123 if test "x$MPI" = "xt" -a "x$MULTI_THREAD" = "xt" ; then
1213     retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/input/eedata.mth"`
1214     if test $retv != 0 ; then
1215     echo "input/eedata.mth tiling misfit -- skipping $dir"
1216     continue
1217     fi
1218     fi
1219 jmc 1.83
1220     # Check whether there are "extra runs" for this testdir
1221     extra_runs=
1222 jmc 1.108 ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
1223     #echo "ex_run_dirs='$ex_run_dirs'"
1224     for exd in $ex_run_dirs ; do
1225     name=`echo $exd | sed -e "s/$inputdir\.//"`
1226     refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1227     outf="$dir/results/$refExOut"
1228     if test -f $outf -a -r $outf ; then
1229 jmc 1.123 if test "x$MULTI_THREAD" = "xt" ; then
1230     if test -r $dir"/"$exd"/eedata.mth" ; then
1231     if test "x$MPI" = "xt" ; then
1232     retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/"$exd"/eedata.mth"`
1233     if test $retv = 0 ; then
1234 jmc 1.83 extra_runs="$extra_runs $name"
1235 jmc 1.123 else
1236     echo $exd"/eedata.mth tiling misfit -- skipping $dir"
1237 jmc 1.83 fi
1238 jmc 1.123 else
1239 jmc 1.108 extra_runs="$extra_runs $name"
1240 jmc 1.123 fi
1241     #else echo $dir"/"$exd"/eedata.mth: not found"
1242 jmc 1.83 fi
1243 jmc 1.123 else
1244     extra_runs="$extra_runs $name"
1245     fi
1246 jmc 1.108 fi
1247     done
1248 edhill 1.28
1249     echo "-------------------------------------------------------------------------------"
1250     echo
1251 jmc 1.83 if test "x$extra_runs" = "x" ; then
1252     echo "Experiment: $dir"
1253     else
1254     echo "Experiment: $dir ; extra_runs=$extra_runs"
1255     fi
1256 edhill 1.28 echo
1257     unset genmake makedepend make run
1258 jmc 1.94 results=$EMPTY_RESULTS
1259 edhill 1.10
1260     # Create an output dir for each OPTFILE/tdir combination
1261 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1262     mkdir $rel_CDIR
1263     CDIR=`pwd`"/$rel_CDIR"
1264 edhill 1.10
1265     if test "x$CLEANUP" = xt ; then
1266 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1267     makeclean $dir/$builddir \
1268     && run_clean $dir/$rundir
1269 edhill 1.10 else
1270 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1271     && makeclean $dir/$builddir \
1272 jmc 1.83 && run_clean $dir/$rundir \
1273 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1274 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1275     && makemodel $dir/$builddir && make=Y \
1276 jmc 1.108 && linkdata $dir/$rundir $inputdir input \
1277 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1278 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1279 edhill 1.10 fi
1280    
1281     echo
1282 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1283 edhill 1.34 echo
1284 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1285 jmc 1.100 touch $CDIR"/summary.txt"
1286 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1287 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1288     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1289     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1290     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1291 jmc 1.108 if test "x$ADM" = xt ; then
1292     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1293     grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1294     >> $CDIR"/summary.txt"
1295     fi
1296 edhill 1.34
1297     for ex in $extra_runs ; do
1298 jmc 1.73 unset run
1299 jmc 1.94 results=$EMPTY_RESULTS
1300 jmc 1.106 # reference output file
1301     refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1302 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1303     rel_CDIR=$DRESULTS"/"$dir"."$ex
1304     mkdir $rel_CDIR
1305     CDIR=`pwd`"/$rel_CDIR"
1306 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1307 jmc 1.106 run_clean $dir/tr_run.$ex
1308 jmc 1.108 linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
1309 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1310 jmc 1.106 && results=`testoutput_run $dir tr_run.$ex $refExOut`
1311 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1312     fres="$fres.$ex"
1313     echo
1314 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1315     touch $CDIR"/summary.txt"
1316 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1317 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1318     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1319     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1320 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1321 jmc 1.108 if test "x$ADM" = xt ; then
1322     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1323     grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
1324     >> $CDIR"/summary.txt"
1325     fi
1326 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1327 jmc 1.100 run_clean $dir/tr_run.$ex
1328 jmc 1.83 fi
1329 edhill 1.34 done
1330 edhill 1.68
1331 jmc 1.83 #postclean $dir/$builddir
1332     if test "x$POSTCLEAN" = xt ; then
1333     makeclean $dir/$builddir \
1334     && run_clean $dir/$rundir
1335     fi
1336 edhill 1.10
1337     echo "-------------------------------------------------------------------------------"
1338    
1339 edhill 1.1 done
1340    
1341 edhill 1.34 printf "Start time: " >> $SUMMARY
1342 jmc 1.114 echo "$start_date" >> $SUMMARY
1343 edhill 1.34 printf "End time: " >> $SUMMARY
1344 edhill 1.13 date >> $SUMMARY
1345 edhill 1.20
1346     # If addresses were supplied and mpack built successfully, then try
1347     # to send email using mpack.
1348     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1349     echo "No results email was sent."
1350     else
1351     if test "x$HAVE_MPACK" = xt ; then
1352     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1353     && gzip $DRESULTS".tar" \
1354 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1355 edhill 1.20 RETVAL=$?
1356     if test "x$RETVAL" != x0 ; then
1357     echo
1358     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1359     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1360     echo " summary of results from the directory \"$DRESULTS\"."
1361     echo
1362     else
1363     echo
1364     echo "An email containing results was sent to the following addresses:"
1365     echo " \"$ADDRESSES\""
1366     echo
1367 jmc 1.129 test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1368     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1369 edhill 1.20 fi
1370     fi
1371     fi
1372 edhill 1.13
1373 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1374 edhill 1.1
1375 edhill 1.12 if test "x$CLEANUP" != xt ; then
1376 jmc 1.102 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1377 edhill 1.25 if test -e tr_out.txt ; then
1378     mv tr_out.txt tr_out.txt.old
1379 edhill 1.14 fi
1380 jmc 1.103 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1381 edhill 1.12 fi
1382 edhill 1.1
1383 edhill 1.40 if test "x$DELDIR" = xt ; then
1384     rm -rf $DRESULTS
1385     fi
1386    

  ViewVC Help
Powered by ViewVC 1.1.22