/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.130 - (hide annotations) (download)
Mon Dec 21 00:01:06 2009 UTC (14 years, 3 months ago) by jmc
Branch: MAIN
CVS Tags: checkpoint62
Changes since 1.129: +5 -5 lines
compatible with shorter monitor prefix for surface forcing (Qnet)

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.130 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.129 2009/12/02 19:16:15 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.122 echo " (-mth) run multi-threaded (using eedata.mth)"
15 jmc 1.83 echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.109 echo " (-gsl) compile with \"-gsl\" flag"
19 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
20 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
21 jmc 1.108 echo " (DEF=\"\" no email is sent)"
22 jmc 1.113 echo " (-mpd|-mpackdir) DIR location of the mpack utility"
23     echo " (DEF=\"../tools/mpack-1.6\")"
24 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
25     echo " (recognized groups: basic, tutorials)"
26     echo " (DEF=\"\" which test all)"
27 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
28     echo " (DEF=\"\" which test all)"
29 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
30     echo " Bourne-compatible \"sh\" shell"
31     echo " (DEF=\"\" for \"bash\")"
32 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
33 utke 1.120 echo " (-oad) perform an OpenAD adjoint run"
34 edhill 1.20 echo " (-command) STRING command to run"
35 edhill 1.6 echo " (DEF=\"make output.txt\")"
36 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
37 edhill 1.8 echo " (DEF=\"make\")"
38 jmc 1.70 echo " (-odir) STRING used to build output directory name"
39     echo " (DEF=\"hostname\")"
40 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
41 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
42 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
43     echo " (DEF=\"12\")"
44 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
45 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
46 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
47     echo " (-nogenmake|-ng) skip the genmake stage"
48     echo " (-noclean|-nc) skip the \"make clean\" stage"
49     echo " (-nodepend|-nd) skip the \"make depend\" stage"
50     echo " (-deldir|-dd) on success, delete the output directory"
51     echo " (-ts) provide timing information per timestep"
52     echo " (-papis) provide MFlop/s per timestep using PAPI"
53     echo " (-pcls) provide MFlop/s per timestep using PCL"
54 edhill 1.6 echo
55 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
56 edhill 1.6 echo "such as:"
57 edhill 1.50 echo
58 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
59     echo " -addr='abc@123.com testing@home.org'"
60 edhill 1.1 echo
61 edhill 1.50 echo "provided that the expression is properly quoted within the current"
62     echo "shell (note the use of single quotes to protect white space)."
63     echo
64 edhill 1.1 exit 1
65     }
66    
67     # build the mpack utility
68     build_mpack()
69     {
70 edhill 1.34 printf "building the mpack utility... "
71 jmc 1.113 MPACK="$MPACKDIR/mpack"
72     if test ! -x $MPACK ; then
73 edhill 1.1 if test ! -d $MPACKDIR ; then
74 edhill 1.20 echo
75 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
76     echo " are you sure this program is being run in the correct "
77     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
78 edhill 1.20 echo
79     HAVE_MPACK=f
80 edhill 1.1 fi
81 edhill 1.26 if test "x$CC" = x ; then
82     export CC=cc
83     fi
84 jmc 1.112 printf "building mpack (using CC=$CC)... "
85 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
86 edhill 1.1 RETVAL=$?
87     if test "x$RETVAL" != x0 ; then
88     echo
89     echo "Error building the mpack tools at: $MPACK_DIR"
90 edhill 1.20 echo
91     HAVE_MPACK=f
92     else
93 edhill 1.25 rm -f tr_build_mpack.out
94 edhill 1.20 HAVE_MPACK=t
95 jmc 1.113 echo "done"
96 edhill 1.1 fi
97 edhill 1.20 else
98     HAVE_MPACK=t
99 jmc 1.113 echo "already exist"
100 edhill 1.1 fi
101     }
102    
103 jmc 1.93 testoutput_var()
104 edhill 1.1 {
105 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
106 edhill 1.1 #
107 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
108 jmc 1.99 # with same output from reference file $dir/results/$reference_output
109 jmc 1.93 # using search strings s1 and text label
110 edhill 1.1
111     if [ $debug -gt 0 ]; then
112 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
113 edhill 1.1 fi
114 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
115     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
116 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
117 jmc 1.100 if [ $lncntA -lt 2 ]; then
118 edhill 1.1 if [ $verbose -gt 0 ]; then
119     echo Not enough lines of output when searching for "$2" 1>&2
120     fi
121     return 99
122     fi
123     else
124 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
125 edhill 1.1 return 99
126     fi
127     if [ $debug -gt 0 ]; then
128 jmc 1.99 echo testoutput_var: grep "$2" $1/results/$5 1>&2
129 edhill 1.1 fi
130 jmc 1.99 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
131 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
132 jmc 1.100 if [ $lncntB -lt 2 ]; then
133 edhill 1.1 if [ $verbose -gt 0 ]; then
134     echo Not enough lines of output when searching for "$2" 1>&2
135     fi
136     return 99
137     fi
138 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
139     if [ $verbose -gt 0 ]; then
140     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
141     fi
142     return 99
143     fi
144 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
145     if [ $has_nan -gt 0 ] ; then
146 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
147 edhill 1.72 return 99
148     fi
149     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
150     if [ $has_inf -gt 0 ] ; then
151 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
152 jmc 1.71 return 99
153     fi
154 edhill 1.1 if [ $debug -gt 0 ]; then
155 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
156 edhill 1.1 fi
157     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
158     if [ $debug -gt 0 ]; then
159 jmc 1.93 echo testoutput_var: compare_lines 1>&2
160 edhill 1.1 fi
161 edhill 1.22 if [ $verbose -gt 1 ]; then
162     cat tmp3.txt 1>&2
163     fi
164     echo "-1" >> tmp3.txt
165 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
166     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
167 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
168 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
169     if [ $verbose -gt 0 ]; then
170 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
171 edhill 1.1 fi
172     digits_of_similarity=99
173     else
174     if [ $verbose -gt 0 ]; then
175 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
176 edhill 1.1 fi
177     fi
178 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
179 edhill 1.1
180     return $digits_of_similarity
181     }
182    
183 jmc 1.93 testoutput_run()
184 edhill 1.1 {
185 jmc 1.99 # testoutput_run directory subdir reference_output
186 edhill 1.1 #
187 jmc 1.93 # test output from 1 run in "directory"
188 jmc 1.103 # --> same processing for adjoint & forward test
189 jmc 1.94 # default list of output variables to be checked:
190     # 1rst : main variable used to decide if it pass or FAIL
191     # others : number of matching digits to be printed in summary.txt
192     listChk=$DEF_CHECK_LIST
193     # load experiment-specific list from file "tr_checklist" (if it exist)
194     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
195     sVar=`echo $listChk | awk '{print $1}'`
196     # remove 1rst var and expand the list: + => min max mean s.d
197 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
198     | sed 's/+//g' | sed "s/^$sVar//"`
199 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
200 jmc 1.117 # check for ptracer output in reference_output file :
201     outpref=$1/results/$3
202     ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
203     ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
204     for ii in $PTRACERS_NUM ; do
205     ptrfound=0
206     for jj in $ptr_mon ; do
207     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
208     tst=`grep $name $outpref | wc -l | awk '{print $1}'`
209     if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
210     done
211     if test $ptrfound = '1' ; then
212     eval "HAVE_PTR0"$ii"=t"
213     else
214     eval "HAVE_PTR0"$ii"=f"
215 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
216 jmc 1.117 # remove this ptr from the list of output variable to check
217     # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
218     listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
219     fi
220     fi
221     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
222     done
223 mlosch 1.116 tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
224 jmc 1.94 if test $tst != 1 ; then
225     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
226     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
227     echo "==> WARNING: in checked list:" $listVar 1>&2
228     #- put it back once:
229     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
230     fi
231     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
232     echo "listVar='$listVar'" > $CDIR"/summary.txt"
233     allargs=""
234     for xx in $listVar
235     do
236     case $xx in
237     'PS') if [ $debug -gt 0 ]
238     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
239 jmc 1.99 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
240 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
241 jmc 1.101 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
242     'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
243 jmc 1.99 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
244     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
245     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
246     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
247     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
248     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
249     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
250     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
251 jmc 1.94 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
252     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
253     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
254 jmc 1.99 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
255 jmc 1.94 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
256     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
257     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
258 jmc 1.99 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
259 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
260     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
261     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
262     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
263     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
264     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
265     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
266 jmc 1.99 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
267 jmc 1.130 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;;
268     'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;;
269     'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;;
270     'Qntsd') testoutput_var $1 "forcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;;
271 jmc 1.101 'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;;
272     'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;;
273     'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;;
274     'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;;
275     'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;;
276 jmc 1.126 'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Heff max" $2 $3; yy=$? ;;
277     'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Heff mean" $2 $3; yy=$? ;;
278     'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Heff StDv" $2 $3; yy=$? ;;
279 jmc 1.101 'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
280     'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;;
281     'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;;
282     'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;;
283     'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;;
284     'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;;
285 jmc 1.94 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
286     esac
287     if test $xx = $sVar
288     then allargs="$allargs > $yy <"
289     else allargs="$allargs $yy"
290     fi
291     done
292    
293     nbVar=`echo $listVar | awk '{print NF}'`
294 jmc 1.97 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
295 jmc 1.94 #-- fill line (up to standard length) with dot:
296 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
297 jmc 1.94 echo $allargs $adNul
298     else
299     echo $allargs
300     fi
301 jmc 1.103 # <-- same processing for adjoint & forward test
302 edhill 1.1 }
303    
304     genmakemodel()
305     {
306     # genmakemodel directory
307 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
308     echo "genmake skipped!"
309     else
310 edhill 1.34 if test "x$BASH" = x ; then
311     GENMAKE2="../../../tools/genmake2"
312     else
313     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
314     fi
315 edhill 1.10 (
316     cd $1;
317 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
318 utke 1.120 if test "x$ADM" = xt ; then
319     command="$command --mods=../code_ad"
320     elif test "x$OADM" = xt ; then
321     command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
322     else
323 jmc 1.124 command="$command -mods=../code"
324 edhill 1.24 fi
325 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
326 jmc 1.124 command="$command -optfile=$OPTFILE"
327 edhill 1.10 fi
328     if test "x$IEEE" != x ; then
329     command="$command -ieee"
330     fi
331 jmc 1.109 if test "x$GSL" = xt ; then
332     command="$command -gsl"
333     fi
334 edhill 1.66 if test "x$MPI" = xt ; then
335     command="$command -mpi"
336     fi
337 jmc 1.122 if test "x$MULTI_THREAD" = xt ; then
338     #- run multi-threaded using OpenMP:
339     command="$command -omp"
340     fi
341 ce107 1.90 if test "x$TS" = xt ; then
342     command="$command -ts"
343     fi
344     if test "x$PAPIS" = xt ; then
345     command="$command -papis"
346     else
347     if test "x$PCLS" = xt ; then
348     command="$command -pcls"
349     fi
350     fi
351 edhill 1.10 printf 'genmake ... ' 1>&2
352 utke 1.120 # $command > make.log 2>&1
353     eval $command > make.log 2>&1
354 edhill 1.10 RETVAL=$?
355 edhill 1.44 # Reduce the size of the testing emails!
356 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
357 edhill 1.10 if test "x$RETVAL" != x0 ; then
358     tail make.log
359     echo "genmakemodel: genmake failed" 1>&2
360 edhill 1.21 cp genmake_* make.log $CDIR
361 edhill 1.10 return 1
362     else
363 edhill 1.20 echo "successful" 1>&2
364 edhill 1.1 fi
365 edhill 1.10 )
366     fi
367 edhill 1.1 }
368    
369     makeclean()
370     {
371     # makeclean directory
372 edhill 1.10 if test "x$NOCLEAN" = xt ; then
373 jmc 1.83 echo "make Clean skipped!"
374 edhill 1.10 else
375     (
376     cd $1;
377 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
378 edhill 1.10 if test -r Makefile ; then
379 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
380     $MAKE Clean >> make.log 2>&1
381 edhill 1.10 RETVAL=$?
382     if test "x$RETVAL" != x0 ; then
383     tail make.log
384 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
385 edhill 1.10 cp make.log $CDIR"/make.log"
386     return 1
387     fi
388 jmc 1.127 echo successful 1>&2
389     else
390     echo ''
391 edhill 1.10 fi
392     exit 0
393     )
394     fi
395     }
396    
397 jmc 1.83 run_clean()
398 edhill 1.68 {
399 jmc 1.83 # run_clean directory
400     if test "x$NOCLEAN" = xt ; then
401     echo "run_clean skipped!"
402     else
403 edhill 1.68 (
404     cd $1;
405 jmc 1.83 printf 'clean run-dir ... ' 2>&1
406     # part of what is done after "make clean" when doing "make CLEAN"
407     find . -name "*.meta" -exec rm {} \;
408     find . -name "*.data" -exec rm {} \;
409     find . -name "fort.*" -exec rm {} \;
410     find . -type l -exec rm {} \;
411     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
412     rm -rf mnc_test_*
413 jmc 1.107 rm -f *_MIT_CE_000.opt0000 costfunction*0000
414 jmc 1.83 echo successful 1>&2
415 edhill 1.68 exit 0
416     )
417     fi
418     }
419    
420 edhill 1.10 makedependmodel()
421     {
422     # makedependmodel directory
423     if test "x$NODEPEND" = xt ; then
424     echo "make depend skipped!"
425     else
426     (
427     cd $1;
428     printf 'make depend ... ' 1>&2
429     $MAKE depend >> make.log 2>&1
430 edhill 1.1 RETVAL=$?
431     if test "x$RETVAL" != x0 ; then
432     tail make.log
433 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
434 edhill 1.1 cp make.log $CDIR"/make.log"
435     return 1
436 edhill 1.10 else
437 edhill 1.20 echo successful 1>&2
438 edhill 1.1 fi
439 edhill 1.10 )
440     fi
441 edhill 1.1 }
442    
443     makemodel()
444     {
445     # makemodel directory
446     (
447     cd $1;
448     if test -r Makefile ; then
449     printf 'make ... ' 1>&2
450 utke 1.120 if test "x$ADM" = xt ; then
451     $MAKE adall >> make.log 2>&1
452     elif test "x$OADM" = xt ; then
453     $MAKE adAll >> make.log 2>&1
454     else
455 edhill 1.43 if test "x$JOBS" = x ; then
456     $MAKE >> make.log 2>&1
457     else
458     $MAKE -j $JOBS >> make.log 2>&1
459     fi
460 edhill 1.24 fi
461 edhill 1.1 RETVAL=$?
462     if test "x$RETVAL" != x0 ; then
463     tail make.log
464     echo failed 1>&2
465     cp make.log $CDIR"/make.log"
466     return 1
467     else
468 edhill 1.20 echo successful 1>&2
469 edhill 1.1 fi
470     fi
471     )
472     }
473    
474 edhill 1.27 symlink_mpifiles()
475     {
476     # Put special links so that MPI specific files are used
477     # This MUST be invoked between makeclean and makelinks because
478     # the Makefile will link to non-mpi files by default
479    
480     dir=$1
481     code_dir=$2
482     BUILD_DIR=$dir/$3
483     CODE_DIR=$dir/$code_dir
484 edhill 1.45
485 edhill 1.27 # These are files that should replace their counter-part when using -mpi
486 mlosch 1.111 MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`
487 edhill 1.27
488     # Is this an MPI run?
489     if test "x$MPI" = xt ; then
490     # YES: We symbolically link these files to the build
491     # dir so long as there is no real file in place
492     for ii in $MPI_FILES ; do
493     i=`echo $ii | sed 's:^\./::'`
494     name=`echo $i | sed 's:_mpi::' `
495     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
496     RETVAL=$?
497     if test "x$RETVAL" != x0 ; then
498     if ! test -f $BUILD_DIR/$i ; then
499 edhill 1.45 #echo Linking $name to $i
500 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
501     fi
502     fi
503     done
504     else
505 edhill 1.45 # NO: We undo any _mpi symbolically linked files
506 edhill 1.27 for ii in $MPI_FILES ; do
507     i=`echo $ii | sed 's:^\./::'`
508     name=`echo $i | sed 's:_mpi::' `
509     if test -L $BUILD_DIR/$name ; then
510 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
511     RETVAL=$?
512     if test "x$RETVAL" = x0 ; then
513     #echo Un-linking $name from $linktarg
514 edhill 1.27 rm -f $BUILD_DIR/$name
515     fi
516     fi
517     done
518     fi
519    
520     }
521    
522 edhill 1.1 linkdata()
523     {
524 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
525 edhill 1.1 #
526     # symbolically link data files to run directory
527 jmc 1.83 if test -d $1 ; then
528 edhill 1.12 (
529 jmc 1.83 cd $1 ; shift
530 jmc 1.125 inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
531     for xx in $inpMPI ; do
532     if test -r "../"$1"/"$xx ; then
533     # found 1 _mpi sfx file in 1rst input dir and it is readable
534     yy=`echo $xx | sed 's:\.mpi$::'`
535     if test "x$MPI" = "xt" ; then
536     # mpi test: remove symbolic link & link _mpi sfx file
537     if test -h $yy ; then rm -f $yy ; fi
538     if test ! -r $yy ; then
539     ln -sf "../"$1"/"$xx $yy ;
540     printf "$xx " 1>&2
541     fi
542     else
543     # not mpi test: remove symbolic link
544     if test -h $yy ; then rm -f $yy ; fi
545     fi
546     fi
547     done
548 jmc 1.83 if test -r "../"$1"/eedata.mth" ; then
549     # found eedata.mth in 1rst input dir and it is readable
550     if test "x$MULTI_THREAD" = "xt" ; then
551     # multi-threaded test: remove symbolic link & link eedata.mth
552     if test -h eedata ; then rm -f eedata ; fi
553     if test ! -r eedata ; then
554     ln -sf "../"$1"/eedata.mth" eedata ;
555     printf 'eedata.mth ' 1>&2
556 edhill 1.24 fi
557 jmc 1.83 else
558     # not multi-threaded test: remove eedata symbolic link
559     if test -h eedata ; then rm -f eedata ; fi
560     fi
561     fi
562 jmc 1.108 prevDir='NONE'
563 jmc 1.83 for ldir in $* ; do
564 jmc 1.108 if test -d "../"$ldir -a $ldir != $prevDir ; then
565 jmc 1.83 printf 'ldir='${ldir} 1>&2
566     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
567     for i in $files ; do
568     if test ! -d "../"$ldir/$i ; then
569     if test ! -r $i ; then
570     printf ' '$i 1>&2
571     ln -sf "../"$ldir"/"$i $i
572     fi
573     fi
574     done
575     if test -x "../"$ldir"/"prepare_run ; then
576     "../"$ldir"/"prepare_run
577 edhill 1.24 fi
578 jmc 1.83 printf ' ; ' 1>&2
579 jmc 1.82 fi
580 jmc 1.108 prevDir=$ldir
581 jmc 1.83 done
582 edhill 1.12 )
583 edhill 1.1 fi
584     }
585    
586     runmodel()
587     {
588 edhill 1.6 # runmodel directory
589 edhill 1.1 #
590 edhill 1.24 # runs "$COMMAND in "directory"
591 edhill 1.6 # (where "$COMMAND" is relative to "directory")
592 edhill 1.1 (
593     cd $1
594 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
595 edhill 1.6 # make output.txt
596 edhill 1.38 echo
597 jmc 1.106 if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
598     diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
599 jmc 1.119 outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi
600 jmc 1.106 fi
601 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
602 jmc 1.119 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
603 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
604     fi
605     if test ! -x $EXECUTABLE ; then
606 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
607     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
608     echo " no executable:" $EXECUTABLE >> $RUNLOG
609 jmc 1.83 RETVAL=8
610 jmc 1.88 ENDVAL=-1
611 jmc 1.75 else
612 jmc 1.98 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
613     # output do not exist or is older than executable:
614 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
615     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
616     ( eval $COMMAND ) >> $RUNLOG 2>&1
617 jmc 1.83 RETVAL=$?
618     else
619     RETVAL=0
620 jmc 1.119 if test -f $RUNLOG ; then
621     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
622     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
623 jmc 1.88 else
624 jmc 1.119 touch $RUNLOG
625     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
626     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
627     echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1
628 jmc 1.88 fi
629 jmc 1.83 fi
630 jmc 1.119 ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
631 jmc 1.75 fi
632 jmc 1.119 rm -f run.log_tmp
633 jmc 1.88 #if test "x$RETVAL" = x0 ; then
634     if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
635 jmc 1.119 tail $RUNLOG
636 edhill 1.20 echo successful 1>&2
637 edhill 1.44 # === Reduce the size of the testing emails!
638 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
639 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
640 edhill 1.6 return 0
641     else
642 jmc 1.119 tail $RUNLOG
643 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
644 jmc 1.119 cp $RUNLOG $CDIR"/"$RUNLOG
645 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
646 edhill 1.6 return 1
647 edhill 1.1 fi
648     )
649     }
650    
651     createcodelet()
652     {
653     # create codelet for comparing model output
654    
655 jmc 1.112 printf "creating the comparison code (using CC=$CC)... "
656 jmc 1.91 cat > tr_cmpnum.c <<EOF
657 edhill 1.22 #include <stdio.h>
658     #include <math.h>
659     int main( int argc, char** argv ) {
660 jmc 1.91 int linnum,cmplin,best,lncnt;
661 edhill 1.52 double a,b,abave,relerr;
662 jmc 1.57 best = -22;
663 adcroft 1.33 lncnt = 0;
664 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
665 edhill 1.22 scanf("%d", &linnum);
666     if (linnum == -1) break;
667     scanf("%lf", &a); scanf("%lf", &b);
668 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
669 jmc 1.91 if ( abave == abave ) {
670     if (abave > 0.0) {
671     relerr=fabs(a-b)/abave;
672     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
673     else { cmplin = -16 ; }
674     best = (best > cmplin) ? best : cmplin; }
675     else { cmplin = -22 ; }
676     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
677     }
678     else {
679     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
680     break; }
681 edhill 1.22 }
682 adcroft 1.33 if (lncnt == 999) best=-29;
683 jmc 1.91 if (linnum != -1) best=-99;
684 edhill 1.22 printf("%d\n", -best);
685     return 0;
686     }
687     EOF
688 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
689 edhill 1.1
690 jmc 1.91 if [ -x ./tr_cmpnum ]; then
691 edhill 1.1 echo "OK"
692     return 0
693     else
694     echo
695 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
696     echo " a C compiler using the CC environment variable."
697 edhill 1.1 exit 1
698     fi
699     }
700    
701     formatresults()
702     {
703     # formatresults expt genmake depend make run results*
704    
705     nm=$1
706     printf '%s %s %s %s' $2 $3 $4 $5
707     shift; shift; shift; shift; shift;
708 jmc 1.94 listPrt=$@
709     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
710     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
711 jmc 1.97 printf '%3s' $listPrt
712 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
713 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
714 edhill 1.1
715 jmc 1.94 if [ $xx = '..' ]; then
716     printf ' N/O '
717     elif [ $xx = '--' ]; then
718     printf ' N/O '
719     elif [ $xx = 99 ]; then
720 edhill 1.1 printf ' N/O '
721     else
722 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
723 edhill 1.1 printf ' pass'
724     else
725     printf ' FAIL'
726     fi
727     fi
728     printf ' %s' $nm
729     printf '\n'
730    
731     }
732    
733     scandirs()
734     {
735 jmc 1.81 if [ $# -eq 1 ]; then
736     for arg in * ; do
737 jmc 1.99 test -f $arg/$1 && echo $arg
738 jmc 1.81 done
739 edhill 1.1 else
740 jmc 1.81 echo $*
741 edhill 1.1 fi
742     }
743    
744    
745 jmc 1.123 check_eedata()
746     {
747     # check_eedata size.h eedata
748     if [ $# -eq 2 ] ; then
749     if test -f $1 -a -f $2 ; then
750     sx=`grep "^ & *nSx *=" $1 | sed "s/^ & *nSx *=//" | sed 's/, *$//'`
751     nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
752     if test "x$nx" = x ; then
753     rx=10
754     else
755     rx=`expr $sx % $nx`
756     fi
757     sy=`grep "^ & *nSy *=" $1 | sed "s/^ & *nSy *=//" | sed 's/, *$//'`
758     ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
759     if test "x$ny" = x ; then
760     ry=20
761     else
762     ry=`expr $sy % $ny`
763     fi
764     echo `expr $rx + $ry`
765     else
766     echo '-1'
767     fi
768     else
769     echo '-2'
770     fi
771    
772     }
773    
774 edhill 1.1 ###############################################################################
775    
776    
777     # Default properties
778     debug=0
779     verbose=1
780     clean=0
781 edhill 1.10
782 edhill 1.40 IEEE=true
783 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
784     IEEE=$MITGCM_IEEE
785     fi
786 jmc 1.109 GSL=f
787 edhill 1.10
788     CLEANUP=f
789     QUICK=f
790     NOGENMAKE=f
791     NOCLEAN=f
792     NODEPEND=f
793 edhill 1.68 POSTCLEAN=f
794 edhill 1.1
795 edhill 1.4 BASH=
796 edhill 1.10 OPTFILE=NONE
797     ADDRESSES=
798 edhill 1.1 TESTDIRS=
799 jmc 1.89 SKIPDIRS=
800 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
801 edhill 1.20 HAVE_MPACK=
802 jmc 1.113 MPACK=
803 edhill 1.24 COMMAND=
804 edhill 1.59 if test "x$MAKE" = x ; then
805     MAKE=make
806     fi
807     if test "x$CC" = x ; then
808     CC=cc
809     fi
810 edhill 1.43 JOBS=
811 edhill 1.7 MPI=f
812 jmc 1.83 MULTI_THREAD=f
813 jmc 1.70 OUTDIR=
814 edhill 1.40 DELDIR=
815 edhill 1.1
816 edhill 1.24 ADM=
817 utke 1.120 OADM=
818 edhill 1.24
819 jmc 1.117 # list of pTracers to check for monitor output
820 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
821 edhill 1.49
822 jmc 1.92 MATCH_CRIT=13
823    
824 edhill 1.34 printf "parsing options... "
825 edhill 1.1
826     ac_prev=
827     for ac_option ; do
828    
829     # If the previous option needs an argument, assign it.
830     if test -n "$ac_prev"; then
831     eval "$ac_prev=\$ac_option"
832     ac_prev=
833     continue
834     fi
835    
836     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
837    
838     case $ac_option in
839    
840     -help | --help | -h | --h)
841     usage ;;
842    
843 edhill 1.2 -optfile | --optfile | -of | --of)
844 edhill 1.10 ac_prev=OPTFILE ;;
845 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
846 edhill 1.10 OPTFILE=$ac_optarg ;;
847 edhill 1.1
848     -addr | --addr | -a | --a)
849     ac_prev=ADDRESSES ;;
850     -addr=* | --addr=*)
851     ADDRESSES=$ac_optarg ;;
852 jmc 1.113 -mpackdir | --mpackdir | -mpd | --mpd)
853     ac_prev=MPACKDIR ;;
854     -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
855     MPACKDIR=$ac_optarg ;;
856 edhill 1.1
857     -tdir | --tdir | -t | --t)
858     ac_prev=TESTDIRS ;;
859     -tdir=* | --tdir=*)
860     TESTDIRS=$ac_optarg ;;
861 edhill 1.4
862 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
863     ac_prev=SKIPDIRS ;;
864     -skipdir=* | --skipdir=*)
865     SKIPDIRS=$ac_optarg ;;
866    
867 edhill 1.4 -bash | --bash | -b | --b)
868     ac_prev=BASH ;;
869     -bash=* | --bash=*)
870     BASH=$ac_optarg ;;
871 edhill 1.5
872 edhill 1.6 -command | --command | -c | --c)
873     ac_prev=COMMAND ;;
874     -command=* | --command=*)
875     COMMAND=$ac_optarg ;;
876 edhill 1.8
877     -make | --make | -m | --m)
878     ac_prev=MAKE ;;
879     -make=* | --make=*)
880     MAKE=$ac_optarg ;;
881 edhill 1.1
882 jmc 1.70 -odir | --odir)
883     ac_prev=OUTDIR ;;
884     -odir=* | --odir=*)
885     OUTDIR=$ac_optarg ;;
886    
887 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
888     ac_prev=PTRACERS_NUM ;;
889     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
890     PTRACERS_NUM=$ac_optarg ;;
891    
892 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
893     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
894    
895 edhill 1.43 -j) ac_prev=JOBS ;;
896     -j=*) JOBS=$ac_optarg ;;
897    
898 edhill 1.10 -clean | --clean)
899 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
900 edhill 1.10
901     -quick | --quick | -q | --q)
902     QUICK=t ;;
903     -nogenmake | --nogenmake | -ng | --ng)
904     NOGENMAKE=t ;;
905     -noclean | --noclean | -nc | --nc)
906     NOCLEAN=t ;;
907     -nodepend | --nodepend | -nd | --nd)
908     NODEPEND=t ;;
909    
910 edhill 1.68 -postclean | --postclean | -pc | --pc)
911     POSTCLEAN=t ;;
912    
913 edhill 1.18 -mpi) MPI=t ;;
914 edhill 1.10
915 jmc 1.83 -mth) MULTI_THREAD=t ;;
916    
917 edhill 1.24 -adm | -ad) ADM=t ;;
918    
919 utke 1.120 -oad) OADM=t; NODEPEND=t ;;
920    
921 edhill 1.10 -ieee) IEEE=true ;;
922     -noieee) IEEE= ;;
923 jmc 1.109 -gsl) GSL=t ;;
924 edhill 1.10
925 edhill 1.1 -verbose) verbose=2 ;;
926     -debug) debug=1 ;;
927     -quiet) verbose=0 ;;
928    
929 edhill 1.40 -deldir | -dd) DELDIR=t ;;
930    
931 ce107 1.90 -ts) TS=t;;
932    
933     -papis) PAPIS=t;;
934    
935     -pcls) PCL=t;;
936    
937 edhill 1.1 -*)
938     echo "Error: unrecognized option: "$ac_option
939     usage
940     ;;
941    
942     *)
943     echo "Error: unrecognized argument: "$ac_option
944     usage
945     ;;
946    
947     esac
948    
949     done
950    
951 edhill 1.10 if test "x$QUICK" = xt ; then
952     NOGENMAKE=t
953     NOCLEAN=t
954     NODEPEND=t
955     fi
956    
957 jmc 1.99 #- setting for forward or ADM testing
958     if test "x$ADM" = xt ; then
959     code_dir=code_ad
960 jmc 1.108 inputdir=input_ad
961 jmc 1.99 ref_outp="output_adm.txt"
962     EXECUTABLE="mitgcmuv_ad"
963 utke 1.120 elif test "x$OADM" = xt ; then
964     code_dir=code_oad
965     inputdir=input_oad
966     ref_outp="output_oadm.txt"
967     EXECUTABLE="mitgcmuv_ad"
968 jmc 1.99 else
969     code_dir=code
970 jmc 1.108 inputdir=input
971 jmc 1.99 ref_outp="output.txt"
972     EXECUTABLE="mitgcmuv"
973     fi
974    
975 edhill 1.1 if test "x$TESTDIRS" = x ; then
976 jmc 1.103 LIST=`scandirs results/$ref_outp`
977 jmc 1.84 else
978     #- expand group of experiments:
979     LIST=" "
980     for xx in $TESTDIRS
981     do
982     case $xx in
983 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
984     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
985     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
986 jmc 1.84 ;;
987     'tutorials')
988     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
989     *) LIST=${LIST}" "$xx ;;
990     esac
991     done
992 jmc 1.89 fi
993     #echo 'LIST='${LIST}'<'
994     #- skip dirs, remove duplicate and non-directory:
995     TESTDIRS=" "
996     count=0
997     for xx in $LIST
998     do
999     yy=`echo $SKIPDIRS | grep -c $xx`
1000     if test $yy = 0 ; then
1001 jmc 1.84 if test -d $xx ; then
1002     yy=`echo $TESTDIRS | grep -c $xx`
1003     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
1004 jmc 1.89 else count=1 ;
1005     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
1006     fi
1007     else
1008     if test $count = 1 ; then echo -n ", \"$xx\""
1009     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
1010 jmc 1.84 fi
1011 jmc 1.89 fi
1012     done
1013     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
1014 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
1015 edhill 1.1
1016 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
1017     OPTFILE=$MITGCM_OF
1018     fi
1019    
1020 jmc 1.119 RUNLOG="run.log"
1021 jmc 1.99 OUTPUTFILE=$ref_outp
1022 jmc 1.87 if test "x$COMMAND" = x ; then
1023 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
1024 edhill 1.24 fi
1025 jmc 1.87 if test "x$MPI" = xt ; then
1026     OUTPUTFILE="STDOUT.0000"
1027 edhill 1.24 fi
1028    
1029 jmc 1.76 echo "OK (COMMAND= $COMMAND )"
1030 edhill 1.1
1031 jmc 1.94 # set the Default List of output variables to be checked:
1032     # (use default or load experiment-specific list from file "tr_checklist")
1033     # content : 1rst = main variable used to decide if it pass or FAIL
1034     # others = number of matching digits to be printed in summary.txt
1035 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1036 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
1037     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
1038 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1039 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
1040 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
1041 jmc 1.99 else
1042 jmc 1.103 DEF_CHECK_LIST='Grad Cost Grad'
1043 jmc 1.99 EMPTY_RESULTS='.. ..'
1044     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1045 jmc 1.94 fi
1046    
1047 edhill 1.1 # create the FORTRAN comparison code
1048     createcodelet
1049    
1050 jmc 1.112 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
1051     if test "x$ADDRESSES" = x ; then
1052 edhill 1.32 echo "skipping mpack build"
1053     else
1054 edhill 1.31 build_mpack
1055     fi
1056 edhill 1.1
1057     # Create a uniquely named directory to store results
1058 jmc 1.110 CMDLINE=$0
1059 jmc 1.118 for xx in "$@" ; do nw=`echo $xx | wc -w`
1060     if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
1061     else CMDLINE="$CMDLINE '$xx'" ; fi
1062     done
1063     #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
1064 edhill 1.1 MACH=`hostname`
1065 edhill 1.2 UNAMEA=`uname -a`
1066 edhill 1.1 DATE=`date +%Y%m%d`
1067 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1068 jmc 1.70 if test "x$OUTDIR" != x ; then
1069     BASE="tr_"$OUTDIR"_"$DATE"_"
1070 jmc 1.121 else
1071     short_name=`hostname | sed 's/\..*$//'`
1072     BASE="tr_"$short_name"_"$DATE"_"
1073 jmc 1.70 fi
1074 edhill 1.1 DNUM=0
1075     DRESULTS="$BASE$DNUM"
1076     while test -e $DRESULTS ; do
1077     DNUM=$(( $DNUM + 1 ))
1078     DRESULTS="$BASE$DNUM"
1079     done
1080     mkdir $DRESULTS
1081     RETVAL=$?
1082     if test "x$RETVAL" != x0 ; then
1083 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1084 edhill 1.1 exit 1
1085     fi
1086     SUMMARY="$DRESULTS/summary.txt"
1087 edhill 1.16 start_date=`date`
1088 edhill 1.17 echo $start_date > $SUMMARY
1089 jmc 1.110 echo 'run:' $CMDLINE >> $SUMMARY
1090     echo 'on :' $UNAMEA >> $SUMMARY
1091 edhill 1.1
1092 edhill 1.11 of_path=
1093 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1094     if test -r $OPTFILE ; then
1095 edhill 1.11 # get the path
1096     path=${OPTFILE%/*}
1097     if test "x$path" = x ; then
1098     of_path=`pwd`
1099     else
1100     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1101     fi
1102     file=${OPTFILE##*/}
1103     OPTFILE=$of_path/$file
1104 edhill 1.21 cp $OPTFILE $DRESULTS
1105     echo >> $SUMMARY
1106     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1107 edhill 1.11 else
1108 edhill 1.21 echo | tee $SUMMARY
1109     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1110     exit 1
1111 edhill 1.10 fi
1112 edhill 1.21 else
1113     echo >> $SUMMARY
1114     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1115     echo " so the genmake default will be used." >> $SUMMARY
1116 edhill 1.10 fi
1117     echo
1118     echo >> $SUMMARY
1119 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1120 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1121 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1122     else line_0="default "$MATCH_CRIT ; fi
1123     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1124 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1125 edhill 1.49 line_1="G D M c m s m s m s m s"
1126     line_2="E p a R g m m e . m m e . m m e . m m e ."
1127     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1128     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1129 edhill 1.50 for ii in $PTRACERS_NUM ; do
1130 edhill 1.49 line_0="$line_0 --PTR 0"$ii"--"
1131     line_1="$line_1 m s"
1132     line_2="$line_2 m m e ."
1133     line_3="$line_3 i a a d"
1134     line_4="$line_4 n x n ."
1135     done
1136     echo "$line_0" | tee -a $SUMMARY
1137     echo "$line_1" | tee -a $SUMMARY
1138     echo "$line_2" | tee -a $SUMMARY
1139     echo "$line_3" | tee -a $SUMMARY
1140     echo "$line_4" | tee -a $SUMMARY
1141     echo " " | tee -a $SUMMARY
1142 edhill 1.24 else
1143     echo "ADJOINT=true" >> $SUMMARY
1144     echo >> $SUMMARY
1145 jmc 1.100 if [ $MATCH_CRIT -lt 10 ] ;
1146 jmc 1.103 then line_0="default "$MATCH_CRIT ;
1147     else line_0="default "$MATCH_CRIT ; fi
1148 jmc 1.100 echo "$line_0" | tee -a $SUMMARY
1149 edhill 1.24 cat << EOF | tee -a $SUMMARY
1150     G D M C G
1151     E p a R o r
1152     N n k u s a
1153     2 d e n t d
1154    
1155     EOF
1156     fi
1157 edhill 1.1
1158 edhill 1.10 # ...and each test directory...
1159     for dir in $TESTDIRS ; do
1160    
1161 jmc 1.106 # set builddir & rundir:
1162     builddir="build"
1163     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1164     rundir="run"
1165     if test ! -d $dir/$rundir ; then
1166     rundir=$builddir
1167     fi
1168     CODE_DIR=$dir/$code_dir
1169     BUILD_DIR=$dir/$builddir
1170    
1171 edhill 1.10 # Cleanup only!
1172     if test "x$CLEANUP" = xt ; then
1173 jmc 1.127 echo -n ' --- dir:' $BUILD_DIR ': '
1174     makeclean $BUILD_DIR
1175 jmc 1.128 rm -f $BUILD_DIR/$EXECUTABLE
1176 jmc 1.106 if test -d $dir/$rundir/CVS ; then
1177 jmc 1.127 echo -n ' --- dir:' $dir/$rundir ': '
1178 jmc 1.106 run_clean $dir/$rundir
1179 edhill 1.10 fi
1180 jmc 1.127 trdir=`( cd $dir ; find . -type d -name "tr_run.*" -print | sed 's/^.\///')`
1181     ttd=`echo $trdir | wc -w`
1182     if test $ttd != 0 ; then
1183     echo ' --- rm dir:' $trdir
1184     ( cd $dir ; rm -rf $trdir )
1185     fi
1186 edhill 1.10 continue
1187 edhill 1.1 fi
1188 edhill 1.3
1189 edhill 1.10 # Verify that the testdir exists and contains previous
1190     # results in the correct location--or skip this directory!
1191 jmc 1.104 fout=$dir"/results/"$ref_outp
1192 edhill 1.24 if test ! -r $fout ; then
1193     echo "can't read \"$fout\" -- skipping $dir"
1194 edhill 1.10 continue
1195     fi
1196 edhill 1.7
1197 jmc 1.117 # Check for specific files for particular type of run
1198 edhill 1.49
1199 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1200 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1201     continue
1202     fi
1203 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1204     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1205     continue
1206     fi
1207 jmc 1.123 if test "x$MPI" = "xt" -a "x$MULTI_THREAD" = "xt" ; then
1208     retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/input/eedata.mth"`
1209     if test $retv != 0 ; then
1210     echo "input/eedata.mth tiling misfit -- skipping $dir"
1211     continue
1212     fi
1213     fi
1214 jmc 1.83
1215     # Check whether there are "extra runs" for this testdir
1216     extra_runs=
1217 jmc 1.108 ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
1218     #echo "ex_run_dirs='$ex_run_dirs'"
1219     for exd in $ex_run_dirs ; do
1220     name=`echo $exd | sed -e "s/$inputdir\.//"`
1221     refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1222     outf="$dir/results/$refExOut"
1223     if test -f $outf -a -r $outf ; then
1224 jmc 1.123 if test "x$MULTI_THREAD" = "xt" ; then
1225     if test -r $dir"/"$exd"/eedata.mth" ; then
1226     if test "x$MPI" = "xt" ; then
1227     retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/"$exd"/eedata.mth"`
1228     if test $retv = 0 ; then
1229 jmc 1.83 extra_runs="$extra_runs $name"
1230 jmc 1.123 else
1231     echo $exd"/eedata.mth tiling misfit -- skipping $dir"
1232 jmc 1.83 fi
1233 jmc 1.123 else
1234 jmc 1.108 extra_runs="$extra_runs $name"
1235 jmc 1.123 fi
1236     #else echo $dir"/"$exd"/eedata.mth: not found"
1237 jmc 1.83 fi
1238 jmc 1.123 else
1239     extra_runs="$extra_runs $name"
1240     fi
1241 jmc 1.108 fi
1242     done
1243 edhill 1.28
1244     echo "-------------------------------------------------------------------------------"
1245     echo
1246 jmc 1.83 if test "x$extra_runs" = "x" ; then
1247     echo "Experiment: $dir"
1248     else
1249     echo "Experiment: $dir ; extra_runs=$extra_runs"
1250     fi
1251 edhill 1.28 echo
1252     unset genmake makedepend make run
1253 jmc 1.94 results=$EMPTY_RESULTS
1254 edhill 1.10
1255     # Create an output dir for each OPTFILE/tdir combination
1256 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1257     mkdir $rel_CDIR
1258     CDIR=`pwd`"/$rel_CDIR"
1259 edhill 1.10
1260     if test "x$CLEANUP" = xt ; then
1261 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1262     makeclean $dir/$builddir \
1263     && run_clean $dir/$rundir
1264 edhill 1.10 else
1265 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1266     && makeclean $dir/$builddir \
1267 jmc 1.83 && run_clean $dir/$rundir \
1268 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1269 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1270     && makemodel $dir/$builddir && make=Y \
1271 jmc 1.108 && linkdata $dir/$rundir $inputdir input \
1272 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1273 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1274 edhill 1.10 fi
1275    
1276     echo
1277 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1278 edhill 1.34 echo
1279 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1280 jmc 1.100 touch $CDIR"/summary.txt"
1281 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1282 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1283     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1284     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1285     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1286 jmc 1.108 if test "x$ADM" = xt ; then
1287     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1288     grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1289     >> $CDIR"/summary.txt"
1290     fi
1291 edhill 1.34
1292     for ex in $extra_runs ; do
1293 jmc 1.73 unset run
1294 jmc 1.94 results=$EMPTY_RESULTS
1295 jmc 1.106 # reference output file
1296     refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1297 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1298     rel_CDIR=$DRESULTS"/"$dir"."$ex
1299     mkdir $rel_CDIR
1300     CDIR=`pwd`"/$rel_CDIR"
1301 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1302 jmc 1.106 run_clean $dir/tr_run.$ex
1303 jmc 1.108 linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
1304 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1305 jmc 1.106 && results=`testoutput_run $dir tr_run.$ex $refExOut`
1306 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1307     fres="$fres.$ex"
1308     echo
1309 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1310     touch $CDIR"/summary.txt"
1311 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1312 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1313     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1314     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1315 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1316 jmc 1.108 if test "x$ADM" = xt ; then
1317     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1318     grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
1319     >> $CDIR"/summary.txt"
1320     fi
1321 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1322 jmc 1.100 run_clean $dir/tr_run.$ex
1323 jmc 1.83 fi
1324 edhill 1.34 done
1325 edhill 1.68
1326 jmc 1.83 #postclean $dir/$builddir
1327     if test "x$POSTCLEAN" = xt ; then
1328     makeclean $dir/$builddir \
1329     && run_clean $dir/$rundir
1330     fi
1331 edhill 1.10
1332     echo "-------------------------------------------------------------------------------"
1333    
1334 edhill 1.1 done
1335    
1336 edhill 1.34 printf "Start time: " >> $SUMMARY
1337 jmc 1.114 echo "$start_date" >> $SUMMARY
1338 edhill 1.34 printf "End time: " >> $SUMMARY
1339 edhill 1.13 date >> $SUMMARY
1340 edhill 1.20
1341     # If addresses were supplied and mpack built successfully, then try
1342     # to send email using mpack.
1343     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1344     echo "No results email was sent."
1345     else
1346     if test "x$HAVE_MPACK" = xt ; then
1347     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1348     && gzip $DRESULTS".tar" \
1349 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1350 edhill 1.20 RETVAL=$?
1351     if test "x$RETVAL" != x0 ; then
1352     echo
1353     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1354     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1355     echo " summary of results from the directory \"$DRESULTS\"."
1356     echo
1357     else
1358     echo
1359     echo "An email containing results was sent to the following addresses:"
1360     echo " \"$ADDRESSES\""
1361     echo
1362 jmc 1.129 test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1363     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1364 edhill 1.20 fi
1365     fi
1366     fi
1367 edhill 1.13
1368 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1369 edhill 1.1
1370 edhill 1.12 if test "x$CLEANUP" != xt ; then
1371 jmc 1.102 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1372 edhill 1.25 if test -e tr_out.txt ; then
1373     mv tr_out.txt tr_out.txt.old
1374 edhill 1.14 fi
1375 jmc 1.103 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1376 edhill 1.12 fi
1377 edhill 1.1
1378 edhill 1.40 if test "x$DELDIR" = xt ; then
1379     rm -rf $DRESULTS
1380     fi
1381    

  ViewVC Help
Powered by ViewVC 1.1.22