/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.125 - (hide annotations) (download)
Tue May 12 18:44:24 2009 UTC (14 years, 10 months ago) by jmc
Branch: MAIN
Changes since 1.124: +19 -1 lines
an other hack to link *.mpi files from input dir, when using -mpi option
(similar to eedata.mth with -mth option)

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.125 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.124 2009/04/22 21:07:00 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.122 echo " (-mth) run multi-threaded (using eedata.mth)"
15 jmc 1.83 echo " (-mpi) compile and run using MPI"
16 edhill 1.10 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 edhill 1.41 echo " (DEF=\"-ieee\")"
18 jmc 1.109 echo " (-gsl) compile with \"-gsl\" flag"
19 jmc 1.89 echo " (-of=|-optfile=)STRING list of optfiles to use"
20 edhill 1.20 echo " (-a|-addr) STRING list of email recipients"
21 jmc 1.108 echo " (DEF=\"\" no email is sent)"
22 jmc 1.113 echo " (-mpd|-mpackdir) DIR location of the mpack utility"
23     echo " (DEF=\"../tools/mpack-1.6\")"
24 jmc 1.84 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
25     echo " (recognized groups: basic, tutorials)"
26     echo " (DEF=\"\" which test all)"
27 jmc 1.89 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
28     echo " (DEF=\"\" which test all)"
29 edhill 1.20 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
30     echo " Bourne-compatible \"sh\" shell"
31     echo " (DEF=\"\" for \"bash\")"
32 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
33 utke 1.120 echo " (-oad) perform an OpenAD adjoint run"
34 edhill 1.20 echo " (-command) STRING command to run"
35 edhill 1.6 echo " (DEF=\"make output.txt\")"
36 edhill 1.20 echo " (-m|-make) STRING command to use for \"make\""
37 edhill 1.8 echo " (DEF=\"make\")"
38 jmc 1.70 echo " (-odir) STRING used to build output directory name"
39     echo " (DEF=\"hostname\")"
40 jmc 1.89 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
41 edhill 1.50 echo " (DEF=\"1 2 3 4 5\")"
42 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
43     echo " (DEF=\"12\")"
44 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
45 edhill 1.10 echo " (-clean) *ONLY* run \"make CLEAN\""
46 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
47     echo " (-nogenmake|-ng) skip the genmake stage"
48     echo " (-noclean|-nc) skip the \"make clean\" stage"
49     echo " (-nodepend|-nd) skip the \"make depend\" stage"
50     echo " (-deldir|-dd) on success, delete the output directory"
51     echo " (-ts) provide timing information per timestep"
52     echo " (-papis) provide MFlop/s per timestep using PAPI"
53     echo " (-pcls) provide MFlop/s per timestep using PCL"
54 edhill 1.6 echo
55 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
56 edhill 1.6 echo "such as:"
57 edhill 1.50 echo
58 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
59     echo " -addr='abc@123.com testing@home.org'"
60 edhill 1.1 echo
61 edhill 1.50 echo "provided that the expression is properly quoted within the current"
62     echo "shell (note the use of single quotes to protect white space)."
63     echo
64 edhill 1.1 exit 1
65     }
66    
67     # build the mpack utility
68     build_mpack()
69     {
70 edhill 1.34 printf "building the mpack utility... "
71 jmc 1.113 MPACK="$MPACKDIR/mpack"
72     if test ! -x $MPACK ; then
73 edhill 1.1 if test ! -d $MPACKDIR ; then
74 edhill 1.20 echo
75 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
76     echo " are you sure this program is being run in the correct "
77     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
78 edhill 1.20 echo
79     HAVE_MPACK=f
80 edhill 1.1 fi
81 edhill 1.26 if test "x$CC" = x ; then
82     export CC=cc
83     fi
84 jmc 1.112 printf "building mpack (using CC=$CC)... "
85 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
86 edhill 1.1 RETVAL=$?
87     if test "x$RETVAL" != x0 ; then
88     echo
89     echo "Error building the mpack tools at: $MPACK_DIR"
90 edhill 1.20 echo
91     HAVE_MPACK=f
92     else
93 edhill 1.25 rm -f tr_build_mpack.out
94 edhill 1.20 HAVE_MPACK=t
95 jmc 1.113 echo "done"
96 edhill 1.1 fi
97 edhill 1.20 else
98     HAVE_MPACK=t
99 jmc 1.113 echo "already exist"
100 edhill 1.1 fi
101     }
102    
103 jmc 1.93 testoutput_var()
104 edhill 1.1 {
105 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
106 edhill 1.1 #
107 jmc 1.93 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
108 jmc 1.99 # with same output from reference file $dir/results/$reference_output
109 jmc 1.93 # using search strings s1 and text label
110 edhill 1.1
111     if [ $debug -gt 0 ]; then
112 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
113 edhill 1.1 fi
114 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
115     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
116 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
117 jmc 1.100 if [ $lncntA -lt 2 ]; then
118 edhill 1.1 if [ $verbose -gt 0 ]; then
119     echo Not enough lines of output when searching for "$2" 1>&2
120     fi
121     return 99
122     fi
123     else
124 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
125 edhill 1.1 return 99
126     fi
127     if [ $debug -gt 0 ]; then
128 jmc 1.99 echo testoutput_var: grep "$2" $1/results/$5 1>&2
129 edhill 1.1 fi
130 jmc 1.99 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
131 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
132 jmc 1.100 if [ $lncntB -lt 2 ]; then
133 edhill 1.1 if [ $verbose -gt 0 ]; then
134     echo Not enough lines of output when searching for "$2" 1>&2
135     fi
136     return 99
137     fi
138 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
139     if [ $verbose -gt 0 ]; then
140     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
141     fi
142     return 99
143     fi
144 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
145     if [ $has_nan -gt 0 ] ; then
146 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
147 edhill 1.72 return 99
148     fi
149     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
150     if [ $has_inf -gt 0 ] ; then
151 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
152 jmc 1.71 return 99
153     fi
154 edhill 1.1 if [ $debug -gt 0 ]; then
155 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
156 edhill 1.1 fi
157     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
158     if [ $debug -gt 0 ]; then
159 jmc 1.93 echo testoutput_var: compare_lines 1>&2
160 edhill 1.1 fi
161 edhill 1.22 if [ $verbose -gt 1 ]; then
162     cat tmp3.txt 1>&2
163     fi
164     echo "-1" >> tmp3.txt
165 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
166     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
167 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
168 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
169     if [ $verbose -gt 0 ]; then
170 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
171 edhill 1.1 fi
172     digits_of_similarity=99
173     else
174     if [ $verbose -gt 0 ]; then
175 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
176 edhill 1.1 fi
177     fi
178 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
179 edhill 1.1
180     return $digits_of_similarity
181     }
182    
183 jmc 1.93 testoutput_run()
184 edhill 1.1 {
185 jmc 1.99 # testoutput_run directory subdir reference_output
186 edhill 1.1 #
187 jmc 1.93 # test output from 1 run in "directory"
188 jmc 1.103 # --> same processing for adjoint & forward test
189 jmc 1.94 # default list of output variables to be checked:
190     # 1rst : main variable used to decide if it pass or FAIL
191     # others : number of matching digits to be printed in summary.txt
192     listChk=$DEF_CHECK_LIST
193     # load experiment-specific list from file "tr_checklist" (if it exist)
194     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
195     sVar=`echo $listChk | awk '{print $1}'`
196     # remove 1rst var and expand the list: + => min max mean s.d
197 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
198     | sed 's/+//g' | sed "s/^$sVar//"`
199 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
200 jmc 1.117 # check for ptracer output in reference_output file :
201     outpref=$1/results/$3
202     ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
203     ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
204     for ii in $PTRACERS_NUM ; do
205     ptrfound=0
206     for jj in $ptr_mon ; do
207     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
208     tst=`grep $name $outpref | wc -l | awk '{print $1}'`
209     if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
210     done
211     if test $ptrfound = '1' ; then
212     eval "HAVE_PTR0"$ii"=t"
213     else
214     eval "HAVE_PTR0"$ii"=f"
215 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
216 jmc 1.117 # remove this ptr from the list of output variable to check
217     # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
218     listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
219     fi
220     fi
221     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
222     done
223 mlosch 1.116 tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
224 jmc 1.94 if test $tst != 1 ; then
225     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
226     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
227     echo "==> WARNING: in checked list:" $listVar 1>&2
228     #- put it back once:
229     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
230     fi
231     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
232     echo "listVar='$listVar'" > $CDIR"/summary.txt"
233     allargs=""
234     for xx in $listVar
235     do
236     case $xx in
237     'PS') if [ $debug -gt 0 ]
238     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
239 jmc 1.99 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
240 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
241 jmc 1.101 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
242     'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
243 jmc 1.99 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
244     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
245     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
246     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
247     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
248     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
249     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
250     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
251 jmc 1.94 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
252     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
253     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
254 jmc 1.99 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
255 jmc 1.94 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
256     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
257     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
258 jmc 1.99 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
259 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
260     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
261     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
262     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
263     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
264     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
265     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
266 jmc 1.99 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
267 jmc 1.101 'Qntmn') testoutput_var $1 "extforcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;;
268     'Qntmx') testoutput_var $1 "extforcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;;
269     'Qntav') testoutput_var $1 "extforcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;;
270     'Qntsd') testoutput_var $1 "extforcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;;
271     'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;;
272     'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;;
273     'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;;
274     'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;;
275     'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;;
276     'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Hell max" $2 $3; yy=$? ;;
277     'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Hell mean" $2 $3; yy=$? ;;
278     'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Hell StDv" $2 $3; yy=$? ;;
279     'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
280     'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;;
281     'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;;
282     'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;;
283     'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;;
284     'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;;
285 jmc 1.94 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
286     esac
287     if test $xx = $sVar
288     then allargs="$allargs > $yy <"
289     else allargs="$allargs $yy"
290     fi
291     done
292    
293     nbVar=`echo $listVar | awk '{print NF}'`
294 jmc 1.97 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
295 jmc 1.94 #-- fill line (up to standard length) with dot:
296 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
297 jmc 1.94 echo $allargs $adNul
298     else
299     echo $allargs
300     fi
301 jmc 1.103 # <-- same processing for adjoint & forward test
302 edhill 1.1 }
303    
304     genmakemodel()
305     {
306     # genmakemodel directory
307 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
308     echo "genmake skipped!"
309     else
310 edhill 1.34 if test "x$BASH" = x ; then
311     GENMAKE2="../../../tools/genmake2"
312     else
313     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
314     fi
315 edhill 1.10 (
316     cd $1;
317 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
318 utke 1.120 if test "x$ADM" = xt ; then
319     command="$command --mods=../code_ad"
320     elif test "x$OADM" = xt ; then
321     command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
322     else
323 jmc 1.124 command="$command -mods=../code"
324 edhill 1.24 fi
325 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
326 jmc 1.124 command="$command -optfile=$OPTFILE"
327 edhill 1.10 fi
328     if test "x$IEEE" != x ; then
329     command="$command -ieee"
330     fi
331 jmc 1.109 if test "x$GSL" = xt ; then
332     command="$command -gsl"
333     fi
334 edhill 1.66 if test "x$MPI" = xt ; then
335     command="$command -mpi"
336     fi
337 jmc 1.122 if test "x$MULTI_THREAD" = xt ; then
338     #- run multi-threaded using OpenMP:
339     command="$command -omp"
340     fi
341 ce107 1.90 if test "x$TS" = xt ; then
342     command="$command -ts"
343     fi
344     if test "x$PAPIS" = xt ; then
345     command="$command -papis"
346     else
347     if test "x$PCLS" = xt ; then
348     command="$command -pcls"
349     fi
350     fi
351 edhill 1.10 printf 'genmake ... ' 1>&2
352 utke 1.120 # $command > make.log 2>&1
353     eval $command > make.log 2>&1
354 edhill 1.10 RETVAL=$?
355 edhill 1.44 # Reduce the size of the testing emails!
356 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
357 edhill 1.10 if test "x$RETVAL" != x0 ; then
358     tail make.log
359     echo "genmakemodel: genmake failed" 1>&2
360 edhill 1.21 cp genmake_* make.log $CDIR
361 edhill 1.10 return 1
362     else
363 edhill 1.20 echo "successful" 1>&2
364 edhill 1.1 fi
365 edhill 1.10 )
366     fi
367 edhill 1.1 }
368    
369     makeclean()
370     {
371     # makeclean directory
372 edhill 1.10 if test "x$NOCLEAN" = xt ; then
373 jmc 1.83 echo "make Clean skipped!"
374 edhill 1.10 else
375     (
376     cd $1;
377 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
378 edhill 1.10 if test -r Makefile ; then
379 jmc 1.83 printf 'clean build-dir: make Clean ... ' 2>&1
380     $MAKE Clean >> make.log 2>&1
381 edhill 1.10 RETVAL=$?
382     if test "x$RETVAL" != x0 ; then
383     tail make.log
384 jmc 1.83 echo "makeclean: \"make Clean\" failed" 1>&2
385 edhill 1.10 cp make.log $CDIR"/make.log"
386     return 1
387     fi
388     fi
389 edhill 1.20 echo successful 1>&2
390 edhill 1.10 exit 0
391     )
392     fi
393     }
394    
395 jmc 1.83 run_clean()
396 edhill 1.68 {
397 jmc 1.83 # run_clean directory
398     if test "x$NOCLEAN" = xt ; then
399     echo "run_clean skipped!"
400     else
401 edhill 1.68 (
402     cd $1;
403 jmc 1.83 printf 'clean run-dir ... ' 2>&1
404     # part of what is done after "make clean" when doing "make CLEAN"
405     find . -name "*.meta" -exec rm {} \;
406     find . -name "*.data" -exec rm {} \;
407     find . -name "fort.*" -exec rm {} \;
408     find . -type l -exec rm {} \;
409     rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
410     rm -rf mnc_test_*
411 jmc 1.107 rm -f *_MIT_CE_000.opt0000 costfunction*0000
412 jmc 1.83 echo successful 1>&2
413 edhill 1.68 exit 0
414     )
415     fi
416     }
417    
418 edhill 1.10 makedependmodel()
419     {
420     # makedependmodel directory
421     if test "x$NODEPEND" = xt ; then
422     echo "make depend skipped!"
423     else
424     (
425     cd $1;
426     printf 'make depend ... ' 1>&2
427     $MAKE depend >> make.log 2>&1
428 edhill 1.1 RETVAL=$?
429     if test "x$RETVAL" != x0 ; then
430     tail make.log
431 edhill 1.10 echo "makedependmodel: make depend failed" 1>&2
432 edhill 1.1 cp make.log $CDIR"/make.log"
433     return 1
434 edhill 1.10 else
435 edhill 1.20 echo successful 1>&2
436 edhill 1.1 fi
437 edhill 1.10 )
438     fi
439 edhill 1.1 }
440    
441     makemodel()
442     {
443     # makemodel directory
444     (
445     cd $1;
446     if test -r Makefile ; then
447     printf 'make ... ' 1>&2
448 utke 1.120 if test "x$ADM" = xt ; then
449     $MAKE adall >> make.log 2>&1
450     elif test "x$OADM" = xt ; then
451     $MAKE adAll >> make.log 2>&1
452     else
453 edhill 1.43 if test "x$JOBS" = x ; then
454     $MAKE >> make.log 2>&1
455     else
456     $MAKE -j $JOBS >> make.log 2>&1
457     fi
458 edhill 1.24 fi
459 edhill 1.1 RETVAL=$?
460     if test "x$RETVAL" != x0 ; then
461     tail make.log
462     echo failed 1>&2
463     cp make.log $CDIR"/make.log"
464     return 1
465     else
466 edhill 1.20 echo successful 1>&2
467 edhill 1.1 fi
468     fi
469     )
470     }
471    
472 edhill 1.27 symlink_mpifiles()
473     {
474     # Put special links so that MPI specific files are used
475     # This MUST be invoked between makeclean and makelinks because
476     # the Makefile will link to non-mpi files by default
477    
478     dir=$1
479     code_dir=$2
480     BUILD_DIR=$dir/$3
481     CODE_DIR=$dir/$code_dir
482 edhill 1.45
483 edhill 1.27 # These are files that should replace their counter-part when using -mpi
484 mlosch 1.111 MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`
485 edhill 1.27
486     # Is this an MPI run?
487     if test "x$MPI" = xt ; then
488     # YES: We symbolically link these files to the build
489     # dir so long as there is no real file in place
490     for ii in $MPI_FILES ; do
491     i=`echo $ii | sed 's:^\./::'`
492     name=`echo $i | sed 's:_mpi::' `
493     cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
494     RETVAL=$?
495     if test "x$RETVAL" != x0 ; then
496     if ! test -f $BUILD_DIR/$i ; then
497 edhill 1.45 #echo Linking $name to $i
498 edhill 1.27 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
499     fi
500     fi
501     done
502     else
503 edhill 1.45 # NO: We undo any _mpi symbolically linked files
504 edhill 1.27 for ii in $MPI_FILES ; do
505     i=`echo $ii | sed 's:^\./::'`
506     name=`echo $i | sed 's:_mpi::' `
507     if test -L $BUILD_DIR/$name ; then
508 edhill 1.45 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
509     RETVAL=$?
510     if test "x$RETVAL" = x0 ; then
511     #echo Un-linking $name from $linktarg
512 edhill 1.27 rm -f $BUILD_DIR/$name
513     fi
514     fi
515     done
516     fi
517    
518     }
519    
520 edhill 1.1 linkdata()
521     {
522 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
523 edhill 1.1 #
524     # symbolically link data files to run directory
525 jmc 1.83 if test -d $1 ; then
526 edhill 1.12 (
527 jmc 1.83 cd $1 ; shift
528 jmc 1.125 inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
529     for xx in $inpMPI ; do
530     if test -r "../"$1"/"$xx ; then
531     # found 1 _mpi sfx file in 1rst input dir and it is readable
532     yy=`echo $xx | sed 's:\.mpi$::'`
533     if test "x$MPI" = "xt" ; then
534     # mpi test: remove symbolic link & link _mpi sfx file
535     if test -h $yy ; then rm -f $yy ; fi
536     if test ! -r $yy ; then
537     ln -sf "../"$1"/"$xx $yy ;
538     printf "$xx " 1>&2
539     fi
540     else
541     # not mpi test: remove symbolic link
542     if test -h $yy ; then rm -f $yy ; fi
543     fi
544     fi
545     done
546 jmc 1.83 if test -r "../"$1"/eedata.mth" ; then
547     # found eedata.mth in 1rst input dir and it is readable
548     if test "x$MULTI_THREAD" = "xt" ; then
549     # multi-threaded test: remove symbolic link & link eedata.mth
550     if test -h eedata ; then rm -f eedata ; fi
551     if test ! -r eedata ; then
552     ln -sf "../"$1"/eedata.mth" eedata ;
553     printf 'eedata.mth ' 1>&2
554 edhill 1.24 fi
555 jmc 1.83 else
556     # not multi-threaded test: remove eedata symbolic link
557     if test -h eedata ; then rm -f eedata ; fi
558     fi
559     fi
560 jmc 1.108 prevDir='NONE'
561 jmc 1.83 for ldir in $* ; do
562 jmc 1.108 if test -d "../"$ldir -a $ldir != $prevDir ; then
563 jmc 1.83 printf 'ldir='${ldir} 1>&2
564     files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
565     for i in $files ; do
566     if test ! -d "../"$ldir/$i ; then
567     if test ! -r $i ; then
568     printf ' '$i 1>&2
569     ln -sf "../"$ldir"/"$i $i
570     fi
571     fi
572     done
573     if test -x "../"$ldir"/"prepare_run ; then
574     "../"$ldir"/"prepare_run
575 edhill 1.24 fi
576 jmc 1.83 printf ' ; ' 1>&2
577 jmc 1.82 fi
578 jmc 1.108 prevDir=$ldir
579 jmc 1.83 done
580 edhill 1.12 )
581 edhill 1.1 fi
582     }
583    
584     runmodel()
585     {
586 edhill 1.6 # runmodel directory
587 edhill 1.1 #
588 edhill 1.24 # runs "$COMMAND in "directory"
589 edhill 1.6 # (where "$COMMAND" is relative to "directory")
590 edhill 1.1 (
591     cd $1
592 jmc 1.75 printf 'runmodel in %s ...' $1 1>&2
593 edhill 1.6 # make output.txt
594 edhill 1.38 echo
595 jmc 1.106 if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
596     diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
597 jmc 1.119 outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi
598 jmc 1.106 fi
599 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
600 jmc 1.119 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
601 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
602     fi
603     if test ! -x $EXECUTABLE ; then
604 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
605     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
606     echo " no executable:" $EXECUTABLE >> $RUNLOG
607 jmc 1.83 RETVAL=8
608 jmc 1.88 ENDVAL=-1
609 jmc 1.75 else
610 jmc 1.98 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
611     # output do not exist or is older than executable:
612 jmc 1.119 rm -f $RUNLOG ; touch $RUNLOG
613     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
614     ( eval $COMMAND ) >> $RUNLOG 2>&1
615 jmc 1.83 RETVAL=$?
616     else
617     RETVAL=0
618 jmc 1.119 if test -f $RUNLOG ; then
619     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
620     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
621 jmc 1.88 else
622 jmc 1.119 touch $RUNLOG
623     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
624     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
625     echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1
626 jmc 1.88 fi
627 jmc 1.83 fi
628 jmc 1.119 ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
629 jmc 1.75 fi
630 jmc 1.119 rm -f run.log_tmp
631 jmc 1.88 #if test "x$RETVAL" = x0 ; then
632     if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
633 jmc 1.119 tail $RUNLOG
634 edhill 1.20 echo successful 1>&2
635 edhill 1.44 # === Reduce the size of the testing emails!
636 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
637 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
638 edhill 1.6 return 0
639     else
640 jmc 1.119 tail $RUNLOG
641 jmc 1.88 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
642 jmc 1.119 cp $RUNLOG $CDIR"/"$RUNLOG
643 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
644 edhill 1.6 return 1
645 edhill 1.1 fi
646     )
647     }
648    
649     createcodelet()
650     {
651     # create codelet for comparing model output
652    
653 jmc 1.112 printf "creating the comparison code (using CC=$CC)... "
654 jmc 1.91 cat > tr_cmpnum.c <<EOF
655 edhill 1.22 #include <stdio.h>
656     #include <math.h>
657     int main( int argc, char** argv ) {
658 jmc 1.91 int linnum,cmplin,best,lncnt;
659 edhill 1.52 double a,b,abave,relerr;
660 jmc 1.57 best = -22;
661 adcroft 1.33 lncnt = 0;
662 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
663 edhill 1.22 scanf("%d", &linnum);
664     if (linnum == -1) break;
665     scanf("%lf", &a); scanf("%lf", &b);
666 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
667 jmc 1.91 if ( abave == abave ) {
668     if (abave > 0.0) {
669     relerr=fabs(a-b)/abave;
670     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
671     else { cmplin = -16 ; }
672     best = (best > cmplin) ? best : cmplin; }
673     else { cmplin = -22 ; }
674     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
675     }
676     else {
677     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
678     break; }
679 edhill 1.22 }
680 adcroft 1.33 if (lncnt == 999) best=-29;
681 jmc 1.91 if (linnum != -1) best=-99;
682 edhill 1.22 printf("%d\n", -best);
683     return 0;
684     }
685     EOF
686 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
687 edhill 1.1
688 jmc 1.91 if [ -x ./tr_cmpnum ]; then
689 edhill 1.1 echo "OK"
690     return 0
691     else
692     echo
693 edhill 1.58 echo "ERROR: failed to compile comparison code -- please specify"
694     echo " a C compiler using the CC environment variable."
695 edhill 1.1 exit 1
696     fi
697     }
698    
699     formatresults()
700     {
701     # formatresults expt genmake depend make run results*
702    
703     nm=$1
704     printf '%s %s %s %s' $2 $3 $4 $5
705     shift; shift; shift; shift; shift;
706 jmc 1.94 listPrt=$@
707     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
708     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
709 jmc 1.97 printf '%3s' $listPrt
710 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
711 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
712 edhill 1.1
713 jmc 1.94 if [ $xx = '..' ]; then
714     printf ' N/O '
715     elif [ $xx = '--' ]; then
716     printf ' N/O '
717     elif [ $xx = 99 ]; then
718 edhill 1.1 printf ' N/O '
719     else
720 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
721 edhill 1.1 printf ' pass'
722     else
723     printf ' FAIL'
724     fi
725     fi
726     printf ' %s' $nm
727     printf '\n'
728    
729     }
730    
731     scandirs()
732     {
733 jmc 1.81 if [ $# -eq 1 ]; then
734     for arg in * ; do
735 jmc 1.99 test -f $arg/$1 && echo $arg
736 jmc 1.81 done
737 edhill 1.1 else
738 jmc 1.81 echo $*
739 edhill 1.1 fi
740     }
741    
742    
743 jmc 1.123 check_eedata()
744     {
745     # check_eedata size.h eedata
746     if [ $# -eq 2 ] ; then
747     if test -f $1 -a -f $2 ; then
748     sx=`grep "^ & *nSx *=" $1 | sed "s/^ & *nSx *=//" | sed 's/, *$//'`
749     nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
750     if test "x$nx" = x ; then
751     rx=10
752     else
753     rx=`expr $sx % $nx`
754     fi
755     sy=`grep "^ & *nSy *=" $1 | sed "s/^ & *nSy *=//" | sed 's/, *$//'`
756     ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
757     if test "x$ny" = x ; then
758     ry=20
759     else
760     ry=`expr $sy % $ny`
761     fi
762     echo `expr $rx + $ry`
763     else
764     echo '-1'
765     fi
766     else
767     echo '-2'
768     fi
769    
770     }
771    
772 edhill 1.1 ###############################################################################
773    
774    
775     # Default properties
776     debug=0
777     verbose=1
778     clean=0
779 edhill 1.10
780 edhill 1.40 IEEE=true
781 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
782     IEEE=$MITGCM_IEEE
783     fi
784 jmc 1.109 GSL=f
785 edhill 1.10
786     CLEANUP=f
787     QUICK=f
788     NOGENMAKE=f
789     NOCLEAN=f
790     NODEPEND=f
791 edhill 1.68 POSTCLEAN=f
792 edhill 1.1
793 edhill 1.4 BASH=
794 edhill 1.10 OPTFILE=NONE
795     ADDRESSES=
796 edhill 1.1 TESTDIRS=
797 jmc 1.89 SKIPDIRS=
798 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
799 edhill 1.20 HAVE_MPACK=
800 jmc 1.113 MPACK=
801 edhill 1.24 COMMAND=
802 edhill 1.59 if test "x$MAKE" = x ; then
803     MAKE=make
804     fi
805     if test "x$CC" = x ; then
806     CC=cc
807     fi
808 edhill 1.43 JOBS=
809 edhill 1.7 MPI=f
810 jmc 1.83 MULTI_THREAD=f
811 jmc 1.70 OUTDIR=
812 edhill 1.40 DELDIR=
813 edhill 1.1
814 edhill 1.24 ADM=
815 utke 1.120 OADM=
816 edhill 1.24
817 jmc 1.117 # list of pTracers to check for monitor output
818 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
819 edhill 1.49
820 jmc 1.92 MATCH_CRIT=13
821    
822 edhill 1.34 printf "parsing options... "
823 edhill 1.1
824     ac_prev=
825     for ac_option ; do
826    
827     # If the previous option needs an argument, assign it.
828     if test -n "$ac_prev"; then
829     eval "$ac_prev=\$ac_option"
830     ac_prev=
831     continue
832     fi
833    
834     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
835    
836     case $ac_option in
837    
838     -help | --help | -h | --h)
839     usage ;;
840    
841 edhill 1.2 -optfile | --optfile | -of | --of)
842 edhill 1.10 ac_prev=OPTFILE ;;
843 edhill 1.2 -optfile=* | --optfile=* | -of=* | --of=*)
844 edhill 1.10 OPTFILE=$ac_optarg ;;
845 edhill 1.1
846     -addr | --addr | -a | --a)
847     ac_prev=ADDRESSES ;;
848     -addr=* | --addr=*)
849     ADDRESSES=$ac_optarg ;;
850 jmc 1.113 -mpackdir | --mpackdir | -mpd | --mpd)
851     ac_prev=MPACKDIR ;;
852     -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
853     MPACKDIR=$ac_optarg ;;
854 edhill 1.1
855     -tdir | --tdir | -t | --t)
856     ac_prev=TESTDIRS ;;
857     -tdir=* | --tdir=*)
858     TESTDIRS=$ac_optarg ;;
859 edhill 1.4
860 jmc 1.89 -skipdir | --skipdir | -skd | --skd)
861     ac_prev=SKIPDIRS ;;
862     -skipdir=* | --skipdir=*)
863     SKIPDIRS=$ac_optarg ;;
864    
865 edhill 1.4 -bash | --bash | -b | --b)
866     ac_prev=BASH ;;
867     -bash=* | --bash=*)
868     BASH=$ac_optarg ;;
869 edhill 1.5
870 edhill 1.6 -command | --command | -c | --c)
871     ac_prev=COMMAND ;;
872     -command=* | --command=*)
873     COMMAND=$ac_optarg ;;
874 edhill 1.8
875     -make | --make | -m | --m)
876     ac_prev=MAKE ;;
877     -make=* | --make=*)
878     MAKE=$ac_optarg ;;
879 edhill 1.1
880 jmc 1.70 -odir | --odir)
881     ac_prev=OUTDIR ;;
882     -odir=* | --odir=*)
883     OUTDIR=$ac_optarg ;;
884    
885 edhill 1.50 -ptracers | --ptracers | -ptr | --ptr)
886     ac_prev=PTRACERS_NUM ;;
887     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
888     PTRACERS_NUM=$ac_optarg ;;
889    
890 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
891     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
892    
893 edhill 1.43 -j) ac_prev=JOBS ;;
894     -j=*) JOBS=$ac_optarg ;;
895    
896 edhill 1.10 -clean | --clean)
897 jmc 1.69 CLEANUP=t ; DELDIR=t ;;
898 edhill 1.10
899     -quick | --quick | -q | --q)
900     QUICK=t ;;
901     -nogenmake | --nogenmake | -ng | --ng)
902     NOGENMAKE=t ;;
903     -noclean | --noclean | -nc | --nc)
904     NOCLEAN=t ;;
905     -nodepend | --nodepend | -nd | --nd)
906     NODEPEND=t ;;
907    
908 edhill 1.68 -postclean | --postclean | -pc | --pc)
909     POSTCLEAN=t ;;
910    
911 edhill 1.18 -mpi) MPI=t ;;
912 edhill 1.10
913 jmc 1.83 -mth) MULTI_THREAD=t ;;
914    
915 edhill 1.24 -adm | -ad) ADM=t ;;
916    
917 utke 1.120 -oad) OADM=t; NODEPEND=t ;;
918    
919 edhill 1.10 -ieee) IEEE=true ;;
920     -noieee) IEEE= ;;
921 jmc 1.109 -gsl) GSL=t ;;
922 edhill 1.10
923 edhill 1.1 -verbose) verbose=2 ;;
924     -debug) debug=1 ;;
925     -quiet) verbose=0 ;;
926    
927 edhill 1.40 -deldir | -dd) DELDIR=t ;;
928    
929 ce107 1.90 -ts) TS=t;;
930    
931     -papis) PAPIS=t;;
932    
933     -pcls) PCL=t;;
934    
935 edhill 1.1 -*)
936     echo "Error: unrecognized option: "$ac_option
937     usage
938     ;;
939    
940     *)
941     echo "Error: unrecognized argument: "$ac_option
942     usage
943     ;;
944    
945     esac
946    
947     done
948    
949 edhill 1.10 if test "x$QUICK" = xt ; then
950     NOGENMAKE=t
951     NOCLEAN=t
952     NODEPEND=t
953     fi
954    
955 jmc 1.99 #- setting for forward or ADM testing
956     if test "x$ADM" = xt ; then
957     code_dir=code_ad
958 jmc 1.108 inputdir=input_ad
959 jmc 1.99 ref_outp="output_adm.txt"
960     EXECUTABLE="mitgcmuv_ad"
961 utke 1.120 elif test "x$OADM" = xt ; then
962     code_dir=code_oad
963     inputdir=input_oad
964     ref_outp="output_oadm.txt"
965     EXECUTABLE="mitgcmuv_ad"
966 jmc 1.99 else
967     code_dir=code
968 jmc 1.108 inputdir=input
969 jmc 1.99 ref_outp="output.txt"
970     EXECUTABLE="mitgcmuv"
971     fi
972    
973 edhill 1.1 if test "x$TESTDIRS" = x ; then
974 jmc 1.103 LIST=`scandirs results/$ref_outp`
975 jmc 1.84 else
976     #- expand group of experiments:
977     LIST=" "
978     for xx in $TESTDIRS
979     do
980     case $xx in
981 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
982     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
983     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
984 jmc 1.84 ;;
985     'tutorials')
986     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
987     *) LIST=${LIST}" "$xx ;;
988     esac
989     done
990 jmc 1.89 fi
991     #echo 'LIST='${LIST}'<'
992     #- skip dirs, remove duplicate and non-directory:
993     TESTDIRS=" "
994     count=0
995     for xx in $LIST
996     do
997     yy=`echo $SKIPDIRS | grep -c $xx`
998     if test $yy = 0 ; then
999 jmc 1.84 if test -d $xx ; then
1000     yy=`echo $TESTDIRS | grep -c $xx`
1001     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
1002 jmc 1.89 else count=1 ;
1003     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
1004     fi
1005     else
1006     if test $count = 1 ; then echo -n ", \"$xx\""
1007     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
1008 jmc 1.84 fi
1009 jmc 1.89 fi
1010     done
1011     if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
1012 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
1013 edhill 1.1
1014 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
1015     OPTFILE=$MITGCM_OF
1016     fi
1017    
1018 jmc 1.119 RUNLOG="run.log"
1019 jmc 1.99 OUTPUTFILE=$ref_outp
1020 jmc 1.87 if test "x$COMMAND" = x ; then
1021 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
1022 edhill 1.24 fi
1023 jmc 1.87 if test "x$MPI" = xt ; then
1024     OUTPUTFILE="STDOUT.0000"
1025 edhill 1.24 fi
1026    
1027 jmc 1.76 echo "OK (COMMAND= $COMMAND )"
1028 edhill 1.1
1029 jmc 1.94 # set the Default List of output variables to be checked:
1030     # (use default or load experiment-specific list from file "tr_checklist")
1031     # content : 1rst = main variable used to decide if it pass or FAIL
1032     # others = number of matching digits to be printed in summary.txt
1033 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1034 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
1035     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
1036 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1037 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
1038 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
1039 jmc 1.99 else
1040 jmc 1.103 DEF_CHECK_LIST='Grad Cost Grad'
1041 jmc 1.99 EMPTY_RESULTS='.. ..'
1042     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1043 jmc 1.94 fi
1044    
1045 edhill 1.1 # create the FORTRAN comparison code
1046     createcodelet
1047    
1048 jmc 1.112 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
1049     if test "x$ADDRESSES" = x ; then
1050 edhill 1.32 echo "skipping mpack build"
1051     else
1052 edhill 1.31 build_mpack
1053     fi
1054 edhill 1.1
1055     # Create a uniquely named directory to store results
1056 jmc 1.110 CMDLINE=$0
1057 jmc 1.118 for xx in "$@" ; do nw=`echo $xx | wc -w`
1058     if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
1059     else CMDLINE="$CMDLINE '$xx'" ; fi
1060     done
1061     #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
1062 edhill 1.1 MACH=`hostname`
1063 edhill 1.2 UNAMEA=`uname -a`
1064 edhill 1.1 DATE=`date +%Y%m%d`
1065 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1066 jmc 1.70 if test "x$OUTDIR" != x ; then
1067     BASE="tr_"$OUTDIR"_"$DATE"_"
1068 jmc 1.121 else
1069     short_name=`hostname | sed 's/\..*$//'`
1070     BASE="tr_"$short_name"_"$DATE"_"
1071 jmc 1.70 fi
1072 edhill 1.1 DNUM=0
1073     DRESULTS="$BASE$DNUM"
1074     while test -e $DRESULTS ; do
1075     DNUM=$(( $DNUM + 1 ))
1076     DRESULTS="$BASE$DNUM"
1077     done
1078     mkdir $DRESULTS
1079     RETVAL=$?
1080     if test "x$RETVAL" != x0 ; then
1081 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1082 edhill 1.1 exit 1
1083     fi
1084     SUMMARY="$DRESULTS/summary.txt"
1085 edhill 1.16 start_date=`date`
1086 edhill 1.17 echo $start_date > $SUMMARY
1087 jmc 1.110 echo 'run:' $CMDLINE >> $SUMMARY
1088     echo 'on :' $UNAMEA >> $SUMMARY
1089 edhill 1.1
1090 edhill 1.11 of_path=
1091 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1092     if test -r $OPTFILE ; then
1093 edhill 1.11 # get the path
1094     path=${OPTFILE%/*}
1095     if test "x$path" = x ; then
1096     of_path=`pwd`
1097     else
1098     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1099     fi
1100     file=${OPTFILE##*/}
1101     OPTFILE=$of_path/$file
1102 edhill 1.21 cp $OPTFILE $DRESULTS
1103     echo >> $SUMMARY
1104     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1105 edhill 1.11 else
1106 edhill 1.21 echo | tee $SUMMARY
1107     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1108     exit 1
1109 edhill 1.10 fi
1110 edhill 1.21 else
1111     echo >> $SUMMARY
1112     echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1113     echo " so the genmake default will be used." >> $SUMMARY
1114 edhill 1.10 fi
1115     echo
1116     echo >> $SUMMARY
1117 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1118 jmc 1.92 if [ $MATCH_CRIT -lt 10 ] ;
1119 jmc 1.94 then line_0="default "$MATCH_CRIT ;
1120     else line_0="default "$MATCH_CRIT ; fi
1121     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1122 jmc 1.92 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1123 edhill 1.49 line_1="G D M c m s m s m s m s"
1124     line_2="E p a R g m m e . m m e . m m e . m m e ."
1125     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1126     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1127 edhill 1.50 for ii in $PTRACERS_NUM ; do
1128 edhill 1.49 line_0="$line_0 --PTR 0"$ii"--"
1129     line_1="$line_1 m s"
1130     line_2="$line_2 m m e ."
1131     line_3="$line_3 i a a d"
1132     line_4="$line_4 n x n ."
1133     done
1134     echo "$line_0" | tee -a $SUMMARY
1135     echo "$line_1" | tee -a $SUMMARY
1136     echo "$line_2" | tee -a $SUMMARY
1137     echo "$line_3" | tee -a $SUMMARY
1138     echo "$line_4" | tee -a $SUMMARY
1139     echo " " | tee -a $SUMMARY
1140 edhill 1.24 else
1141     echo "ADJOINT=true" >> $SUMMARY
1142     echo >> $SUMMARY
1143 jmc 1.100 if [ $MATCH_CRIT -lt 10 ] ;
1144 jmc 1.103 then line_0="default "$MATCH_CRIT ;
1145     else line_0="default "$MATCH_CRIT ; fi
1146 jmc 1.100 echo "$line_0" | tee -a $SUMMARY
1147 edhill 1.24 cat << EOF | tee -a $SUMMARY
1148     G D M C G
1149     E p a R o r
1150     N n k u s a
1151     2 d e n t d
1152    
1153     EOF
1154     fi
1155 edhill 1.1
1156 edhill 1.10 # ...and each test directory...
1157     for dir in $TESTDIRS ; do
1158    
1159 jmc 1.106 # set builddir & rundir:
1160     builddir="build"
1161     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1162     rundir="run"
1163     if test ! -d $dir/$rundir ; then
1164     rundir=$builddir
1165     fi
1166     CODE_DIR=$dir/$code_dir
1167     BUILD_DIR=$dir/$builddir
1168    
1169 edhill 1.10 # Cleanup only!
1170     if test "x$CLEANUP" = xt ; then
1171 jmc 1.106 if test -r $BUILD_DIR/Makefile ; then
1172 jmc 1.115 echo ' ------ clean dir:' $dir/$builddir
1173 jmc 1.106 ( cd $BUILD_DIR ; make CLEAN )
1174 edhill 1.10 fi
1175 jmc 1.106 if test -d $dir/$rundir/CVS ; then
1176     echo ' ------ clean dir:' $dir/$rundir
1177     run_clean $dir/$rundir
1178 edhill 1.10 fi
1179 edhill 1.62 (
1180     cd $dir
1181     rm -rf tr_run.*
1182     )
1183 edhill 1.10 continue
1184 edhill 1.1 fi
1185 edhill 1.3
1186 edhill 1.10 # Verify that the testdir exists and contains previous
1187     # results in the correct location--or skip this directory!
1188 jmc 1.104 fout=$dir"/results/"$ref_outp
1189 edhill 1.24 if test ! -r $fout ; then
1190     echo "can't read \"$fout\" -- skipping $dir"
1191 edhill 1.10 continue
1192     fi
1193 edhill 1.7
1194 jmc 1.117 # Check for specific files for particular type of run
1195 edhill 1.49
1196 edhill 1.30 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1197 edhill 1.28 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1198     continue
1199     fi
1200 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1201     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1202     continue
1203     fi
1204 jmc 1.123 if test "x$MPI" = "xt" -a "x$MULTI_THREAD" = "xt" ; then
1205     retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/input/eedata.mth"`
1206     if test $retv != 0 ; then
1207     echo "input/eedata.mth tiling misfit -- skipping $dir"
1208     continue
1209     fi
1210     fi
1211 jmc 1.83
1212     # Check whether there are "extra runs" for this testdir
1213     extra_runs=
1214 jmc 1.108 ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
1215     #echo "ex_run_dirs='$ex_run_dirs'"
1216     for exd in $ex_run_dirs ; do
1217     name=`echo $exd | sed -e "s/$inputdir\.//"`
1218     refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1219     outf="$dir/results/$refExOut"
1220     if test -f $outf -a -r $outf ; then
1221 jmc 1.123 if test "x$MULTI_THREAD" = "xt" ; then
1222     if test -r $dir"/"$exd"/eedata.mth" ; then
1223     if test "x$MPI" = "xt" ; then
1224     retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/"$exd"/eedata.mth"`
1225     if test $retv = 0 ; then
1226 jmc 1.83 extra_runs="$extra_runs $name"
1227 jmc 1.123 else
1228     echo $exd"/eedata.mth tiling misfit -- skipping $dir"
1229 jmc 1.83 fi
1230 jmc 1.123 else
1231 jmc 1.108 extra_runs="$extra_runs $name"
1232 jmc 1.123 fi
1233     #else echo $dir"/"$exd"/eedata.mth: not found"
1234 jmc 1.83 fi
1235 jmc 1.123 else
1236     extra_runs="$extra_runs $name"
1237     fi
1238 jmc 1.108 fi
1239     done
1240 edhill 1.28
1241     echo "-------------------------------------------------------------------------------"
1242     echo
1243 jmc 1.83 if test "x$extra_runs" = "x" ; then
1244     echo "Experiment: $dir"
1245     else
1246     echo "Experiment: $dir ; extra_runs=$extra_runs"
1247     fi
1248 edhill 1.28 echo
1249     unset genmake makedepend make run
1250 jmc 1.94 results=$EMPTY_RESULTS
1251 edhill 1.10
1252     # Create an output dir for each OPTFILE/tdir combination
1253 edhill 1.20 rel_CDIR=$DRESULTS"/"$dir
1254     mkdir $rel_CDIR
1255     CDIR=`pwd`"/$rel_CDIR"
1256 edhill 1.10
1257     if test "x$CLEANUP" = xt ; then
1258 jmc 1.83 echo '====>>> this is to check that we never go through this part <<< ==='
1259     makeclean $dir/$builddir \
1260     && run_clean $dir/$rundir
1261 edhill 1.10 else
1262 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1263     && makeclean $dir/$builddir \
1264 jmc 1.83 && run_clean $dir/$rundir \
1265 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1266 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1267     && makemodel $dir/$builddir && make=Y \
1268 jmc 1.108 && linkdata $dir/$rundir $inputdir input \
1269 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1270 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1271 edhill 1.10 fi
1272    
1273     echo
1274 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1275 edhill 1.34 echo
1276 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1277 jmc 1.100 touch $CDIR"/summary.txt"
1278 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1279 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1280     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1281     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1282     echo "tdir='$dir'" >> $CDIR"/summary.txt"
1283 jmc 1.108 if test "x$ADM" = xt ; then
1284     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1285     grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1286     >> $CDIR"/summary.txt"
1287     fi
1288 edhill 1.34
1289     for ex in $extra_runs ; do
1290 jmc 1.73 unset run
1291 jmc 1.94 results=$EMPTY_RESULTS
1292 jmc 1.106 # reference output file
1293     refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1294 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1295     rel_CDIR=$DRESULTS"/"$dir"."$ex
1296     mkdir $rel_CDIR
1297     CDIR=`pwd`"/$rel_CDIR"
1298 edhill 1.61 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1299 jmc 1.106 run_clean $dir/tr_run.$ex
1300 jmc 1.108 linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
1301 jmc 1.42 runmodel $dir/tr_run.$ex && run=Y \
1302 jmc 1.106 && results=`testoutput_run $dir tr_run.$ex $refExOut`
1303 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1304     fres="$fres.$ex"
1305     echo
1306 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1307     touch $CDIR"/summary.txt"
1308 jmc 1.97 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1309 edhill 1.34 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1310     echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1311     echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1312 jmc 1.73 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1313 jmc 1.108 if test "x$ADM" = xt ; then
1314     head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1315     grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
1316     >> $CDIR"/summary.txt"
1317     fi
1318 jmc 1.83 if test "x$POSTCLEAN" = xt ; then
1319 jmc 1.100 run_clean $dir/tr_run.$ex
1320 jmc 1.83 fi
1321 edhill 1.34 done
1322 edhill 1.68
1323 jmc 1.83 #postclean $dir/$builddir
1324     if test "x$POSTCLEAN" = xt ; then
1325     makeclean $dir/$builddir \
1326     && run_clean $dir/$rundir
1327     fi
1328 edhill 1.10
1329     echo "-------------------------------------------------------------------------------"
1330    
1331 edhill 1.1 done
1332    
1333 edhill 1.34 printf "Start time: " >> $SUMMARY
1334 jmc 1.114 echo "$start_date" >> $SUMMARY
1335 edhill 1.34 printf "End time: " >> $SUMMARY
1336 edhill 1.13 date >> $SUMMARY
1337 edhill 1.20
1338     # If addresses were supplied and mpack built successfully, then try
1339     # to send email using mpack.
1340     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1341     echo "No results email was sent."
1342     else
1343     if test "x$HAVE_MPACK" = xt ; then
1344     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1345     && gzip $DRESULTS".tar" \
1346 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1347 edhill 1.20 RETVAL=$?
1348     if test "x$RETVAL" != x0 ; then
1349     echo
1350     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1351     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1352     echo " summary of results from the directory \"$DRESULTS\"."
1353     echo
1354     else
1355     echo
1356     echo "An email containing results was sent to the following addresses:"
1357     echo " \"$ADDRESSES\""
1358     echo
1359     fi
1360     test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1361     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1362     fi
1363     fi
1364 edhill 1.13
1365 jmc 1.91 rm -f tr_cmpnum.c tr_cmpnum
1366 edhill 1.1
1367 edhill 1.12 if test "x$CLEANUP" != xt ; then
1368 jmc 1.102 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1369 edhill 1.25 if test -e tr_out.txt ; then
1370     mv tr_out.txt tr_out.txt.old
1371 edhill 1.14 fi
1372 jmc 1.103 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1373 edhill 1.12 fi
1374 edhill 1.1
1375 edhill 1.40 if test "x$DELDIR" = xt ; then
1376     rm -rf $DRESULTS
1377     fi
1378    

  ViewVC Help
Powered by ViewVC 1.1.22