/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Contents of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.120 - (show annotations) (download)
Thu Jan 29 21:45:43 2009 UTC (15 years, 2 months ago) by utke
Branch: MAIN
Changes since 1.119: +25 -11 lines
modified for OpenAD verification - new flag -oad, new variable OADM,
modified test conditions that use ADM, use 'eval' to run the command
string for genmake or else the single quotes for the multiple
mods directories are lost and genmake doesn't parse the mods argument
correctly.

1 #! /usr/bin/env bash
2 #
3 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.119 2009/01/26 04:09:34 jmc Exp $
4 # $Name: $
5 #
6
7 usage()
8 {
9 echo
10 echo "Usage: $0 [OPTIONS]"
11 echo
12 echo "where possible OPTIONS are:"
13 echo " (-help|-h) print usage"
14 echo " (-mth) run multi threaded (using eedata.mth)"
15 echo " (-mpi) compile and run using MPI"
16 echo " (-ieee|-noieee) if possible, use IEEE compiler flags"
17 echo " (DEF=\"-ieee\")"
18 echo " (-gsl) compile with \"-gsl\" flag"
19 echo " (-of=|-optfile=)STRING list of optfiles to use"
20 echo " (-a|-addr) STRING list of email recipients"
21 echo " (DEF=\"\" no email is sent)"
22 echo " (-mpd|-mpackdir) DIR location of the mpack utility"
23 echo " (DEF=\"../tools/mpack-1.6\")"
24 echo " (-t|-tdir) STRING list of group and/or exp. dirs to test"
25 echo " (recognized groups: basic, tutorials)"
26 echo " (DEF=\"\" which test all)"
27 echo " (-skd|-skipdir) STRING list of exp. dirs to skip"
28 echo " (DEF=\"\" which test all)"
29 echo " (-b|-bash) STRING preferred location of a \"bash\" or"
30 echo " Bourne-compatible \"sh\" shell"
31 echo " (DEF=\"\" for \"bash\")"
32 echo " (-adm|-ad) perform an adjoint run"
33 echo " (-oad) perform an OpenAD adjoint run"
34 echo " (-command) STRING command to run"
35 echo " (DEF=\"make output.txt\")"
36 echo " (-m|-make) STRING command to use for \"make\""
37 echo " (DEF=\"make\")"
38 echo " (-odir) STRING used to build output directory name"
39 echo " (DEF=\"hostname\")"
40 echo " (-ptr|-ptracers) STRING specify which ptracers to test"
41 echo " (DEF=\"1 2 3 4 5\")"
42 echo " (-match) NUMBER Matching Criteria (number of digits)"
43 echo " (DEF=\"12\")"
44 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
45 echo " (-clean) *ONLY* run \"make CLEAN\""
46 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
47 echo " (-nogenmake|-ng) skip the genmake stage"
48 echo " (-noclean|-nc) skip the \"make clean\" stage"
49 echo " (-nodepend|-nd) skip the \"make depend\" stage"
50 echo " (-deldir|-dd) on success, delete the output directory"
51 echo " (-ts) provide timing information per timestep"
52 echo " (-papis) provide MFlop/s per timestep using PAPI"
53 echo " (-pcls) provide MFlop/s per timestep using PCL"
54 echo
55 echo "and where STRING can be a whitespace-delimited list"
56 echo "such as:"
57 echo
58 echo " -t 'exp0 exp2 exp3' "
59 echo " -addr='abc@123.com testing@home.org'"
60 echo
61 echo "provided that the expression is properly quoted within the current"
62 echo "shell (note the use of single quotes to protect white space)."
63 echo
64 exit 1
65 }
66
67 # build the mpack utility
68 build_mpack()
69 {
70 printf "building the mpack utility... "
71 MPACK="$MPACKDIR/mpack"
72 if test ! -x $MPACK ; then
73 if test ! -d $MPACKDIR ; then
74 echo
75 echo "Error: can't find \"$MPACKDIR\""
76 echo " are you sure this program is being run in the correct "
77 echo " (that is, \"MITGCM_ROOT\verification\") directory?"
78 echo
79 HAVE_MPACK=f
80 fi
81 if test "x$CC" = x ; then
82 export CC=cc
83 fi
84 printf "building mpack (using CC=$CC)... "
85 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
86 RETVAL=$?
87 if test "x$RETVAL" != x0 ; then
88 echo
89 echo "Error building the mpack tools at: $MPACK_DIR"
90 echo
91 HAVE_MPACK=f
92 else
93 rm -f tr_build_mpack.out
94 HAVE_MPACK=t
95 echo "done"
96 fi
97 else
98 HAVE_MPACK=t
99 echo "already exist"
100 fi
101 }
102
103 testoutput_var()
104 {
105 # testoutput_var dir s1 label subdir reference_output
106 #
107 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
108 # with same output from reference file $dir/results/$reference_output
109 # using search strings s1 and text label
110
111 if [ $debug -gt 0 ]; then
112 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
113 fi
114 if [ -r $1/$4/$OUTPUTFILE ]; then
115 grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
116 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
117 if [ $lncntA -lt 2 ]; then
118 if [ $verbose -gt 0 ]; then
119 echo Not enough lines of output when searching for "$2" 1>&2
120 fi
121 return 99
122 fi
123 else
124 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
125 return 99
126 fi
127 if [ $debug -gt 0 ]; then
128 echo testoutput_var: grep "$2" $1/results/$5 1>&2
129 fi
130 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
131 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
132 if [ $lncntB -lt 2 ]; then
133 if [ $verbose -gt 0 ]; then
134 echo Not enough lines of output when searching for "$2" 1>&2
135 fi
136 return 99
137 fi
138 if [ $lncntA -ne $lncntB ]; then
139 if [ $verbose -gt 0 ]; then
140 echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
141 fi
142 return 99
143 fi
144 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
145 if [ $has_nan -gt 0 ] ; then
146 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
147 return 99
148 fi
149 has_inf=`cat tmp1.txt | grep -i inf | wc -l`
150 if [ $has_inf -gt 0 ] ; then
151 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
152 return 99
153 fi
154 if [ $debug -gt 0 ]; then
155 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
156 fi
157 join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
158 if [ $debug -gt 0 ]; then
159 echo testoutput_var: compare_lines 1>&2
160 fi
161 if [ $verbose -gt 1 ]; then
162 cat tmp3.txt 1>&2
163 fi
164 echo "-1" >> tmp3.txt
165 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
166 cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
167 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
168 if [ $digits_of_similarity -eq 99 ]; then
169 if [ $verbose -gt 0 ]; then
170 echo testoutput_var: No comparison was available for \"$3\" 1>&2
171 fi
172 digits_of_similarity=99
173 else
174 if [ $verbose -gt 0 ]; then
175 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
176 fi
177 fi
178 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
179
180 return $digits_of_similarity
181 }
182
183 testoutput_run()
184 {
185 # testoutput_run directory subdir reference_output
186 #
187 # test output from 1 run in "directory"
188 # --> same processing for adjoint & forward test
189 # default list of output variables to be checked:
190 # 1rst : main variable used to decide if it pass or FAIL
191 # others : number of matching digits to be printed in summary.txt
192 listChk=$DEF_CHECK_LIST
193 # load experiment-specific list from file "tr_checklist" (if it exist)
194 if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
195 sVar=`echo $listChk | awk '{print $1}'`
196 # remove 1rst var and expand the list: + => min max mean s.d
197 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
198 | sed 's/+//g' | sed "s/^$sVar//"`
199 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
200 # check for ptracer output in reference_output file :
201 outpref=$1/results/$3
202 ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
203 ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
204 for ii in $PTRACERS_NUM ; do
205 ptrfound=0
206 for jj in $ptr_mon ; do
207 name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
208 tst=`grep $name $outpref | wc -l | awk '{print $1}'`
209 if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
210 done
211 if test $ptrfound = '1' ; then
212 eval "HAVE_PTR0"$ii"=t"
213 else
214 eval "HAVE_PTR0"$ii"=f"
215 if test "x$ADM" = x -a "x$OADM" = x; then
216 # remove this ptr from the list of output variable to check
217 # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
218 listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
219 fi
220 fi
221 # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
222 done
223 tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
224 if test $tst != 1 ; then
225 if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
226 else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
227 echo "==> WARNING: in checked list:" $listVar 1>&2
228 #- put it back once:
229 listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
230 fi
231 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
232 echo "listVar='$listVar'" > $CDIR"/summary.txt"
233 allargs=""
234 for xx in $listVar
235 do
236 case $xx in
237 'PS') if [ $debug -gt 0 ]
238 then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
239 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
240 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
241 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
242 'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
243 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
244 'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
245 'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
246 'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
247 'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
248 'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
249 'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
250 'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
251 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
252 'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
253 'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
254 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
255 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
256 'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
257 'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
258 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
259 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
260 testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
261 'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
262 testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
263 'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
264 testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
265 'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
266 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
267 'Qntmn') testoutput_var $1 "extforcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;;
268 'Qntmx') testoutput_var $1 "extforcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;;
269 'Qntav') testoutput_var $1 "extforcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;;
270 'Qntsd') testoutput_var $1 "extforcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;;
271 'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;;
272 'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;;
273 'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;;
274 'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;;
275 'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;;
276 'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Hell max" $2 $3; yy=$? ;;
277 'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Hell mean" $2 $3; yy=$? ;;
278 'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Hell StDv" $2 $3; yy=$? ;;
279 'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
280 'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;;
281 'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;;
282 'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;;
283 'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;;
284 'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;;
285 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
286 esac
287 if test $xx = $sVar
288 then allargs="$allargs > $yy <"
289 else allargs="$allargs $yy"
290 fi
291 done
292
293 nbVar=`echo $listVar | awk '{print NF}'`
294 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
295 #-- fill line (up to standard length) with dot:
296 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
297 echo $allargs $adNul
298 else
299 echo $allargs
300 fi
301 # <-- same processing for adjoint & forward test
302 }
303
304 genmakemodel()
305 {
306 # genmakemodel directory
307 if test "x$NOGENMAKE" = xt ; then
308 echo "genmake skipped!"
309 else
310 if test "x$BASH" = x ; then
311 GENMAKE2="../../../tools/genmake2"
312 else
313 GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
314 fi
315 (
316 cd $1;
317 command="$GENMAKE2 -ds -m $MAKE"
318 if test "x$ADM" = xt ; then
319 command="$command --mods=../code_ad"
320 elif test "x$OADM" = xt ; then
321 command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
322 else
323 command="$command --mods=../code"
324 fi
325 if test "x$OPTFILE" != xNONE ; then
326 command="$command --optfile=$OPTFILE"
327 fi
328 if test "x$IEEE" != x ; then
329 command="$command -ieee"
330 fi
331 if test "x$GSL" = xt ; then
332 command="$command -gsl"
333 fi
334 if test "x$MPI" = xt ; then
335 command="$command -mpi"
336 fi
337 if test "x$TS" = xt ; then
338 command="$command -ts"
339 fi
340 if test "x$PAPIS" = xt ; then
341 command="$command -papis"
342 else
343 if test "x$PCLS" = xt ; then
344 command="$command -pcls"
345 fi
346 fi
347 printf 'genmake ... ' 1>&2
348 # $command > make.log 2>&1
349 eval $command > make.log 2>&1
350 RETVAL=$?
351 # Reduce the size of the testing emails!
352 head -100 Makefile > $CDIR/Makefile_head
353 if test "x$RETVAL" != x0 ; then
354 tail make.log
355 echo "genmakemodel: genmake failed" 1>&2
356 cp genmake_* make.log $CDIR
357 return 1
358 else
359 echo "successful" 1>&2
360 fi
361 )
362 fi
363 }
364
365 makeclean()
366 {
367 # makeclean directory
368 if test "x$NOCLEAN" = xt ; then
369 echo "make Clean skipped!"
370 else
371 (
372 cd $1;
373 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
374 if test -r Makefile ; then
375 printf 'clean build-dir: make Clean ... ' 2>&1
376 $MAKE Clean >> make.log 2>&1
377 RETVAL=$?
378 if test "x$RETVAL" != x0 ; then
379 tail make.log
380 echo "makeclean: \"make Clean\" failed" 1>&2
381 cp make.log $CDIR"/make.log"
382 return 1
383 fi
384 fi
385 echo successful 1>&2
386 exit 0
387 )
388 fi
389 }
390
391 run_clean()
392 {
393 # run_clean directory
394 if test "x$NOCLEAN" = xt ; then
395 echo "run_clean skipped!"
396 else
397 (
398 cd $1;
399 printf 'clean run-dir ... ' 2>&1
400 # part of what is done after "make clean" when doing "make CLEAN"
401 find . -name "*.meta" -exec rm {} \;
402 find . -name "*.data" -exec rm {} \;
403 find . -name "fort.*" -exec rm {} \;
404 find . -type l -exec rm {} \;
405 rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime
406 rm -rf mnc_test_*
407 rm -f *_MIT_CE_000.opt0000 costfunction*0000
408 echo successful 1>&2
409 exit 0
410 )
411 fi
412 }
413
414 makedependmodel()
415 {
416 # makedependmodel directory
417 if test "x$NODEPEND" = xt ; then
418 echo "make depend skipped!"
419 else
420 (
421 cd $1;
422 printf 'make depend ... ' 1>&2
423 $MAKE depend >> make.log 2>&1
424 RETVAL=$?
425 if test "x$RETVAL" != x0 ; then
426 tail make.log
427 echo "makedependmodel: make depend failed" 1>&2
428 cp make.log $CDIR"/make.log"
429 return 1
430 else
431 echo successful 1>&2
432 fi
433 )
434 fi
435 }
436
437 makemodel()
438 {
439 # makemodel directory
440 (
441 cd $1;
442 if test -r Makefile ; then
443 printf 'make ... ' 1>&2
444 if test "x$ADM" = xt ; then
445 $MAKE adall >> make.log 2>&1
446 elif test "x$OADM" = xt ; then
447 $MAKE adAll >> make.log 2>&1
448 else
449 if test "x$JOBS" = x ; then
450 $MAKE >> make.log 2>&1
451 else
452 $MAKE -j $JOBS >> make.log 2>&1
453 fi
454 fi
455 RETVAL=$?
456 if test "x$RETVAL" != x0 ; then
457 tail make.log
458 echo failed 1>&2
459 cp make.log $CDIR"/make.log"
460 return 1
461 else
462 echo successful 1>&2
463 fi
464 fi
465 )
466 }
467
468 symlink_mpifiles()
469 {
470 # Put special links so that MPI specific files are used
471 # This MUST be invoked between makeclean and makelinks because
472 # the Makefile will link to non-mpi files by default
473
474 dir=$1
475 code_dir=$2
476 BUILD_DIR=$dir/$3
477 CODE_DIR=$dir/$code_dir
478
479 # These are files that should replace their counter-part when using -mpi
480 MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)`
481
482 # Is this an MPI run?
483 if test "x$MPI" = xt ; then
484 # YES: We symbolically link these files to the build
485 # dir so long as there is no real file in place
486 for ii in $MPI_FILES ; do
487 i=`echo $ii | sed 's:^\./::'`
488 name=`echo $i | sed 's:_mpi::' `
489 cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1
490 RETVAL=$?
491 if test "x$RETVAL" != x0 ; then
492 if ! test -f $BUILD_DIR/$i ; then
493 #echo Linking $name to $i
494 (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name)
495 fi
496 fi
497 done
498 else
499 # NO: We undo any _mpi symbolically linked files
500 for ii in $MPI_FILES ; do
501 i=`echo $ii | sed 's:^\./::'`
502 name=`echo $i | sed 's:_mpi::' `
503 if test -L $BUILD_DIR/$name ; then
504 cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1
505 RETVAL=$?
506 if test "x$RETVAL" = x0 ; then
507 #echo Un-linking $name from $linktarg
508 rm -f $BUILD_DIR/$name
509 fi
510 fi
511 done
512 fi
513
514 }
515
516 linkdata()
517 {
518 # linkdata run_dir input_dir_1 input_dir_2 ...
519 #
520 # symbolically link data files to run directory
521 if test -d $1 ; then
522 (
523 cd $1 ; shift
524 if test -r "../"$1"/eedata.mth" ; then
525 # found eedata.mth in 1rst input dir and it is readable
526 if test "x$MULTI_THREAD" = "xt" ; then
527 # multi-threaded test: remove symbolic link & link eedata.mth
528 if test -h eedata ; then rm -f eedata ; fi
529 if test ! -r eedata ; then
530 ln -sf "../"$1"/eedata.mth" eedata ;
531 printf 'eedata.mth ' 1>&2
532 fi
533 else
534 # not multi-threaded test: remove eedata symbolic link
535 if test -h eedata ; then rm -f eedata ; fi
536 fi
537 fi
538 prevDir='NONE'
539 for ldir in $* ; do
540 if test -d "../"$ldir -a $ldir != $prevDir ; then
541 printf 'ldir='${ldir} 1>&2
542 files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
543 for i in $files ; do
544 if test ! -d "../"$ldir/$i ; then
545 if test ! -r $i ; then
546 printf ' '$i 1>&2
547 ln -sf "../"$ldir"/"$i $i
548 fi
549 fi
550 done
551 if test -x "../"$ldir"/"prepare_run ; then
552 "../"$ldir"/"prepare_run
553 fi
554 printf ' ; ' 1>&2
555 fi
556 prevDir=$ldir
557 done
558 )
559 fi
560 }
561
562 runmodel()
563 {
564 # runmodel directory
565 #
566 # runs "$COMMAND in "directory"
567 # (where "$COMMAND" is relative to "directory")
568 (
569 cd $1
570 printf 'runmodel in %s ...' $1 1>&2
571 # make output.txt
572 echo
573 if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
574 diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
575 outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi
576 fi
577 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
578 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
579 ln -sf "../"$builddir"/"$EXECUTABLE .
580 fi
581 if test ! -x $EXECUTABLE ; then
582 rm -f $RUNLOG ; touch $RUNLOG
583 if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
584 echo " no executable:" $EXECUTABLE >> $RUNLOG
585 RETVAL=8
586 ENDVAL=-1
587 else
588 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
589 # output do not exist or is older than executable:
590 rm -f $RUNLOG ; touch $RUNLOG
591 if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
592 ( eval $COMMAND ) >> $RUNLOG 2>&1
593 RETVAL=$?
594 else
595 RETVAL=0
596 if test -f $RUNLOG ; then
597 if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
598 echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
599 else
600 touch $RUNLOG
601 if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
602 echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
603 echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1
604 fi
605 fi
606 ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'`
607 fi
608 rm -f run.log_tmp
609 #if test "x$RETVAL" = x0 ; then
610 if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
611 tail $RUNLOG
612 echo successful 1>&2
613 # === Reduce the size of the testing emails!
614 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
615 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
616 return 0
617 else
618 tail $RUNLOG
619 echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2
620 cp $RUNLOG $CDIR"/"$RUNLOG
621 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
622 return 1
623 fi
624 )
625 }
626
627 createcodelet()
628 {
629 # create codelet for comparing model output
630
631 printf "creating the comparison code (using CC=$CC)... "
632 cat > tr_cmpnum.c <<EOF
633 #include <stdio.h>
634 #include <math.h>
635 int main( int argc, char** argv ) {
636 int linnum,cmplin,best,lncnt;
637 double a,b,abave,relerr;
638 best = -22;
639 lncnt = 0;
640 while( 1 & ( (lncnt+=1) < 999 ) ) {
641 scanf("%d", &linnum);
642 if (linnum == -1) break;
643 scanf("%lf", &a); scanf("%lf", &b);
644 abave = 0.5*(fabs(a)+fabs(b));
645 if ( abave == abave ) {
646 if (abave > 0.0) {
647 relerr=fabs(a-b)/abave;
648 if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
649 else { cmplin = -16 ; }
650 best = (best > cmplin) ? best : cmplin; }
651 else { cmplin = -22 ; }
652 /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
653 }
654 else {
655 /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
656 break; }
657 }
658 if (lncnt == 999) best=-29;
659 if (linnum != -1) best=-99;
660 printf("%d\n", -best);
661 return 0;
662 }
663 EOF
664 $CC -o tr_cmpnum tr_cmpnum.c -lm
665
666 if [ -x ./tr_cmpnum ]; then
667 echo "OK"
668 return 0
669 else
670 echo
671 echo "ERROR: failed to compile comparison code -- please specify"
672 echo " a C compiler using the CC environment variable."
673 exit 1
674 fi
675 }
676
677 formatresults()
678 {
679 # formatresults expt genmake depend make run results*
680
681 nm=$1
682 printf '%s %s %s %s' $2 $3 $4 $5
683 shift; shift; shift; shift; shift;
684 listPrt=$@
685 listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
686 xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
687 printf '%3s' $listPrt
688 # line below does not work on hp-ux_ia64 : do those substitutions later on
689 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
690
691 if [ $xx = '..' ]; then
692 printf ' N/O '
693 elif [ $xx = '--' ]; then
694 printf ' N/O '
695 elif [ $xx = 99 ]; then
696 printf ' N/O '
697 else
698 if [ $xx -ge $MATCH_CRIT ]; then
699 printf ' pass'
700 else
701 printf ' FAIL'
702 fi
703 fi
704 printf ' %s' $nm
705 printf '\n'
706
707 }
708
709 scandirs()
710 {
711 if [ $# -eq 1 ]; then
712 for arg in * ; do
713 test -f $arg/$1 && echo $arg
714 done
715 else
716 echo $*
717 fi
718 }
719
720
721 ###############################################################################
722
723
724 # Default properties
725 debug=0
726 verbose=1
727 clean=0
728
729 IEEE=true
730 if test "x$MITGCM_IEEE" != x ; then
731 IEEE=$MITGCM_IEEE
732 fi
733 GSL=f
734
735 CLEANUP=f
736 QUICK=f
737 NOGENMAKE=f
738 NOCLEAN=f
739 NODEPEND=f
740 POSTCLEAN=f
741
742 BASH=
743 OPTFILE=NONE
744 ADDRESSES=
745 TESTDIRS=
746 SKIPDIRS=
747 MPACKDIR="../tools/mpack-1.6"
748 HAVE_MPACK=
749 MPACK=
750 COMMAND=
751 if test "x$MAKE" = x ; then
752 MAKE=make
753 fi
754 if test "x$CC" = x ; then
755 CC=cc
756 fi
757 JOBS=
758 MPI=f
759 MULTI_THREAD=f
760 OUTDIR=
761 DELDIR=
762
763 ADM=
764 OADM=
765
766 # list of pTracers to check for monitor output
767 PTRACERS_NUM="1 2 3 4 5"
768
769 MATCH_CRIT=13
770
771 printf "parsing options... "
772
773 ac_prev=
774 for ac_option ; do
775
776 # If the previous option needs an argument, assign it.
777 if test -n "$ac_prev"; then
778 eval "$ac_prev=\$ac_option"
779 ac_prev=
780 continue
781 fi
782
783 ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
784
785 case $ac_option in
786
787 -help | --help | -h | --h)
788 usage ;;
789
790 -optfile | --optfile | -of | --of)
791 ac_prev=OPTFILE ;;
792 -optfile=* | --optfile=* | -of=* | --of=*)
793 OPTFILE=$ac_optarg ;;
794
795 -addr | --addr | -a | --a)
796 ac_prev=ADDRESSES ;;
797 -addr=* | --addr=*)
798 ADDRESSES=$ac_optarg ;;
799 -mpackdir | --mpackdir | -mpd | --mpd)
800 ac_prev=MPACKDIR ;;
801 -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*)
802 MPACKDIR=$ac_optarg ;;
803
804 -tdir | --tdir | -t | --t)
805 ac_prev=TESTDIRS ;;
806 -tdir=* | --tdir=*)
807 TESTDIRS=$ac_optarg ;;
808
809 -skipdir | --skipdir | -skd | --skd)
810 ac_prev=SKIPDIRS ;;
811 -skipdir=* | --skipdir=*)
812 SKIPDIRS=$ac_optarg ;;
813
814 -bash | --bash | -b | --b)
815 ac_prev=BASH ;;
816 -bash=* | --bash=*)
817 BASH=$ac_optarg ;;
818
819 -command | --command | -c | --c)
820 ac_prev=COMMAND ;;
821 -command=* | --command=*)
822 COMMAND=$ac_optarg ;;
823
824 -make | --make | -m | --m)
825 ac_prev=MAKE ;;
826 -make=* | --make=*)
827 MAKE=$ac_optarg ;;
828
829 -odir | --odir)
830 ac_prev=OUTDIR ;;
831 -odir=* | --odir=*)
832 OUTDIR=$ac_optarg ;;
833
834 -ptracers | --ptracers | -ptr | --ptr)
835 ac_prev=PTRACERS_NUM ;;
836 -ptracers=* | --ptracers=* | -ptr=* | --ptr=*)
837 PTRACERS_NUM=$ac_optarg ;;
838
839 -match | --match ) ac_prev=MATCH_CRIT ;;
840 -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
841
842 -j) ac_prev=JOBS ;;
843 -j=*) JOBS=$ac_optarg ;;
844
845 -clean | --clean)
846 CLEANUP=t ; DELDIR=t ;;
847
848 -quick | --quick | -q | --q)
849 QUICK=t ;;
850 -nogenmake | --nogenmake | -ng | --ng)
851 NOGENMAKE=t ;;
852 -noclean | --noclean | -nc | --nc)
853 NOCLEAN=t ;;
854 -nodepend | --nodepend | -nd | --nd)
855 NODEPEND=t ;;
856
857 -postclean | --postclean | -pc | --pc)
858 POSTCLEAN=t ;;
859
860 -mpi) MPI=t ;;
861
862 -mth) MULTI_THREAD=t ;;
863
864 -adm | -ad) ADM=t ;;
865
866 -oad) OADM=t; NODEPEND=t ;;
867
868 -ieee) IEEE=true ;;
869 -noieee) IEEE= ;;
870 -gsl) GSL=t ;;
871
872 -verbose) verbose=2 ;;
873 -debug) debug=1 ;;
874 -quiet) verbose=0 ;;
875
876 -deldir | -dd) DELDIR=t ;;
877
878 -ts) TS=t;;
879
880 -papis) PAPIS=t;;
881
882 -pcls) PCL=t;;
883
884 -*)
885 echo "Error: unrecognized option: "$ac_option
886 usage
887 ;;
888
889 *)
890 echo "Error: unrecognized argument: "$ac_option
891 usage
892 ;;
893
894 esac
895
896 done
897
898 if test "x$QUICK" = xt ; then
899 NOGENMAKE=t
900 NOCLEAN=t
901 NODEPEND=t
902 fi
903
904 #- setting for forward or ADM testing
905 if test "x$ADM" = xt ; then
906 code_dir=code_ad
907 inputdir=input_ad
908 ref_outp="output_adm.txt"
909 EXECUTABLE="mitgcmuv_ad"
910 elif test "x$OADM" = xt ; then
911 code_dir=code_oad
912 inputdir=input_oad
913 ref_outp="output_oadm.txt"
914 EXECUTABLE="mitgcmuv_ad"
915 else
916 code_dir=code
917 inputdir=input
918 ref_outp="output.txt"
919 EXECUTABLE="mitgcmuv"
920 fi
921
922 if test "x$TESTDIRS" = x ; then
923 LIST=`scandirs results/$ref_outp`
924 else
925 #- expand group of experiments:
926 LIST=" "
927 for xx in $TESTDIRS
928 do
929 case $xx in
930 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
931 LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
932 LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
933 ;;
934 'tutorials')
935 LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
936 *) LIST=${LIST}" "$xx ;;
937 esac
938 done
939 fi
940 #echo 'LIST='${LIST}'<'
941 #- skip dirs, remove duplicate and non-directory:
942 TESTDIRS=" "
943 count=0
944 for xx in $LIST
945 do
946 yy=`echo $SKIPDIRS | grep -c $xx`
947 if test $yy = 0 ; then
948 if test -d $xx ; then
949 yy=`echo $TESTDIRS | grep -c $xx`
950 if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
951 else count=1 ;
952 echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
953 fi
954 else
955 if test $count = 1 ; then echo -n ", \"$xx\""
956 else count=1 ; echo "" ; echo -n " skip: \"$xx\""
957 fi
958 fi
959 done
960 if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
961 #echo 'TESTDIRS='${TESTDIRS}'<'
962
963 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
964 OPTFILE=$MITGCM_OF
965 fi
966
967 RUNLOG="run.log"
968 OUTPUTFILE=$ref_outp
969 if test "x$COMMAND" = x ; then
970 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
971 fi
972 if test "x$MPI" = xt ; then
973 OUTPUTFILE="STDOUT.0000"
974 fi
975
976 echo "OK (COMMAND= $COMMAND )"
977
978 # set the Default List of output variables to be checked:
979 # (use default or load experiment-specific list from file "tr_checklist")
980 # content : 1rst = main variable used to decide if it pass or FAIL
981 # others = number of matching digits to be printed in summary.txt
982 if test "x$ADM" = x -a "x$OADM" = x; then
983 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
984 EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
985 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
986 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
987 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
988 else
989 DEF_CHECK_LIST='Grad Cost Grad'
990 EMPTY_RESULTS='.. ..'
991 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
992 fi
993
994 # create the FORTRAN comparison code
995 createcodelet
996
997 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
998 if test "x$ADDRESSES" = x ; then
999 echo "skipping mpack build"
1000 else
1001 build_mpack
1002 fi
1003
1004 # Create a uniquely named directory to store results
1005 CMDLINE=$0
1006 for xx in "$@" ; do nw=`echo $xx | wc -w`
1007 if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
1008 else CMDLINE="$CMDLINE '$xx'" ; fi
1009 done
1010 #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
1011 MACH=`hostname`
1012 UNAMEA=`uname -a`
1013 DATE=`date +%Y%m%d`
1014 BASE="tr_"$MACH"_"$DATE"_"
1015 if test "x$OUTDIR" != x ; then
1016 BASE="tr_"$OUTDIR"_"$DATE"_"
1017 fi
1018 DNUM=0
1019 DRESULTS="$BASE$DNUM"
1020 while test -e $DRESULTS ; do
1021 DNUM=$(( $DNUM + 1 ))
1022 DRESULTS="$BASE$DNUM"
1023 done
1024 mkdir $DRESULTS
1025 RETVAL=$?
1026 if test "x$RETVAL" != x0 ; then
1027 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1028 exit 1
1029 fi
1030 SUMMARY="$DRESULTS/summary.txt"
1031 start_date=`date`
1032 echo $start_date > $SUMMARY
1033 echo 'run:' $CMDLINE >> $SUMMARY
1034 echo 'on :' $UNAMEA >> $SUMMARY
1035
1036 of_path=
1037 if test "x$OPTFILE" != xNONE ; then
1038 if test -r $OPTFILE ; then
1039 # get the path
1040 path=${OPTFILE%/*}
1041 if test "x$path" = x ; then
1042 of_path=`pwd`
1043 else
1044 of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1045 fi
1046 file=${OPTFILE##*/}
1047 OPTFILE=$of_path/$file
1048 cp $OPTFILE $DRESULTS
1049 echo >> $SUMMARY
1050 echo " OPTFILE=$OPTFILE" >> $SUMMARY
1051 else
1052 echo | tee $SUMMARY
1053 echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1054 exit 1
1055 fi
1056 else
1057 echo >> $SUMMARY
1058 echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY
1059 echo " so the genmake default will be used." >> $SUMMARY
1060 fi
1061 echo
1062 echo >> $SUMMARY
1063 if test "x$ADM" = x -a "x$OADM" = x; then
1064 if [ $MATCH_CRIT -lt 10 ] ;
1065 then line_0="default "$MATCH_CRIT ;
1066 else line_0="default "$MATCH_CRIT ; fi
1067 line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1068 # line_0=" ----T----- ----S----- ----U----- ----V-----"
1069 line_1="G D M c m s m s m s m s"
1070 line_2="E p a R g m m e . m m e . m m e . m m e ."
1071 line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1072 line_4="2 d e n d n x n . n x n . n x n . n x n ."
1073 for ii in $PTRACERS_NUM ; do
1074 line_0="$line_0 --PTR 0"$ii"--"
1075 line_1="$line_1 m s"
1076 line_2="$line_2 m m e ."
1077 line_3="$line_3 i a a d"
1078 line_4="$line_4 n x n ."
1079 done
1080 echo "$line_0" | tee -a $SUMMARY
1081 echo "$line_1" | tee -a $SUMMARY
1082 echo "$line_2" | tee -a $SUMMARY
1083 echo "$line_3" | tee -a $SUMMARY
1084 echo "$line_4" | tee -a $SUMMARY
1085 echo " " | tee -a $SUMMARY
1086 else
1087 echo "ADJOINT=true" >> $SUMMARY
1088 echo >> $SUMMARY
1089 if [ $MATCH_CRIT -lt 10 ] ;
1090 then line_0="default "$MATCH_CRIT ;
1091 else line_0="default "$MATCH_CRIT ; fi
1092 echo "$line_0" | tee -a $SUMMARY
1093 cat << EOF | tee -a $SUMMARY
1094 G D M C G
1095 E p a R o r
1096 N n k u s a
1097 2 d e n t d
1098
1099 EOF
1100 fi
1101
1102 # ...and each test directory...
1103 for dir in $TESTDIRS ; do
1104
1105 # set builddir & rundir:
1106 builddir="build"
1107 if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1108 rundir="run"
1109 if test ! -d $dir/$rundir ; then
1110 rundir=$builddir
1111 fi
1112 CODE_DIR=$dir/$code_dir
1113 BUILD_DIR=$dir/$builddir
1114
1115 # Cleanup only!
1116 if test "x$CLEANUP" = xt ; then
1117 if test -r $BUILD_DIR/Makefile ; then
1118 echo ' ------ clean dir:' $dir/$builddir
1119 ( cd $BUILD_DIR ; make CLEAN )
1120 fi
1121 if test -d $dir/$rundir/CVS ; then
1122 echo ' ------ clean dir:' $dir/$rundir
1123 run_clean $dir/$rundir
1124 fi
1125 (
1126 cd $dir
1127 rm -rf tr_run.*
1128 )
1129 continue
1130 fi
1131
1132 # Verify that the testdir exists and contains previous
1133 # results in the correct location--or skip this directory!
1134 fout=$dir"/results/"$ref_outp
1135 if test ! -r $fout ; then
1136 echo "can't read \"$fout\" -- skipping $dir"
1137 continue
1138 fi
1139
1140 # Check for specific files for particular type of run
1141
1142 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then
1143 echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1144 continue
1145 fi
1146 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1147 echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1148 continue
1149 fi
1150
1151 # Check whether there are "extra runs" for this testdir
1152 extra_runs=
1153 ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
1154 #echo "ex_run_dirs='$ex_run_dirs'"
1155 for exd in $ex_run_dirs ; do
1156 name=`echo $exd | sed -e "s/$inputdir\.//"`
1157 refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1158 outf="$dir/results/$refExOut"
1159 if test -f $outf -a -r $outf ; then
1160 if test "x$MULTI_THREAD" = "xt" ; then
1161 if test -r $dir"/"$exd"/eedata.mth" ; then
1162 extra_runs="$extra_runs $name"
1163 #else echo $dir"/"$exd"/eedata.mth: not found"
1164 fi
1165 else
1166 extra_runs="$extra_runs $name"
1167 fi
1168 fi
1169 done
1170
1171 echo "-------------------------------------------------------------------------------"
1172 echo
1173 if test "x$extra_runs" = "x" ; then
1174 echo "Experiment: $dir"
1175 else
1176 echo "Experiment: $dir ; extra_runs=$extra_runs"
1177 fi
1178 echo
1179 unset genmake makedepend make run
1180 results=$EMPTY_RESULTS
1181
1182 # Create an output dir for each OPTFILE/tdir combination
1183 rel_CDIR=$DRESULTS"/"$dir
1184 mkdir $rel_CDIR
1185 CDIR=`pwd`"/$rel_CDIR"
1186
1187 if test "x$CLEANUP" = xt ; then
1188 echo '====>>> this is to check that we never go through this part <<< ==='
1189 makeclean $dir/$builddir \
1190 && run_clean $dir/$rundir
1191 else
1192 genmakemodel $dir/$builddir && genmake=Y \
1193 && makeclean $dir/$builddir \
1194 && run_clean $dir/$rundir \
1195 && symlink_mpifiles $dir $code_dir $builddir \
1196 && makedependmodel $dir/$builddir && makedepend=Y \
1197 && makemodel $dir/$builddir && make=Y \
1198 && linkdata $dir/$rundir $inputdir input \
1199 && runmodel $dir/$rundir && run=Y \
1200 && results=`testoutput_run $dir $rundir $ref_outp`
1201 fi
1202
1203 echo
1204 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1205 echo
1206 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1207 touch $CDIR"/summary.txt"
1208 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1209 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1210 echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1211 echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1212 echo "tdir='$dir'" >> $CDIR"/summary.txt"
1213 if test "x$ADM" = xt ; then
1214 head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1215 grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1216 >> $CDIR"/summary.txt"
1217 fi
1218
1219 for ex in $extra_runs ; do
1220 unset run
1221 results=$EMPTY_RESULTS
1222 # reference output file
1223 refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1224 # Create an output dir for each OPTFILE/tdir.ex combination
1225 rel_CDIR=$DRESULTS"/"$dir"."$ex
1226 mkdir $rel_CDIR
1227 CDIR=`pwd`"/$rel_CDIR"
1228 test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex"
1229 run_clean $dir/tr_run.$ex
1230 linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input
1231 runmodel $dir/tr_run.$ex && run=Y \
1232 && results=`testoutput_run $dir tr_run.$ex $refExOut`
1233 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1234 fres="$fres.$ex"
1235 echo
1236 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1237 touch $CDIR"/summary.txt"
1238 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt"
1239 echo "MACH='$MACH'" >> $CDIR"/summary.txt"
1240 echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt"
1241 echo "DATE='$DATE'" >> $CDIR"/summary.txt"
1242 echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt"
1243 if test "x$ADM" = xt ; then
1244 head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt"
1245 grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \
1246 >> $CDIR"/summary.txt"
1247 fi
1248 if test "x$POSTCLEAN" = xt ; then
1249 run_clean $dir/tr_run.$ex
1250 fi
1251 done
1252
1253 #postclean $dir/$builddir
1254 if test "x$POSTCLEAN" = xt ; then
1255 makeclean $dir/$builddir \
1256 && run_clean $dir/$rundir
1257 fi
1258
1259 echo "-------------------------------------------------------------------------------"
1260
1261 done
1262
1263 printf "Start time: " >> $SUMMARY
1264 echo "$start_date" >> $SUMMARY
1265 printf "End time: " >> $SUMMARY
1266 date >> $SUMMARY
1267
1268 # If addresses were supplied and mpack built successfully, then try
1269 # to send email using mpack.
1270 if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1271 echo "No results email was sent."
1272 else
1273 if test "x$HAVE_MPACK" = xt ; then
1274 tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1275 && gzip $DRESULTS".tar" \
1276 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1277 RETVAL=$?
1278 if test "x$RETVAL" != x0 ; then
1279 echo
1280 echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1281 echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1282 echo " summary of results from the directory \"$DRESULTS\"."
1283 echo
1284 else
1285 echo
1286 echo "An email containing results was sent to the following addresses:"
1287 echo " \"$ADDRESSES\""
1288 echo
1289 fi
1290 test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1291 test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1292 fi
1293 fi
1294
1295 rm -f tr_cmpnum.c tr_cmpnum
1296
1297 if test "x$CLEANUP" != xt ; then
1298 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1299 if test -e tr_out.txt ; then
1300 mv tr_out.txt tr_out.txt.old
1301 fi
1302 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1303 fi
1304
1305 if test "x$DELDIR" = xt ; then
1306 rm -rf $DRESULTS
1307 fi

  ViewVC Help
Powered by ViewVC 1.1.22