/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.157 - (hide annotations) (download)
Thu Jan 27 04:51:03 2011 UTC (13 years, 2 months ago) by jmc
Branch: MAIN
Changes since 1.156: +5 -2 lines
report Nb of TAF ERROR & Nb of TAF RECOMPUTATION WARNING to local summary.txt

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.157 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.156 2011/01/20 21:55:52 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.122 echo " (-mth) run multi-threaded (using eedata.mth)"
15 jmc 1.154 echo " (-mpi) use MPI to compile and run on 2 processors"
16     echo " (-MPI) NUMBER use MPI to compile and run on max NUMBER procs"
17 jmc 1.156 echo " (-mfile|-mf) STRING MPI: file with list of possible machines to run on"
18 jmc 1.136 echo " (-ieee/-noieee) if possible, use IEEE compiler flags"
19 edhill 1.41 echo " (DEF=\"-ieee\")"
20 jmc 1.109 echo " (-gsl) compile with \"-gsl\" flag"
21 jmc 1.136 echo " (-use_r4|-ur4) if allowed, use real*4 type for '_RS' variable"
22     echo " (-optfile|-of) STRING list of optfiles to use"
23     echo " (-addr|-a) STRING list of email recipients"
24 jmc 1.108 echo " (DEF=\"\" no email is sent)"
25 jmc 1.136 echo " (-mpackdir|-mpd) DIR location of the mpack utility"
26 jmc 1.113 echo " (DEF=\"../tools/mpack-1.6\")"
27 jmc 1.136 echo " (-tdir|-t) STRING list of group and/or exp. dirs to test"
28 jmc 1.84 echo " (recognized groups: basic, tutorials)"
29     echo " (DEF=\"\" which test all)"
30 jmc 1.149 echo " (if list= 'start_from THIS_EXP' then"
31     echo " test THIS_EXP + all the following)"
32 jmc 1.136 echo " (-skipdir|-skd) STRING list of exp. dirs to skip"
33 jmc 1.89 echo " (DEF=\"\" which test all)"
34 jmc 1.136 echo " (-bash|-b) STRING preferred location of a \"bash\" or"
35 edhill 1.20 echo " Bourne-compatible \"sh\" shell"
36     echo " (DEF=\"\" for \"bash\")"
37 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
38 utke 1.120 echo " (-oad) perform an OpenAD adjoint run"
39 jmc 1.136 echo " (-command|-c) STRING command to run"
40 jmc 1.141 echo " (-makedepend|-md) STRING command to use for \"makedepend\""
41 jmc 1.136 echo " (-make|-m) STRING command to use for \"make\""
42 edhill 1.8 echo " (DEF=\"make\")"
43 jmc 1.70 echo " (-odir) STRING used to build output directory name"
44     echo " (DEF=\"hostname\")"
45 jmc 1.156 # echo " (-ptracers|-ptr) STRING specify which ptracers to test"
46     # echo " (DEF=\"1 2 3 4 5\")"
47 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
48 jmc 1.141 echo " (DEF=\"$MATCH_CRIT\")"
49 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
50 jmc 1.156 echo " (-clean) *ONLY* run \"make CLEAN\" & clean run-dir"
51 jmc 1.133 echo " (-norun|-nr) skip the \"runmodel\" stage (stop after make)"
52 jmc 1.136 echo " (-runonly|-ro) *ONLY* run stage (=\"-quick\" without make)"
53 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
54     echo " (-nogenmake|-ng) skip the genmake stage"
55     echo " (-noclean|-nc) skip the \"make clean\" stage"
56     echo " (-nodepend|-nd) skip the \"make depend\" stage"
57 jmc 1.156 echo " (-postclean|-pc) after each exp. test, clean build-dir & run-dir"
58     echo " (-deloutp|-do) delete output files after successful run"
59 ce107 1.90 echo " (-deldir|-dd) on success, delete the output directory"
60     echo " (-ts) provide timing information per timestep"
61     echo " (-papis) provide MFlop/s per timestep using PAPI"
62     echo " (-pcls) provide MFlop/s per timestep using PCL"
63 edhill 1.6 echo
64 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
65 edhill 1.6 echo "such as:"
66 edhill 1.50 echo
67 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
68     echo " -addr='abc@123.com testing@home.org'"
69 edhill 1.1 echo
70 edhill 1.50 echo "provided that the expression is properly quoted within the current"
71     echo "shell (note the use of single quotes to protect white space)."
72     echo
73 edhill 1.1 exit 1
74     }
75    
76     # build the mpack utility
77     build_mpack()
78     {
79 edhill 1.34 printf "building the mpack utility... "
80 jmc 1.113 MPACK="$MPACKDIR/mpack"
81     if test ! -x $MPACK ; then
82 edhill 1.1 if test ! -d $MPACKDIR ; then
83 edhill 1.20 echo
84 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
85     echo " are you sure this program is being run in the correct "
86     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
87 edhill 1.20 echo
88     HAVE_MPACK=f
89 edhill 1.1 fi
90 edhill 1.26 if test "x$CC" = x ; then
91     export CC=cc
92     fi
93 jmc 1.112 printf "building mpack (using CC=$CC)... "
94 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
95 edhill 1.1 RETVAL=$?
96     if test "x$RETVAL" != x0 ; then
97     echo
98     echo "Error building the mpack tools at: $MPACK_DIR"
99 edhill 1.20 echo
100     HAVE_MPACK=f
101     else
102 edhill 1.25 rm -f tr_build_mpack.out
103 edhill 1.20 HAVE_MPACK=t
104 jmc 1.113 echo "done"
105 edhill 1.1 fi
106 edhill 1.20 else
107     HAVE_MPACK=t
108 jmc 1.113 echo "already exist"
109 edhill 1.1 fi
110     }
111    
112 jmc 1.93 testoutput_var()
113 edhill 1.1 {
114 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
115 edhill 1.1 #
116 jmc 1.135 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
117 jmc 1.99 # with same output from reference file $dir/results/$reference_output
118 jmc 1.93 # using search strings s1 and text label
119 edhill 1.1
120     if [ $debug -gt 0 ]; then
121 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
122 edhill 1.1 fi
123 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
124     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
125 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
126 jmc 1.100 if [ $lncntA -lt 2 ]; then
127 edhill 1.1 if [ $verbose -gt 0 ]; then
128     echo Not enough lines of output when searching for "$2" 1>&2
129     fi
130     return 99
131     fi
132     else
133 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
134 edhill 1.1 return 99
135     fi
136     if [ $debug -gt 0 ]; then
137 jmc 1.99 echo testoutput_var: grep "$2" $1/results/$5 1>&2
138 edhill 1.1 fi
139 jmc 1.99 grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt
140 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
141 jmc 1.100 if [ $lncntB -lt 2 ]; then
142 edhill 1.1 if [ $verbose -gt 0 ]; then
143     echo Not enough lines of output when searching for "$2" 1>&2
144     fi
145     return 99
146     fi
147 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
148     if [ $verbose -gt 0 ]; then
149     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
150     fi
151     return 99
152     fi
153 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
154     if [ $has_nan -gt 0 ] ; then
155 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
156 edhill 1.72 return 99
157     fi
158     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
159     if [ $has_inf -gt 0 ] ; then
160 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
161 jmc 1.71 return 99
162     fi
163 edhill 1.1 if [ $debug -gt 0 ]; then
164 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
165 edhill 1.1 fi
166     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
167     if [ $debug -gt 0 ]; then
168 jmc 1.93 echo testoutput_var: compare_lines 1>&2
169 edhill 1.1 fi
170 edhill 1.22 if [ $verbose -gt 1 ]; then
171     cat tmp3.txt 1>&2
172     fi
173     echo "-1" >> tmp3.txt
174 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
175     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
176 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
177 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
178     if [ $verbose -gt 0 ]; then
179 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
180 edhill 1.1 fi
181     digits_of_similarity=99
182     else
183     if [ $verbose -gt 0 ]; then
184 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
185 edhill 1.1 fi
186     fi
187 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
188 jmc 1.135
189 edhill 1.1 return $digits_of_similarity
190     }
191    
192 jmc 1.93 testoutput_run()
193 edhill 1.1 {
194 jmc 1.99 # testoutput_run directory subdir reference_output
195 edhill 1.1 #
196 jmc 1.93 # test output from 1 run in "directory"
197 jmc 1.103 # --> same processing for adjoint & forward test
198 jmc 1.94 # default list of output variables to be checked:
199     # 1rst : main variable used to decide if it pass or FAIL
200     # others : number of matching digits to be printed in summary.txt
201     listChk=$DEF_CHECK_LIST
202     # load experiment-specific list from file "tr_checklist" (if it exist)
203     if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
204     sVar=`echo $listChk | awk '{print $1}'`
205     # remove 1rst var and expand the list: + => min max mean s.d
206 jmc 1.97 listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
207     | sed 's/+//g' | sed "s/^$sVar//"`
208 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
209 jmc 1.117 # check for ptracer output in reference_output file :
210     outpref=$1/results/$3
211     ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
212     ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
213     for ii in $PTRACERS_NUM ; do
214     ptrfound=0
215     for jj in $ptr_mon ; do
216     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
217     tst=`grep $name $outpref | wc -l | awk '{print $1}'`
218     if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
219     done
220     if test $ptrfound = '1' ; then
221     eval "HAVE_PTR0"$ii"=t"
222     else
223     eval "HAVE_PTR0"$ii"=f"
224 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
225 jmc 1.117 # remove this ptr from the list of output variable to check
226     # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
227     listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
228     fi
229     fi
230     # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
231     done
232 mlosch 1.116 tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
233 jmc 1.94 if test $tst != 1 ; then
234     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
235     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
236     echo "==> WARNING: in checked list:" $listVar 1>&2
237     #- put it back once:
238     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
239     fi
240     if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
241 jmc 1.154 #- report to this experiment local summary file ---
242     echo "MACH='$MACH'" > $locDIR"/summary.txt"
243     echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
244     echo "DATE='$DATE'" >> $locDIR"/summary.txt"
245     grep '(PID\.TID 0000\.0001) n.. =' $1/$2/$OUTPUTFILE \
246     | sed 's/(PID.TID 0000.0001) //' >> $locDIR"/summary.txt"
247     echo "listVar='$listVar'" >> $locDIR"/summary.txt"
248     #---
249 jmc 1.94 allargs=""
250     for xx in $listVar
251     do
252     case $xx in
253     'PS') if [ $debug -gt 0 ]
254     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
255 jmc 1.99 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$?
256 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
257 jmc 1.101 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;;
258     'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;;
259 jmc 1.99 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;;
260     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;;
261     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;;
262     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;;
263     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;;
264     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;;
265     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;;
266     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;;
267 jmc 1.94 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;;
268     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;;
269     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;;
270 jmc 1.99 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;;
271 jmc 1.94 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;;
272     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;;
273     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;;
274 jmc 1.99 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;;
275 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
276     testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;;
277     'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
278     testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;;
279     'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
280     testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;;
281     'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
282 jmc 1.99 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;;
283 jmc 1.130 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;;
284     'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;;
285     'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;;
286     'Qntsd') testoutput_var $1 "forcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;;
287 jmc 1.101 'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;;
288     'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;;
289     'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;;
290     'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;;
291     'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;;
292 jmc 1.126 'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Heff max" $2 $3; yy=$? ;;
293     'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Heff mean" $2 $3; yy=$? ;;
294     'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Heff StDv" $2 $3; yy=$? ;;
295 jmc 1.101 'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;;
296     'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;;
297     'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;;
298     'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;;
299     'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;;
300     'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;;
301 jmc 1.94 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
302     esac
303     if test $xx = $sVar
304     then allargs="$allargs > $yy <"
305     else allargs="$allargs $yy"
306     fi
307     done
308    
309     nbVar=`echo $listVar | awk '{print NF}'`
310 jmc 1.135 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
311 jmc 1.94 #-- fill line (up to standard length) with dot:
312 jmc 1.97 adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
313 jmc 1.94 echo $allargs $adNul
314     else
315     echo $allargs
316 jmc 1.135 fi
317 jmc 1.103 # <-- same processing for adjoint & forward test
318 edhill 1.1 }
319    
320     genmakemodel()
321     {
322     # genmakemodel directory
323 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
324     echo "genmake skipped!"
325     else
326 edhill 1.34 if test "x$BASH" = x ; then
327     GENMAKE2="../../../tools/genmake2"
328     else
329     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
330     fi
331 edhill 1.10 (
332     cd $1;
333 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
334 jmc 1.141 if test "x$MKDEPEND" != x ; then
335     command="$command -makedepend=$MKDEPEND"
336     fi
337 utke 1.120 if test "x$ADM" = xt ; then
338     command="$command --mods=../code_ad"
339     elif test "x$OADM" = xt ; then
340     command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
341     else
342 jmc 1.124 command="$command -mods=../code"
343 edhill 1.24 fi
344 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
345 jmc 1.124 command="$command -optfile=$OPTFILE"
346 edhill 1.10 fi
347     if test "x$IEEE" != x ; then
348     command="$command -ieee"
349     fi
350 jmc 1.109 if test "x$GSL" = xt ; then
351     command="$command -gsl"
352     fi
353 jmc 1.154 if test "x$MPI" != x0 ; then
354 edhill 1.66 command="$command -mpi"
355     fi
356 jmc 1.122 if test "x$MULTI_THREAD" = xt ; then
357     #- run multi-threaded using OpenMP:
358     command="$command -omp"
359     fi
360 jmc 1.135 if test "x$USE_R4" = xt ; then
361     command="$command -use_r4"
362     fi
363 ce107 1.90 if test "x$TS" = xt ; then
364     command="$command -ts"
365     fi
366     if test "x$PAPIS" = xt ; then
367     command="$command -papis"
368 jmc 1.135 else
369 ce107 1.90 if test "x$PCLS" = xt ; then
370     command="$command -pcls"
371     fi
372     fi
373 jmc 1.139 printf 'genmake ... '
374 jmc 1.146 eval $command > genmake.tr_log 2>&1
375 edhill 1.10 RETVAL=$?
376 edhill 1.44 # Reduce the size of the testing emails!
377 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
378 edhill 1.10 if test "x$RETVAL" != x0 ; then
379 jmc 1.146 tail genmake.tr_log
380 jmc 1.139 echo "genmakemodel: genmake failed"
381 jmc 1.148 cp genmake.log genmake_* genmake.tr_log $CDIR
382 edhill 1.10 return 1
383     else
384 jmc 1.139 echo "successful"
385 edhill 1.1 fi
386 edhill 1.10 )
387     fi
388 edhill 1.1 }
389    
390     makeclean()
391     {
392     # makeclean directory
393 jmc 1.146 if test "x$NODEPEND" = xf ; then rm -f $1/make.tr_log ; fi
394 edhill 1.10 if test "x$NOCLEAN" = xt ; then
395 jmc 1.83 echo "make Clean skipped!"
396 edhill 1.10 else
397     (
398     cd $1;
399 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
400 edhill 1.10 if test -r Makefile ; then
401 jmc 1.139 printf 'clean build-dir: make Clean ... '
402 jmc 1.146 $MAKE Clean >> make.tr_log 2>&1
403 edhill 1.10 RETVAL=$?
404     if test "x$RETVAL" != x0 ; then
405 jmc 1.146 tail make.tr_log
406 jmc 1.139 echo "makeclean: \"make Clean\" failed"
407 jmc 1.148 cp make.tr_log genmake.log genmake.tr_log $CDIR
408 edhill 1.10 return 1
409     fi
410 jmc 1.139 echo successful
411 jmc 1.127 else
412     echo ''
413 edhill 1.10 fi
414     exit 0
415     )
416     fi
417     }
418    
419 jmc 1.83 run_clean()
420 edhill 1.68 {
421 jmc 1.83 # run_clean directory
422     if test "x$NOCLEAN" = xt ; then
423     echo "run_clean skipped!"
424     else
425 edhill 1.68 (
426     cd $1;
427 jmc 1.139 printf 'clean run-dir ... '
428 jmc 1.83 # part of what is done after "make clean" when doing "make CLEAN"
429     find . -name "*.meta" -exec rm {} \;
430     find . -name "*.data" -exec rm {} \;
431     find . -name "fort.*" -exec rm {} \;
432     find . -type l -exec rm {} \;
433 jmc 1.145 rm -f $EXECUTABLE $RUNLOG *.txt STD* *diagnostics.log datetime
434 jmc 1.83 rm -rf mnc_test_*
435 jmc 1.107 rm -f *_MIT_CE_000.opt0000 costfunction*0000
436 jmc 1.139 echo successful
437 edhill 1.68 exit 0
438     )
439     fi
440     }
441    
442 edhill 1.10 makedependmodel()
443     {
444     # makedependmodel directory
445     if test "x$NODEPEND" = xt ; then
446     echo "make depend skipped!"
447     else
448     (
449     cd $1;
450 jmc 1.139 printf 'make depend ... '
451 jmc 1.146 $MAKE depend >> make.tr_log 2>&1
452 edhill 1.1 RETVAL=$?
453     if test "x$RETVAL" != x0 ; then
454 jmc 1.146 tail make.tr_log
455 jmc 1.139 echo "makedependmodel: make depend failed"
456 jmc 1.148 cp make.tr_log genmake.log genmake.tr_log $CDIR
457 edhill 1.1 return 1
458 edhill 1.10 else
459 jmc 1.139 echo successful
460 edhill 1.1 fi
461 edhill 1.10 )
462     fi
463 edhill 1.1 }
464    
465     makemodel()
466     {
467     # makemodel directory
468     (
469 jmc 1.136 if test "x$NOMAKE" = xt ; then
470     cd $1;
471     if test -x $EXECUTABLE ; then
472     echo "make skipped!"
473     else
474     echo "no executable!"
475     return 2
476     fi
477     else
478 edhill 1.1 cd $1;
479     if test -r Makefile ; then
480 jmc 1.139 printf 'make ... '
481 utke 1.120 if test "x$ADM" = xt ; then
482 jmc 1.146 $MAKE adall >> make.tr_log 2>&1
483 utke 1.120 elif test "x$OADM" = xt ; then
484 jmc 1.146 $MAKE adAll >> make.tr_log 2>&1
485 utke 1.120 else
486 edhill 1.43 if test "x$JOBS" = x ; then
487 jmc 1.146 $MAKE >> make.tr_log 2>&1
488 edhill 1.43 else
489 jmc 1.146 $MAKE -j $JOBS >> make.tr_log 2>&1
490 edhill 1.43 fi
491 edhill 1.24 fi
492 edhill 1.1 RETVAL=$?
493     if test "x$RETVAL" != x0 ; then
494 jmc 1.146 tail make.tr_log
495 jmc 1.139 echo failed
496 jmc 1.148 cp make.tr_log genmake.log genmake.tr_log $CDIR
497 jmc 1.131 rm -f $EXECUTABLE
498 edhill 1.1 return 1
499     else
500 jmc 1.139 echo successful
501 edhill 1.1 fi
502     fi
503 jmc 1.136 fi
504 edhill 1.1 )
505     }
506    
507 jmc 1.154 mk_mpi_size()
508     {
509     # mk_mpi_size input_file output_file proc_Nb
510     #
511     # make new SIZE.h (=output_file) from SIZE.h_mpi (=input_file)
512     # for an MPI build with no more than proc_Nb processors ;
513     # return the effective number of processors.
514    
515     inp=$1
516     out=$2
517     np=$3
518     tmp=TTT.$$
519    
520     px=`grep "^ & *nPx *=" $inp | sed "s/^ & *nPx *= *//" | sed 's/, *$//'`
521     py=`grep "^ & *nPy *=" $inp | sed "s/^ & *nPy *= *//" | sed 's/, *$//'`
522     sx=`grep "^ & *nSx *=" $inp | sed "s/^ & *nSx *= *//" | sed 's/, *$//'`
523     sy=`grep "^ & *nSy *=" $inp | sed "s/^ & *nSy *= *//" | sed 's/, *$//'`
524    
525     #- find the largest divisor of input_file proc Nb, but not larger than $np
526     pp=0
527 jmc 1.155 i=1
528     while [ $i -le $px ] ; do
529 jmc 1.154 if [ `expr $px % $i` -eq 0 ] ; then
530 jmc 1.155 j=1
531     while [ $j -le $py ] ; do
532     if [ `expr $py % $j` -eq 0 ] ; then
533     ij=`expr $i \* $j`
534     if [ $ij -gt $pp ] ; then
535     if [ $ij -le $np ] ; then
536     ix=$i ; jy=$j ; pp=$ij
537     #echo " ix,jy= $ix,$jy"
538     fi
539     fi
540     fi
541     j=`expr $j + 1`
542     done
543 jmc 1.154 fi
544 jmc 1.155 i=`expr $i + 1`
545 jmc 1.154 done
546    
547     #- create new SIZE.h type file:
548     sx=`expr $sx \* $px / $ix`
549     sy=`expr $sy \* $py / $jy`
550     if [ $verbose -gt 1 ]; then
551     echo " px,py,np= $px,$py,$np : New MPI size: px,py= $ix,$jy : sx,sy= $sx,$sy"
552     fi
553     sed "/^ \& *nPx *=/s/[0-9]*,/$ix,/" $inp > $tmp
554     sed "/^ \& *nPy *=/s/[0-9]*,/$jy,/" $tmp > $out
555     sed "/^ \& *nSx *=/s/[0-9]*,/$sx,/" $out > $tmp
556     sed "/^ \& *nSy *=/s/[0-9]*,/$sy,/" $tmp > $out
557     rm -f $tmp
558     return $pp
559     }
560    
561 edhill 1.27 symlink_mpifiles()
562     {
563     # Put special links so that MPI specific files are used
564     # This MUST be invoked between makeclean and makelinks because
565     # the Makefile will link to non-mpi files by default
566    
567     dir=$1
568     code_dir=$2
569 jmc 1.154 build_dir=$dir/$3
570 jmc 1.135
571 edhill 1.27 # These are files that should replace their counter-part when using -mpi
572 jmc 1.154 MPI_FILES=`(cd $dir/$code_dir; find . -name "*_mpi" -print)`
573 edhill 1.27
574 jmc 1.154 for ii in $MPI_FILES ; do
575     i=`echo $ii | sed 's:^\./::'`
576     name=`echo $i | sed 's:_mpi::'`
577     file="../$code_dir/$i"
578     if test $name = 'SIZE.h' ; then file="SIZE.h.mpi" ; fi
579    
580     # Is this an MPI run?
581     if test "x$MPI" = x0 ; then
582     # NO: We undo any _mpi symbolically linked files
583     if test -L $build_dir/$name ; then
584     ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 )
585     RETVAL=$?
586     if test "x$RETVAL" = x0 ; then
587     if [ $verbose -gt 1 ]; then
588     echo " Un-linking $name from ../$code_dir" ; fi
589     rm -f $build_dir/$name
590     fi
591     fi
592     else
593     # YES: We symbolically link these files to the build
594     # dir so long as there is no real file in place
595     ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 )
596 edhill 1.27 RETVAL=$?
597 jmc 1.154 if [ $verbose -gt 1 ]; then echo " cmp $name $file returns: $RETVAL" ; fi
598 edhill 1.27 if test "x$RETVAL" != x0 ; then
599 jmc 1.154 if test -h $build_dir/$name ; then rm -f $build_dir/$name ; fi
600     if test ! -r $build_dir/$name ; then
601     if [ $verbose -gt 1 ]; then echo " Linking $name to $file" ; fi
602     (cd $build_dir; ln -sf $file $name)
603 edhill 1.27 fi
604     fi
605 jmc 1.154 fi
606     done
607 edhill 1.27 }
608    
609 edhill 1.1 linkdata()
610     {
611 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
612 edhill 1.1 #
613     # symbolically link data files to run directory
614 jmc 1.83 if test -d $1 ; then
615 edhill 1.12 (
616 jmc 1.83 cd $1 ; shift
617 jmc 1.139 echo 'linkdata from dirs:' $*
618 jmc 1.125 inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
619     for xx in $inpMPI ; do
620     if test -r "../"$1"/"$xx ; then
621     # found 1 _mpi sfx file in 1rst input dir and it is readable
622     yy=`echo $xx | sed 's:\.mpi$::'`
623 jmc 1.154 if test "x$MPI" = "x0" ; then
624     # not mpi test: remove symbolic link
625     if test -h $yy ; then rm -f $yy ; fi
626     else
627     # mpi test: remove symbolic link & link .mpi sfx file
628 jmc 1.125 if test -h $yy ; then rm -f $yy ; fi
629 jmc 1.135 if test ! -r $yy ; then
630 jmc 1.125 ln -sf "../"$1"/"$xx $yy ;
631 jmc 1.139 printf " $xx" 1>&2
632 jmc 1.125 fi
633     fi
634     fi
635     done
636 jmc 1.83 if test -r "../"$1"/eedata.mth" ; then
637     # found eedata.mth in 1rst input dir and it is readable
638     if test "x$MULTI_THREAD" = "xt" ; then
639     # multi-threaded test: remove symbolic link & link eedata.mth
640     if test -h eedata ; then rm -f eedata ; fi
641 jmc 1.135 if test ! -r eedata ; then
642 jmc 1.83 ln -sf "../"$1"/eedata.mth" eedata ;
643 jmc 1.139 printf ' eedata.mth' 1>&2
644 edhill 1.24 fi
645 jmc 1.83 else
646     # not multi-threaded test: remove eedata symbolic link
647     if test -h eedata ; then rm -f eedata ; fi
648     fi
649     fi
650 jmc 1.108 prevDir='NONE'
651 jmc 1.83 for ldir in $* ; do
652 jmc 1.108 if test -d "../"$ldir -a $ldir != $prevDir ; then
653 jmc 1.139 printf " ldir=${ldir}:" 1>&2
654 jmc 1.83 files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
655     for i in $files ; do
656     if test ! -d "../"$ldir/$i ; then
657     if test ! -r $i ; then
658     printf ' '$i 1>&2
659     ln -sf "../"$ldir"/"$i $i
660     fi
661     fi
662     done
663 jmc 1.139 printf ' ;' 1>&2
664 jmc 1.83 if test -x "../"$ldir"/"prepare_run ; then
665 jmc 1.139 "../"$ldir"/"prepare_run 1>&2
666     else
667     echo '' 1>&2
668 edhill 1.24 fi
669 jmc 1.82 fi
670 jmc 1.108 prevDir=$ldir
671 jmc 1.83 done
672 edhill 1.12 )
673 edhill 1.1 fi
674     }
675    
676     runmodel()
677     {
678 edhill 1.6 # runmodel directory
679 edhill 1.1 #
680 jmc 1.154 # runs $COMMAND in "directory"
681 edhill 1.6 # (where "$COMMAND" is relative to "directory")
682 edhill 1.1 (
683     cd $1
684 jmc 1.139 printf 'runmodel in %s ... ' $1
685 jmc 1.134 if test -L $EXECUTABLE ; then
686     if test -x "../"$builddir"/"$EXECUTABLE ; then
687 jmc 1.154 cmp $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
688     outD=$? ; if test "x$outD" != x0 ; then rm -f $EXECUTABLE ; fi
689 jmc 1.134 else rm -f $EXECUTABLE
690     fi
691 jmc 1.106 fi
692 jmc 1.83 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
693 jmc 1.119 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
694 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
695     fi
696 jmc 1.154 if test "x$MPI" != x0 ; then
697     #- adjust the MPI run command with the right number of Procs
698 jmc 1.156 #echo '' ; echo " COMMAND='$COMMAND'"
699     COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"`
700     if test "x$MPI_MFILE" != x ; then
701     COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"`
702     fi
703     #echo " COMMAND='$COMMAND'"
704 jmc 1.154 fi
705 jmc 1.83 if test ! -x $EXECUTABLE ; then
706 jmc 1.156 rm -f $RUNLOG ; touch $RUNLOG
707     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
708     echo " no executable:" $EXECUTABLE >> $RUNLOG
709     RETVAL=8
710     ENDVAL=-1
711 jmc 1.75 else
712 jmc 1.156 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
713     # output do not exist or is older than executable:
714     rm -f $RUNLOG ; touch $RUNLOG
715     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
716     ( eval $COMMAND ) >> $RUNLOG 2>&1
717     RETVAL=$?
718     ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'`
719     if [ $POSTCLEAN -eq 1 -a $ENDVAL -gt 0 ] ; then
720     find . -name "*.meta" -exec rm {} \;
721     find . -name "*.data" -exec rm {} \;
722     rm -rf mnc_test_*
723 jmc 1.83 fi
724 jmc 1.156 else
725     RETVAL=0
726 jmc 1.153 ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'`
727 jmc 1.156 touch $RUNLOG
728     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
729     echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
730     fi
731 jmc 1.75 fi
732 jmc 1.119 rm -f run.log_tmp
733 jmc 1.88 if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
734 jmc 1.139 echo successful
735     printf '=> output from running in %s :\n' $1 1>&2
736     tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
737 edhill 1.44 # === Reduce the size of the testing emails!
738 jmc 1.87 #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE
739 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
740 edhill 1.6 return 0
741 jmc 1.132 elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then
742     #-- for some weird cases (run is finihed but with error code)
743 jmc 1.139 echo 'finished with error (run:' $RETVAL ' end:' $ENDVAL ')'
744     printf '=> output from running in %s :\n' $1 1>&2
745     tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
746 jmc 1.132 return 0
747 edhill 1.6 else
748 jmc 1.139 echo 'failed (run:' $RETVAL ' end:' $ENDVAL ')'
749     printf '=> output from running in %s :\n' $1 1>&2
750     tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
751 jmc 1.119 cp $RUNLOG $CDIR"/"$RUNLOG
752 jmc 1.48 if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
753 edhill 1.6 return 1
754 edhill 1.1 fi
755     )
756     }
757    
758     createcodelet()
759     {
760     # create codelet for comparing model output
761    
762 jmc 1.112 printf "creating the comparison code (using CC=$CC)... "
763 jmc 1.91 cat > tr_cmpnum.c <<EOF
764 edhill 1.22 #include <stdio.h>
765     #include <math.h>
766     int main( int argc, char** argv ) {
767 jmc 1.91 int linnum,cmplin,best,lncnt;
768 edhill 1.52 double a,b,abave,relerr;
769 jmc 1.57 best = -22;
770 adcroft 1.33 lncnt = 0;
771 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
772 edhill 1.22 scanf("%d", &linnum);
773     if (linnum == -1) break;
774     scanf("%lf", &a); scanf("%lf", &b);
775 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
776 jmc 1.91 if ( abave == abave ) {
777     if (abave > 0.0) {
778     relerr=fabs(a-b)/abave;
779     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
780     else { cmplin = -16 ; }
781     best = (best > cmplin) ? best : cmplin; }
782     else { cmplin = -22 ; }
783     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
784     }
785     else {
786     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
787     break; }
788 edhill 1.22 }
789 adcroft 1.33 if (lncnt == 999) best=-29;
790 jmc 1.91 if (linnum != -1) best=-99;
791 edhill 1.22 printf("%d\n", -best);
792     return 0;
793     }
794     EOF
795 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
796 edhill 1.1
797 jmc 1.91 if [ -x ./tr_cmpnum ]; then
798 edhill 1.1 echo "OK"
799     return 0
800     else
801     echo
802 jmc 1.135 echo "ERROR: failed to compile comparison code -- please specify"
803 edhill 1.58 echo " a C compiler using the CC environment variable."
804 edhill 1.1 exit 1
805     fi
806     }
807    
808     formatresults()
809     {
810     # formatresults expt genmake depend make run results*
811    
812     nm=$1
813     printf '%s %s %s %s' $2 $3 $4 $5
814     shift; shift; shift; shift; shift;
815 jmc 1.94 listPrt=$@
816     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
817     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
818 jmc 1.97 printf '%3s' $listPrt
819 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
820 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
821 jmc 1.135
822 jmc 1.94 if [ $xx = '..' ]; then
823     printf ' N/O '
824     elif [ $xx = '--' ]; then
825     printf ' N/O '
826     elif [ $xx = 99 ]; then
827 edhill 1.1 printf ' N/O '
828     else
829 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
830 edhill 1.1 printf ' pass'
831     else
832     printf ' FAIL'
833     fi
834     fi
835     printf ' %s' $nm
836     printf '\n'
837 jmc 1.135
838 edhill 1.1 }
839    
840     scandirs()
841     {
842 jmc 1.81 if [ $# -eq 1 ]; then
843     for arg in * ; do
844 jmc 1.99 test -f $arg/$1 && echo $arg
845 jmc 1.81 done
846 edhill 1.1 else
847 jmc 1.81 echo $*
848 edhill 1.1 fi
849     }
850    
851    
852 jmc 1.123 check_eedata()
853     {
854     # check_eedata size.h eedata
855     if [ $# -eq 2 ] ; then
856     if test -f $1 -a -f $2 ; then
857     sx=`grep "^ & *nSx *=" $1 | sed "s/^ & *nSx *=//" | sed 's/, *$//'`
858     nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
859     if test "x$nx" = x ; then
860     rx=10
861     else
862     rx=`expr $sx % $nx`
863     fi
864     sy=`grep "^ & *nSy *=" $1 | sed "s/^ & *nSy *=//" | sed 's/, *$//'`
865     ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
866     if test "x$ny" = x ; then
867     ry=20
868     else
869     ry=`expr $sy % $ny`
870     fi
871     echo `expr $rx + $ry`
872     else
873     echo '-1'
874     fi
875     else
876     echo '-2'
877     fi
878    
879     }
880    
881 edhill 1.1 ###############################################################################
882    
883    
884     # Default properties
885     debug=0
886     verbose=1
887 edhill 1.10
888 edhill 1.40 IEEE=true
889 edhill 1.10 if test "x$MITGCM_IEEE" != x ; then
890     IEEE=$MITGCM_IEEE
891     fi
892 jmc 1.109 GSL=f
893 edhill 1.10
894     CLEANUP=f
895 jmc 1.133 NORUN=f
896 edhill 1.10 QUICK=f
897 jmc 1.136 NOMAKE=f
898 edhill 1.10 NOGENMAKE=f
899     NOCLEAN=f
900     NODEPEND=f
901 jmc 1.156 POSTCLEAN=0
902 edhill 1.1
903 edhill 1.4 BASH=
904 edhill 1.10 OPTFILE=NONE
905     ADDRESSES=
906 edhill 1.1 TESTDIRS=
907 jmc 1.89 SKIPDIRS=
908 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
909 edhill 1.20 HAVE_MPACK=
910 jmc 1.113 MPACK=
911 edhill 1.24 COMMAND=
912 jmc 1.141 MKDEPEND=
913 edhill 1.59 if test "x$MAKE" = x ; then
914     MAKE=make
915     fi
916     if test "x$CC" = x ; then
917     CC=cc
918     fi
919 edhill 1.43 JOBS=
920 jmc 1.154 MPI=0
921 jmc 1.156 MPI_MFILE=
922 jmc 1.83 MULTI_THREAD=f
923 jmc 1.70 OUTDIR=
924 edhill 1.40 DELDIR=
925 jmc 1.135 USE_R4=
926 edhill 1.1
927 edhill 1.24 ADM=
928 utke 1.120 OADM=
929 edhill 1.24
930 jmc 1.117 # list of pTracers to check for monitor output
931 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
932 edhill 1.49
933 jmc 1.147 MATCH_CRIT=10
934 jmc 1.92
935 edhill 1.34 printf "parsing options... "
936 edhill 1.1
937     ac_prev=
938     for ac_option ; do
939    
940     # If the previous option needs an argument, assign it.
941     if test -n "$ac_prev"; then
942     eval "$ac_prev=\$ac_option"
943     ac_prev=
944     continue
945     fi
946    
947     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
948 jmc 1.135
949 edhill 1.1 case $ac_option in
950    
951 jmc 1.156 -help | --help | -h | --h) usage ;;
952    
953     -optfile | --optfile | -of | --of) ac_prev=OPTFILE ;;
954     -optfile=* | --optfile=* | -of=* | --of=*) OPTFILE=$ac_optarg ;;
955    
956     -addr | --addr | -a | --a) ac_prev=ADDRESSES ;;
957     -addr=* | --addr=* | -a=* | --a=*) ADDRESSES=$ac_optarg ;;
958     -mpackdir | --mpackdir | -mpd | --mpd) ac_prev=MPACKDIR ;;
959     -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*) MPACKDIR=$ac_optarg ;;
960    
961     -tdir | --tdir | -t | --t) ac_prev=TESTDIRS ;;
962     -tdir=* | --tdir=* | -t=* | --t=*) TESTDIRS=$ac_optarg ;;
963     -skipdir | --skipdir | -skd | --skd) ac_prev=SKIPDIRS ;;
964     -skipdir=* | --skipdir=* | -skd=* | --skd=*) SKIPDIRS=$ac_optarg ;;
965    
966     -bash | --bash | -b | --b) ac_prev=BASH ;;
967     -bash=* | --bash=* | -b=* | --b=*) BASH=$ac_optarg ;;
968    
969     -command | --command | -c | --c) ac_prev=COMMAND ;;
970     -command=* | --command=* | -c=* | --c=*) COMMAND=$ac_optarg ;;
971    
972     -makedepend | --makedepend | -md | --md) ac_prev=MKDEPEND ;;
973     -makedepend=* | --makedepend=* | -md=* | --md=*) MKDEPEND=$ac_optarg ;;
974    
975     -make | --make | -m | --m) ac_prev=MAKE ;;
976     -make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;;
977    
978     -odir | --odir) ac_prev=OUTDIR ;;
979     -odir=* | --odir=*) OUTDIR=$ac_optarg ;;
980    
981     -ptracers | --ptracers | -ptr | --ptr) ac_prev=PTRACERS_NUM ;;
982     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*) PTRACERS_NUM=$ac_optarg ;;
983 edhill 1.50
984 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
985     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
986    
987 jmc 1.136 -j | --j) ac_prev=JOBS ;;
988     -j=* | --j=*) JOBS=$ac_optarg ;;
989 edhill 1.43
990 jmc 1.156 -clean | --clean) CLEANUP=t ; DELDIR=t ;;
991 edhill 1.10
992 jmc 1.156 -norun | --norun | -nr | --nr) NORUN=t ;;
993     -runonly | --runonly | -ro | --ro) QUICK=t ; NOMAKE=t ;;
994     -quick | --quick | -q | --q) QUICK=t ;;
995     -nogenmake | --nogenmake | -ng | --ng) NOGENMAKE=t ;;
996     -noclean | --noclean | -nc | --nc) NOCLEAN=t ;;
997     -nodepend | --nodepend | -nd | --nd) NODEPEND=t ;;
998 edhill 1.10
999 jmc 1.156 -postclean | --postclean | -pc | --pc) POSTCLEAN=2 ;;
1000     -deloutp | --deloutp | -do | --do) POSTCLEAN=1 ;;
1001 edhill 1.68
1002 jmc 1.154 -mpi | --mpi) MPI=2 ;;
1003     -MPI | --MPI) ac_prev=MPI ;;
1004     -MPI=* | --MPI=*) MPI=$ac_optarg ;;
1005 edhill 1.10
1006 jmc 1.156 -mfile | --mfile | -mf | --mf) ac_prev=MPI_MFILE ;;
1007     -mfile=* | --mfile=* | -mf=* | --mf=*) MPI_MFILE=$ac_optarg ;;
1008    
1009 jmc 1.83 -mth) MULTI_THREAD=t ;;
1010    
1011 edhill 1.24 -adm | -ad) ADM=t ;;
1012 utke 1.120 -oad) OADM=t; NODEPEND=t ;;
1013    
1014 edhill 1.10 -ieee) IEEE=true ;;
1015     -noieee) IEEE= ;;
1016 jmc 1.156 -gsl) GSL=t ;;
1017 edhill 1.10
1018 edhill 1.1 -verbose) verbose=2 ;;
1019     -debug) debug=1 ;;
1020     -quiet) verbose=0 ;;
1021    
1022 edhill 1.40 -deldir | -dd) DELDIR=t ;;
1023    
1024 jmc 1.136 -use_r4|-ur4) USE_R4=t ;;
1025 jmc 1.135
1026 ce107 1.90 -ts) TS=t;;
1027     -papis) PAPIS=t;;
1028     -pcls) PCL=t;;
1029    
1030 jmc 1.156 -*) echo "Error: unrecognized option: "$ac_option
1031     usage ;;
1032     *) echo "Error: unrecognized argument: "$ac_option
1033     usage ;;
1034 jmc 1.135
1035 edhill 1.1 esac
1036 jmc 1.135
1037 edhill 1.1 done
1038    
1039 edhill 1.10 if test "x$QUICK" = xt ; then
1040     NOGENMAKE=t
1041     NOCLEAN=t
1042     NODEPEND=t
1043     fi
1044    
1045 jmc 1.156 #- check length of MPI machine file:
1046     if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then
1047     if test -r $MPI_MFILE ; then
1048     nl=`wc -l $MPI_MFILE | awk '{print $1}'`
1049     if [ $nl -lt $MPI ] ; then
1050     echo "Error: need at least $MPI nodes (currently only $nl) in MPI_MFILE=$MPI_FILE"
1051     usage
1052     fi
1053     if [ $verbose -gt 1 ]; then
1054     echo " MPI_MFILE=$MPI_MFILE : $nl procs for MPI=$MPI run"
1055     fi
1056     else
1057     echo "Error: cannot access MPI_MFILE=$MPI_FILE"
1058     usage
1059     fi
1060     fi
1061    
1062 jmc 1.99 #- setting for forward or ADM testing
1063     if test "x$ADM" = xt ; then
1064     code_dir=code_ad
1065 jmc 1.108 inputdir=input_ad
1066 jmc 1.99 ref_outp="output_adm.txt"
1067     EXECUTABLE="mitgcmuv_ad"
1068 utke 1.120 elif test "x$OADM" = xt ; then
1069     code_dir=code_oad
1070     inputdir=input_oad
1071     ref_outp="output_oadm.txt"
1072     EXECUTABLE="mitgcmuv_ad"
1073 jmc 1.99 else
1074     code_dir=code
1075 jmc 1.108 inputdir=input
1076 jmc 1.99 ref_outp="output.txt"
1077     EXECUTABLE="mitgcmuv"
1078     fi
1079    
1080 jmc 1.142 xx=`echo $TESTDIRS | awk '{print $1}'`
1081 edhill 1.1 if test "x$TESTDIRS" = x ; then
1082 jmc 1.103 LIST=`scandirs results/$ref_outp`
1083 jmc 1.142 elif test $xx = 'start_from' ; then
1084     xx=`echo $TESTDIRS | awk '{print $2}'`
1085     LIST=`scandirs results/$ref_outp | sed -n "/$xx/,$ p"`
1086 jmc 1.84 else
1087     #- expand group of experiments:
1088     LIST=" "
1089     for xx in $TESTDIRS
1090     do
1091     case $xx in
1092 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
1093     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
1094     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
1095 jmc 1.84 ;;
1096     'tutorials')
1097     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
1098     *) LIST=${LIST}" "$xx ;;
1099     esac
1100 jmc 1.135 done
1101 jmc 1.89 fi
1102     #echo 'LIST='${LIST}'<'
1103     #- skip dirs, remove duplicate and non-directory:
1104     TESTDIRS=" "
1105     count=0
1106     for xx in $LIST
1107     do
1108     yy=`echo $SKIPDIRS | grep -c $xx`
1109     if test $yy = 0 ; then
1110 jmc 1.84 if test -d $xx ; then
1111     yy=`echo $TESTDIRS | grep -c $xx`
1112     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
1113 jmc 1.89 else count=1 ;
1114     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
1115     fi
1116     else
1117     if test $count = 1 ; then echo -n ", \"$xx\""
1118     else count=1 ; echo "" ; echo -n " skip: \"$xx\""
1119 jmc 1.84 fi
1120 jmc 1.89 fi
1121 jmc 1.135 done
1122 jmc 1.89 if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
1123 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
1124 edhill 1.1
1125 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
1126     OPTFILE=$MITGCM_OF
1127     fi
1128    
1129 jmc 1.156 LOC_MFILE='tr_mpi_mfile'
1130 jmc 1.146 RUNLOG="run.tr_log"
1131 jmc 1.99 OUTPUTFILE=$ref_outp
1132 jmc 1.87 if test "x$COMMAND" = x ; then
1133 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
1134 edhill 1.24 fi
1135 jmc 1.154 if test "x$MPI" != x0 ; then
1136 jmc 1.87 OUTPUTFILE="STDOUT.0000"
1137 edhill 1.24 fi
1138    
1139 jmc 1.76 echo "OK (COMMAND= $COMMAND )"
1140 edhill 1.1
1141 jmc 1.94 # set the Default List of output variables to be checked:
1142     # (use default or load experiment-specific list from file "tr_checklist")
1143     # content : 1rst = main variable used to decide if it pass or FAIL
1144     # others = number of matching digits to be printed in summary.txt
1145 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1146 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
1147     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
1148 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1149 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
1150 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
1151 jmc 1.99 else
1152 jmc 1.103 DEF_CHECK_LIST='Grad Cost Grad'
1153 jmc 1.99 EMPTY_RESULTS='.. ..'
1154     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1155 jmc 1.94 fi
1156    
1157 edhill 1.1 # create the FORTRAN comparison code
1158 jmc 1.133 if test -x tr_cmpnum ; then
1159     echo "skipping comparison code build"
1160     else
1161     createcodelet
1162     fi
1163 edhill 1.1
1164 jmc 1.112 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
1165     if test "x$ADDRESSES" = x ; then
1166 edhill 1.32 echo "skipping mpack build"
1167     else
1168 edhill 1.31 build_mpack
1169     fi
1170 edhill 1.1
1171     # Create a uniquely named directory to store results
1172 jmc 1.110 CMDLINE=$0
1173 jmc 1.118 for xx in "$@" ; do nw=`echo $xx | wc -w`
1174     if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
1175     else CMDLINE="$CMDLINE '$xx'" ; fi
1176     done
1177     #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
1178 edhill 1.1 MACH=`hostname`
1179 edhill 1.2 UNAMEA=`uname -a`
1180 edhill 1.1 DATE=`date +%Y%m%d`
1181 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1182 jmc 1.70 if test "x$OUTDIR" != x ; then
1183     BASE="tr_"$OUTDIR"_"$DATE"_"
1184 jmc 1.121 else
1185     short_name=`hostname | sed 's/\..*$//'`
1186     BASE="tr_"$short_name"_"$DATE"_"
1187 jmc 1.70 fi
1188 edhill 1.1 DNUM=0
1189     DRESULTS="$BASE$DNUM"
1190     while test -e $DRESULTS ; do
1191     DNUM=$(( $DNUM + 1 ))
1192     DRESULTS="$BASE$DNUM"
1193     done
1194     mkdir $DRESULTS
1195     RETVAL=$?
1196     if test "x$RETVAL" != x0 ; then
1197 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1198 edhill 1.1 exit 1
1199     fi
1200     SUMMARY="$DRESULTS/summary.txt"
1201 edhill 1.16 start_date=`date`
1202 edhill 1.17 echo $start_date > $SUMMARY
1203 jmc 1.110 echo 'run:' $CMDLINE >> $SUMMARY
1204     echo 'on :' $UNAMEA >> $SUMMARY
1205 edhill 1.1
1206 edhill 1.11 of_path=
1207 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1208     if test -r $OPTFILE ; then
1209 edhill 1.11 # get the path
1210     path=${OPTFILE%/*}
1211     if test "x$path" = x ; then
1212     of_path=`pwd`
1213     else
1214     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1215     fi
1216     file=${OPTFILE##*/}
1217     OPTFILE=$of_path/$file
1218 edhill 1.21 cp $OPTFILE $DRESULTS
1219     echo >> $SUMMARY
1220     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1221 edhill 1.11 else
1222 edhill 1.21 echo | tee $SUMMARY
1223     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY
1224     exit 1
1225 edhill 1.10 fi
1226 edhill 1.21 else
1227     echo >> $SUMMARY
1228 jmc 1.152 echo "No \"OPTFILE\" was specified ; genmake2 found and uses:" >> $SUMMARY
1229     #-note: to be filled later after 1rst run
1230 edhill 1.10 fi
1231     echo
1232     echo >> $SUMMARY
1233 utke 1.120 if test "x$ADM" = x -a "x$OADM" = x; then
1234 jmc 1.135 if [ $MATCH_CRIT -lt 10 ] ;
1235     then line_0="default "$MATCH_CRIT ;
1236 jmc 1.94 else line_0="default "$MATCH_CRIT ; fi
1237     line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1238 edhill 1.49 line_1="G D M c m s m s m s m s"
1239     line_2="E p a R g m m e . m m e . m m e . m m e ."
1240     line_3="N n k u 2 i a a d i a a d i a a d i a a d"
1241     line_4="2 d e n d n x n . n x n . n x n . n x n ."
1242 edhill 1.50 for ii in $PTRACERS_NUM ; do
1243 edhill 1.49 line_0="$line_0 --PTR 0"$ii"--"
1244     line_1="$line_1 m s"
1245     line_2="$line_2 m m e ."
1246     line_3="$line_3 i a a d"
1247     line_4="$line_4 n x n ."
1248     done
1249     echo "$line_0" | tee -a $SUMMARY
1250     echo "$line_1" | tee -a $SUMMARY
1251     echo "$line_2" | tee -a $SUMMARY
1252     echo "$line_3" | tee -a $SUMMARY
1253     echo "$line_4" | tee -a $SUMMARY
1254     echo " " | tee -a $SUMMARY
1255 edhill 1.24 else
1256     echo "ADJOINT=true" >> $SUMMARY
1257     echo >> $SUMMARY
1258 jmc 1.135 if [ $MATCH_CRIT -lt 10 ] ;
1259     then line_0="default "$MATCH_CRIT ;
1260 jmc 1.103 else line_0="default "$MATCH_CRIT ; fi
1261 jmc 1.100 echo "$line_0" | tee -a $SUMMARY
1262 edhill 1.24 cat << EOF | tee -a $SUMMARY
1263     G D M C G
1264     E p a R o r
1265     N n k u s a
1266     2 d e n t d
1267    
1268     EOF
1269     fi
1270 jmc 1.139 echo "-------------------------------------------------------------------------------"
1271 edhill 1.1
1272 edhill 1.10 # ...and each test directory...
1273     for dir in $TESTDIRS ; do
1274 jmc 1.135
1275 jmc 1.106 # set builddir & rundir:
1276     builddir="build"
1277     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1278     rundir="run"
1279 jmc 1.143 pfxdir="tr_$rundir"
1280 jmc 1.106 if test ! -d $dir/$rundir ; then
1281     rundir=$builddir
1282     fi
1283     CODE_DIR=$dir/$code_dir
1284     BUILD_DIR=$dir/$builddir
1285    
1286 edhill 1.10 # Cleanup only!
1287     if test "x$CLEANUP" = xt ; then
1288 jmc 1.127 echo -n ' --- dir:' $BUILD_DIR ': '
1289     makeclean $BUILD_DIR
1290 jmc 1.145 ( cd $BUILD_DIR
1291     rm -f $EXECUTABLE *.bak
1292 jmc 1.146 rm -f genmake_state genmake_*optfile genmake.log
1293 jmc 1.154 rm -f SIZE.h.mpi genmake.tr_log make.tr_log
1294 jmc 1.145 )
1295 jmc 1.106 if test -d $dir/$rundir/CVS ; then
1296 jmc 1.127 echo -n ' --- dir:' $dir/$rundir ': '
1297 jmc 1.106 run_clean $dir/$rundir
1298 edhill 1.10 fi
1299 jmc 1.143 trdir=`( cd $dir ; find . -type d -name "$pfxdir.*" -print | sed 's/^.\///')`
1300 jmc 1.127 ttd=`echo $trdir | wc -w`
1301     if test $ttd != 0 ; then
1302     echo ' --- rm dir:' $trdir
1303     ( cd $dir ; rm -rf $trdir )
1304     fi
1305 edhill 1.10 continue
1306 edhill 1.1 fi
1307 edhill 1.3
1308 jmc 1.135 # Verify that the testdir exists and contains previous
1309 edhill 1.10 # results in the correct location--or skip this directory!
1310 jmc 1.104 fout=$dir"/results/"$ref_outp
1311 edhill 1.24 if test ! -r $fout ; then
1312     echo "can't read \"$fout\" -- skipping $dir"
1313 edhill 1.10 continue
1314     fi
1315 edhill 1.7
1316 jmc 1.117 # Check for specific files for particular type of run
1317 edhill 1.49
1318 jmc 1.154 if test "x$MPI" != "x0" ; then
1319     if test -r $CODE_DIR"/SIZE.h_mpi" ; then
1320     #- create new SIZE.h with no more than '$MPI' Procs
1321     mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI
1322 jmc 1.156 LOC_NPROC=$?
1323 jmc 1.154 ( cd $BUILD_DIR
1324     if test -r SIZE.h.mpi ; then
1325     cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$?
1326     else RETVAL=1
1327     fi
1328     if test "x$RETVAL" = x0 ; then
1329     rm -f tr_size.mpi
1330     else
1331     rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi
1332     fi
1333     )
1334     else
1335     echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1336     continue
1337     fi
1338 jmc 1.156 if test "x$MPI_MFILE" != x ; then
1339     #- create new MPI machine-file with the right number of Procs
1340     rm -f $LOC_MFILE
1341     cat $MPI_MFILE | sort | uniq | head -$LOC_NPROC > $LOC_MFILE
1342     nl=`wc -l $LOC_MFILE | awk '{print $1}'`
1343     if [ $nl -lt $LOC_NPROC ] ; then
1344     rm -f $LOC_MFILE
1345     cat $MPI_MFILE | head -$LOC_NPROC > $LOC_MFILE
1346     #sed -n "1,$LOC_NPROC p" $MPI_MFILE > $LOC_MFILE
1347     fi
1348     if [ $verbose -gt 1 ]; then
1349     nl=`wc -l $LOC_MFILE | awk '{print $1}'`
1350     echo " new LOC_MFILE=$LOC_MFILE : $nl procs for LOC_NPROC=$LOC_NPROC"
1351     fi
1352     fi
1353 edhill 1.28 fi
1354 jmc 1.83 if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1355     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1356     continue
1357     fi
1358 jmc 1.154 if test "x$MPI" != "x0" -a "x$MULTI_THREAD" = "xt" ; then
1359     retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/input/eedata.mth"`
1360     if test $retv != 0 ; then
1361     echo "input/eedata.mth tiling misfit -- skipping $dir"
1362     continue
1363     fi
1364 jmc 1.123 fi
1365 jmc 1.83
1366     # Check whether there are "extra runs" for this testdir
1367     extra_runs=
1368 jmc 1.133 if test "x$NORUN" = xf ; then
1369     ex_run_dirs=`( cd $dir ; echo $inputdir.* )`
1370     fi
1371 jmc 1.108 #echo "ex_run_dirs='$ex_run_dirs'"
1372     for exd in $ex_run_dirs ; do
1373     name=`echo $exd | sed -e "s/$inputdir\.//"`
1374     refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1375     outf="$dir/results/$refExOut"
1376     if test -f $outf -a -r $outf ; then
1377 jmc 1.135 if test "x$MULTI_THREAD" = "xt" ; then
1378 jmc 1.123 if test -r $dir"/"$exd"/eedata.mth" ; then
1379 jmc 1.154 if test "x$MPI" = "x0" ; then
1380     extra_runs="$extra_runs $name"
1381     else
1382     retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/"$exd"/eedata.mth"`
1383 jmc 1.123 if test $retv = 0 ; then
1384 jmc 1.83 extra_runs="$extra_runs $name"
1385 jmc 1.123 else
1386     echo $exd"/eedata.mth tiling misfit -- skipping $dir"
1387 jmc 1.83 fi
1388 jmc 1.123 fi
1389     #else echo $dir"/"$exd"/eedata.mth: not found"
1390 jmc 1.83 fi
1391 jmc 1.123 else
1392     extra_runs="$extra_runs $name"
1393     fi
1394 jmc 1.108 fi
1395     done
1396 edhill 1.28
1397     echo
1398 jmc 1.135 if test "x$extra_runs" = "x" ; then
1399 jmc 1.83 echo "Experiment: $dir"
1400     else
1401     echo "Experiment: $dir ; extra_runs=$extra_runs"
1402     fi
1403 edhill 1.28 echo
1404     unset genmake makedepend make run
1405 jmc 1.94 results=$EMPTY_RESULTS
1406 edhill 1.10
1407     # Create an output dir for each OPTFILE/tdir combination
1408 jmc 1.138 locDIR=$DRESULTS"/"$dir
1409     mkdir $locDIR
1410     CDIR=`pwd`"/$locDIR"
1411 jmc 1.135
1412 jmc 1.133 if test "x$NORUN" = xt ; then
1413     run=N
1414     genmakemodel $dir/$builddir && genmake=Y \
1415     && makeclean $dir/$builddir \
1416     && symlink_mpifiles $dir $code_dir $builddir \
1417     && makedependmodel $dir/$builddir && makedepend=Y \
1418     && makemodel $dir/$builddir && make=Y
1419 edhill 1.10 else
1420 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1421     && makeclean $dir/$builddir \
1422 edhill 1.27 && symlink_mpifiles $dir $code_dir $builddir \
1423 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1424     && makemodel $dir/$builddir && make=Y \
1425 jmc 1.139 && run_clean $dir/$rundir \
1426 jmc 1.140 && linkdata $dir/$rundir $inputdir \
1427 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1428 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1429 edhill 1.10 fi
1430 jmc 1.139 #echo "results='$results'"
1431 jmc 1.135
1432 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1433 jmc 1.139 echo 1>&2
1434 jmc 1.98 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1435 jmc 1.138 touch $locDIR"/summary.txt"
1436     echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
1437     echo "tdir='$dir'" >> $locDIR"/summary.txt"
1438 jmc 1.108 if test "x$ADM" = xt ; then
1439 jmc 1.138 head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt"
1440 jmc 1.157 nerr=`grep -c 'TAF *.* ERROR ' $dir/$builddir/taf_ad.log`
1441     nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' $dir/$builddir/taf_ad.log`
1442     echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \
1443     >> $locDIR"/summary.txt"
1444 jmc 1.108 grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \
1445 jmc 1.138 >> $locDIR"/summary.txt"
1446 jmc 1.108 fi
1447 edhill 1.34
1448     for ex in $extra_runs ; do
1449 jmc 1.73 unset run
1450 jmc 1.94 results=$EMPTY_RESULTS
1451 jmc 1.106 # reference output file
1452     refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1453 jmc 1.73 # Create an output dir for each OPTFILE/tdir.ex combination
1454 jmc 1.138 locDIR=$DRESULTS"/"$dir"."$ex
1455     mkdir $locDIR
1456     CDIR=`pwd`"/$locDIR"
1457 jmc 1.143 test ! -e "$dir/$pfxdir.$ex" && mkdir "$dir/$pfxdir.$ex"
1458     run_clean $dir/$pfxdir.$ex
1459     linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir
1460     runmodel $dir/$pfxdir.$ex && run=Y \
1461     && results=`testoutput_run $dir $pfxdir.$ex $refExOut`
1462 edhill 1.34 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1463     fres="$fres.$ex"
1464 jmc 1.139 echo 1>&2
1465 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1466 jmc 1.138 touch $locDIR"/summary.txt"
1467     echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
1468     echo "tdir='$dir.$ex'" >> $locDIR"/summary.txt"
1469 jmc 1.108 if test "x$ADM" = xt ; then
1470 jmc 1.143 grep -A3 'Seconds in section "ALL' $dir/$pfxdir.$ex/$OUTPUTFILE \
1471 jmc 1.138 >> $locDIR"/summary.txt"
1472 jmc 1.108 fi
1473 jmc 1.156 if test "x$POSTCLEAN" = x2 ; then
1474 jmc 1.143 run_clean $dir/$pfxdir.$ex
1475 jmc 1.83 fi
1476 edhill 1.34 done
1477 edhill 1.68
1478 jmc 1.139 if test -f $DRESULTS"/"genmake_state ; then : ; else
1479 jmc 1.137 if test -f $dir/$builddir/Makefile ; then
1480     mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
1481     echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state
1482     sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
1483 jmc 1.151 echo " $mkOpt" >> $DRESULTS/genmake_state
1484 jmc 1.152 if test "x$OPTFILE" = xNONE ; then
1485     eval $mkOpt
1486     sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\
1487     $SUMMARY > tr_0.tmp_log
1488     RETVAL=$?
1489     if test "x$RETVAL" = x0 ; then rm -f $SUMMARY
1490     cp tr_0.tmp_log $SUMMARY
1491     else rm -f tr_0.tmp_log
1492     fi
1493     fi
1494 jmc 1.150 gmkLog=$dir/$builddir/genmake.log
1495     grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1
1496     RETVAL=$?
1497     if test "x$RETVAL" = x0 ; then
1498     echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state
1499 jmc 1.151 sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \
1500 jmc 1.152 | grep -v '^... compiler version ' > tr_1.tmp_log
1501     sed -n '1,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \
1502 jmc 1.151 >> $DRESULTS/genmake_state
1503 jmc 1.152 rm -f tr_1.tmp_log
1504 jmc 1.150 fi
1505 jmc 1.137 fi
1506     fi
1507 jmc 1.83 #postclean $dir/$builddir
1508 jmc 1.156 if test "x$POSTCLEAN" = x2 ; then
1509 jmc 1.83 makeclean $dir/$builddir \
1510     && run_clean $dir/$rundir
1511     fi
1512 jmc 1.156 if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then rm -f $LOC_MFILE ; fi
1513 jmc 1.135
1514 edhill 1.10 echo "-------------------------------------------------------------------------------"
1515 jmc 1.135
1516 edhill 1.1 done
1517    
1518 edhill 1.34 printf "Start time: " >> $SUMMARY
1519 jmc 1.114 echo "$start_date" >> $SUMMARY
1520 edhill 1.34 printf "End time: " >> $SUMMARY
1521 edhill 1.13 date >> $SUMMARY
1522 edhill 1.20
1523     # If addresses were supplied and mpack built successfully, then try
1524     # to send email using mpack.
1525     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1526     echo "No results email was sent."
1527     else
1528     if test "x$HAVE_MPACK" = xt ; then
1529     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1530     && gzip $DRESULTS".tar" \
1531 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1532 edhill 1.20 RETVAL=$?
1533     if test "x$RETVAL" != x0 ; then
1534     echo
1535     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1536     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1537     echo " summary of results from the directory \"$DRESULTS\"."
1538     echo
1539     else
1540     echo
1541     echo "An email containing results was sent to the following addresses:"
1542     echo " \"$ADDRESSES\""
1543     echo
1544 jmc 1.129 test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1545     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1546 edhill 1.20 fi
1547     fi
1548     fi
1549 edhill 1.13
1550 jmc 1.133 if test "x$QUICK" = xf -a "x$NORUN" = xf ; then
1551     rm -f tr_cmpnum.c tr_cmpnum
1552     fi
1553 edhill 1.1
1554 edhill 1.12 if test "x$CLEANUP" != xt ; then
1555 jmc 1.102 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1556 edhill 1.25 if test -e tr_out.txt ; then
1557     mv tr_out.txt tr_out.txt.old
1558 edhill 1.14 fi
1559 jmc 1.103 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1560 edhill 1.12 fi
1561 edhill 1.1
1562 edhill 1.40 if test "x$DELDIR" = xt ; then
1563     rm -rf $DRESULTS
1564     fi
1565    

  ViewVC Help
Powered by ViewVC 1.1.22