/[MITgcm]/MITgcm/verification/testreport
ViewVC logotype

Annotation of /MITgcm/verification/testreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.187 - (hide annotations) (download)
Thu Jul 18 22:11:58 2013 UTC (10 years, 8 months ago) by jmc
Branch: MAIN
Changes since 1.186: +7 -1 lines
add a hack to run with Divided adjoint (DIVA)

1 edhill 1.19 #! /usr/bin/env bash
2 edhill 1.1 #
3 jmc 1.187 # $Header: /u/gcmpack/MITgcm/verification/testreport,v 1.186 2013/07/09 22:11:47 jmc Exp $
4 edhill 1.12 # $Name: $
5 edhill 1.1 #
6    
7     usage()
8     {
9     echo
10     echo "Usage: $0 [OPTIONS]"
11     echo
12     echo "where possible OPTIONS are:"
13     echo " (-help|-h) print usage"
14 jmc 1.122 echo " (-mth) run multi-threaded (using eedata.mth)"
15 jmc 1.154 echo " (-mpi) use MPI to compile and run on 2 processors"
16     echo " (-MPI) NUMBER use MPI to compile and run on max NUMBER procs"
17 jmc 1.156 echo " (-mfile|-mf) STRING MPI: file with list of possible machines to run on"
18 jmc 1.164 echo " (-fast) use optfile default for compiler flags (no '-ieee')"
19     echo " DEF=off => use IEEE numerics option (if available)"
20     echo " (-devel) use optfile developement flags (if available)"
21 jmc 1.109 echo " (-gsl) compile with \"-gsl\" flag"
22 jmc 1.136 echo " (-use_r4|-ur4) if allowed, use real*4 type for '_RS' variable"
23     echo " (-optfile|-of) STRING list of optfiles to use"
24     echo " (-addr|-a) STRING list of email recipients"
25 jmc 1.108 echo " (DEF=\"\" no email is sent)"
26 jmc 1.136 echo " (-mpackdir|-mpd) DIR location of the mpack utility"
27 jmc 1.113 echo " (DEF=\"../tools/mpack-1.6\")"
28 jmc 1.136 echo " (-tdir|-t) STRING list of group and/or exp. dirs to test"
29 jmc 1.84 echo " (recognized groups: basic, tutorials)"
30     echo " (DEF=\"\" which test all)"
31 jmc 1.149 echo " (if list= 'start_from THIS_EXP' then"
32     echo " test THIS_EXP + all the following)"
33 jmc 1.136 echo " (-skipdir|-skd) STRING list of exp. dirs to skip"
34 jmc 1.89 echo " (DEF=\"\" which test all)"
35 jmc 1.136 echo " (-bash|-b) STRING preferred location of a \"bash\" or"
36 edhill 1.20 echo " Bourne-compatible \"sh\" shell"
37     echo " (DEF=\"\" for \"bash\")"
38 jmc 1.175 echo " (-tlm) perform a Tangent Linear run"
39 edhill 1.24 echo " (-adm|-ad) perform an adjoint run"
40 utke 1.120 echo " (-oad) perform an OpenAD adjoint run"
41 jmc 1.136 echo " (-command|-c) STRING command to run"
42 jmc 1.141 echo " (-makedepend|-md) STRING command to use for \"makedepend\""
43 jmc 1.136 echo " (-make|-m) STRING command to use for \"make\""
44 edhill 1.8 echo " (DEF=\"make\")"
45 jmc 1.70 echo " (-odir) STRING used to build output directory name"
46     echo " (DEF=\"hostname\")"
47 jmc 1.156 # echo " (-ptracers|-ptr) STRING specify which ptracers to test"
48     # echo " (DEF=\"1 2 3 4 5\")"
49 jmc 1.92 echo " (-match) NUMBER Matching Criteria (number of digits)"
50 jmc 1.141 echo " (DEF=\"$MATCH_CRIT\")"
51 edhill 1.43 echo " (-j) JOBS use \"make -j JOBS\" for parallel builds"
52 jmc 1.165 echo " (-ef) STRING used as genmake2 \"-extra_flag\" argument"
53 jmc 1.156 echo " (-clean) *ONLY* run \"make CLEAN\" & clean run-dir"
54 jmc 1.133 echo " (-norun|-nr) skip the \"runmodel\" stage (stop after make)"
55 jmc 1.165 echo " (-obj) only produces objects (=norun & no executable)"
56 jmc 1.136 echo " (-runonly|-ro) *ONLY* run stage (=\"-quick\" without make)"
57 ce107 1.90 echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\""
58     echo " (-nogenmake|-ng) skip the genmake stage"
59     echo " (-noclean|-nc) skip the \"make clean\" stage"
60     echo " (-nodepend|-nd) skip the \"make depend\" stage"
61 jmc 1.156 echo " (-postclean|-pc) after each exp. test, clean build-dir & run-dir"
62     echo " (-deloutp|-do) delete output files after successful run"
63 ce107 1.90 echo " (-deldir|-dd) on success, delete the output directory"
64     echo " (-ts) provide timing information per timestep"
65     echo " (-papis) provide MFlop/s per timestep using PAPI"
66     echo " (-pcls) provide MFlop/s per timestep using PCL"
67 edhill 1.6 echo
68 edhill 1.50 echo "and where STRING can be a whitespace-delimited list"
69 edhill 1.6 echo "such as:"
70 edhill 1.50 echo
71 edhill 1.6 echo " -t 'exp0 exp2 exp3' "
72     echo " -addr='abc@123.com testing@home.org'"
73 edhill 1.1 echo
74 edhill 1.50 echo "provided that the expression is properly quoted within the current"
75     echo "shell (note the use of single quotes to protect white space)."
76     echo
77 edhill 1.1 exit 1
78     }
79    
80     # build the mpack utility
81     build_mpack()
82     {
83 edhill 1.34 printf "building the mpack utility... "
84 jmc 1.113 MPACK="$MPACKDIR/mpack"
85     if test ! -x $MPACK ; then
86 edhill 1.1 if test ! -d $MPACKDIR ; then
87 edhill 1.20 echo
88 edhill 1.1 echo "Error: can't find \"$MPACKDIR\""
89     echo " are you sure this program is being run in the correct "
90     echo " (that is, \"MITGCM_ROOT\verification\") directory?"
91 edhill 1.20 echo
92     HAVE_MPACK=f
93 edhill 1.1 fi
94 edhill 1.26 if test "x$CC" = x ; then
95     export CC=cc
96     fi
97 jmc 1.112 printf "building mpack (using CC=$CC)... "
98 edhill 1.25 ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1
99 edhill 1.1 RETVAL=$?
100     if test "x$RETVAL" != x0 ; then
101     echo
102     echo "Error building the mpack tools at: $MPACK_DIR"
103 edhill 1.20 echo
104     HAVE_MPACK=f
105     else
106 edhill 1.25 rm -f tr_build_mpack.out
107 edhill 1.20 HAVE_MPACK=t
108 jmc 1.113 echo "done"
109 edhill 1.1 fi
110 edhill 1.20 else
111     HAVE_MPACK=t
112 jmc 1.113 echo "already exist"
113 edhill 1.1 fi
114     }
115    
116 jmc 1.93 testoutput_var()
117 edhill 1.1 {
118 jmc 1.99 # testoutput_var dir s1 label subdir reference_output
119 edhill 1.1 #
120 jmc 1.135 # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE
121 jmc 1.175 # with same output from reference file $dir/$reference_output
122 jmc 1.93 # using search strings s1 and text label
123 edhill 1.1
124     if [ $debug -gt 0 ]; then
125 jmc 1.93 echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2
126 edhill 1.1 fi
127 jmc 1.87 if [ -r $1/$4/$OUTPUTFILE ]; then
128     grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt
129 jmc 1.67 lncntA=`wc -l tmp1.txt | awk '{print $1}' `
130 jmc 1.100 if [ $lncntA -lt 2 ]; then
131 edhill 1.1 if [ $verbose -gt 0 ]; then
132     echo Not enough lines of output when searching for "$2" 1>&2
133     fi
134 jmc 1.175 rm -f tmp1.txt ; return 99
135 edhill 1.1 fi
136     else
137 jmc 1.93 echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2
138 edhill 1.1 return 99
139     fi
140     if [ $debug -gt 0 ]; then
141 jmc 1.175 echo testoutput_var: grep "$2" $1/$5 1>&2
142 edhill 1.1 fi
143 jmc 1.175 grep "$2" $1/$5 | sed 's/.*=//' | cat -n > tmp2.txt
144 jmc 1.67 lncntB=`wc -l tmp2.txt | awk '{print $1}' `
145 jmc 1.100 if [ $lncntB -lt 2 ]; then
146 edhill 1.1 if [ $verbose -gt 0 ]; then
147     echo Not enough lines of output when searching for "$2" 1>&2
148     fi
149 jmc 1.175 rm -f tmp1.txt tmp2.txt ; return 99
150 edhill 1.1 fi
151 jmc 1.67 if [ $lncntA -ne $lncntB ]; then
152 jmc 1.175 if [ $verbose -gt 0 ]; then
153     echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2
154     fi
155     rm -f tmp1.txt tmp2.txt ; return 99
156 jmc 1.67 fi
157 edhill 1.72 has_nan=`cat tmp1.txt | grep -i nan | wc -l`
158     if [ $has_nan -gt 0 ] ; then
159 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2
160 jmc 1.175 rm -f tmp1.txt tmp2.txt ; return 99
161 edhill 1.72 fi
162     has_inf=`cat tmp1.txt | grep -i inf | wc -l`
163     if [ $has_inf -gt 0 ] ; then
164 jmc 1.93 echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2
165 jmc 1.175 rm -f tmp1.txt tmp2.txt ; return 99
166 jmc 1.71 fi
167 edhill 1.1 if [ $debug -gt 0 ]; then
168 jmc 1.93 echo testoutput_var: join tmp1.txt tmp2.txt 1>&2
169 edhill 1.1 fi
170     join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt
171     if [ $debug -gt 0 ]; then
172 jmc 1.93 echo testoutput_var: compare_lines 1>&2
173 edhill 1.1 fi
174 edhill 1.22 if [ $verbose -gt 1 ]; then
175     cat tmp3.txt 1>&2
176     fi
177     echo "-1" >> tmp3.txt
178 edhill 1.23 # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number
179     cat tmp3.txt | sed -e 's|:||g' > tmp4.txt
180 jmc 1.91 digits_of_similarity=`./tr_cmpnum < tmp4.txt`
181 edhill 1.1 if [ $digits_of_similarity -eq 99 ]; then
182     if [ $verbose -gt 0 ]; then
183 jmc 1.99 echo testoutput_var: No comparison was available for \"$3\" 1>&2
184 edhill 1.1 fi
185     digits_of_similarity=99
186     else
187     if [ $verbose -gt 0 ]; then
188 jmc 1.99 echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2
189 edhill 1.1 fi
190     fi
191 edhill 1.23 rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt
192 jmc 1.135
193 edhill 1.1 return $digits_of_similarity
194     }
195    
196 jmc 1.93 testoutput_run()
197 edhill 1.1 {
198 jmc 1.99 # testoutput_run directory subdir reference_output
199 edhill 1.1 #
200 jmc 1.93 # test output from 1 run in "directory"
201 jmc 1.103 # --> same processing for adjoint & forward test
202 jmc 1.175 # default list of output variables to be checked:
203     # 1rst : main variable used to decide if it pass or FAIL
204     # others : number of matching digits to be printed in summary.txt
205     listChk=$DEF_CHECK_LIST
206 jmc 1.94 # load experiment-specific list from file "tr_checklist" (if it exist)
207 jmc 1.175 if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi
208     sVar=`echo $listChk | awk '{print $1}'`
209     # remove 1rst var and expand the list: + => min max mean s.d
210     listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \
211     | sed 's/+//g' | sed "s/^$sVar//"`
212 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi
213 jmc 1.175 # set ref.outp file-name and, if compressed, uncompress it into subdir
214     refoutp=results/$3
215     if test ! -r $1/results/$3 ; then
216     if test -r $1/results/${3}.gz ; then
217     tmpNam1=$3
218     tmpNam2=`echo $tmpNam1 | sed 's/\..*\././'`
219     if test "x$tmpNam1" = "x$tmpNam2" ; then
220     tmpNam1=`echo $tmpNam2 | sed 's/\./.std./'`
221     fi
222     refoutp=$2/$tmpNam1
223     if [ $debug -gt 0 ]; then
224     echo "testoutput_run: gunzip 'results/$3.gz' into '$refoutp'" 1>&2
225     fi
226     gzip -cd $1/results/${3}.gz > $1/$refoutp
227     else
228     echo "missing reference output file '$3' (.gz) in '$1/results/'"
229     fi
230     fi
231     # check for ptracer output in reference_output file :
232     if test $KIND = 0 ; then
233     ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max"
234     ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd"
235     for ii in $PTRACERS_NUM ; do
236 jmc 1.117 ptrfound=0
237     for jj in $ptr_mon ; do
238     name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"`
239 jmc 1.175 tst=`grep $name $1/$refoutp | wc -l | awk '{print $1}'`
240 jmc 1.117 if test ! "x$tst" = x0 ; then ptrfound=1 ; fi
241     done
242     if test $ptrfound = '1' ; then
243     eval "HAVE_PTR0"$ii"=t"
244     else
245     eval "HAVE_PTR0"$ii"=f"
246     # remove this ptr from the list of output variable to check
247     # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2
248     listVar=`echo "$listVar" | sed "s/ pt$ii..//g"`
249     fi
250 jmc 1.175 # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2
251     done
252     fi
253     tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'`
254     if test $tst != 1 ; then
255     if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2
256     else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi
257     echo "==> WARNING: in checked list:" $listVar 1>&2
258     #- put it back once:
259     listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"`
260     fi
261 jmc 1.94 if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi
262 jmc 1.154 echo "listVar='$listVar'" >> $locDIR"/summary.txt"
263 jmc 1.175 #---
264     allargs=""
265     for xx in $listVar
266     do
267     case $xx in
268 jmc 1.94 'PS') if [ $debug -gt 0 ]
269     then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi
270 jmc 1.175 testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $refoutp ; yy=$?
271 jmc 1.94 if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;;
272 jmc 1.175 'Cost') testoutput_var $1 "ADM ref_cost_function" "ADM CostFct" $2 $refoutp ; yy=$? ;;
273     'AdGrd') testoutput_var $1 "ADM adjoint_gradient" "ADM Ad Grad" $2 $refoutp ; yy=$? ;;
274     'FDGrd') testoutput_var $1 "ADM finite-diff_grad" "ADM FD Grad" $2 $refoutp ; yy=$? ;;
275     'tlmCst') testoutput_var $1 "TLM ref_cost_function" "TLM CostFct" $2 $refoutp ; yy=$? ;;
276     'tlmGrd') testoutput_var $1 "TLM tangent-lin_grad" "TLM TL Grad" $2 $refoutp ; yy=$? ;;
277     'fwdGrd') testoutput_var $1 "TLM finite-diff_grad" "TLM FD Grad" $2 $refoutp ; yy=$? ;;
278     'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $refoutp ; yy=$? ;;
279     'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $refoutp ; yy=$? ;;
280     'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $refoutp ; yy=$? ;;
281     'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $refoutp ; yy=$? ;;
282     'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $refoutp ; yy=$? ;;
283     'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $refoutp ; yy=$? ;;
284     'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $refoutp ; yy=$? ;;
285     'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $refoutp ; yy=$? ;;
286     'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $refoutp ; yy=$? ;;
287     'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $refoutp ; yy=$? ;;
288     'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $refoutp ; yy=$? ;;
289     'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $refoutp ; yy=$? ;;
290     'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $refoutp ; yy=$? ;;
291     'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $refoutp ; yy=$? ;;
292     'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $refoutp ; yy=$? ;;
293     'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $refoutp ; yy=$? ;;
294 jmc 1.94 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
295 jmc 1.175 testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $refoutp ; yy=$? ;;
296 jmc 1.94 'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
297 jmc 1.175 testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $refoutp ; yy=$? ;;
298 jmc 1.94 'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
299 jmc 1.175 testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $refoutp ; yy=$? ;;
300 jmc 1.94 'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'`
301 jmc 1.175 testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $refoutp ; yy=$? ;;
302     'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum" $2 $refoutp ; yy=$? ;;
303     'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum" $2 $refoutp ; yy=$? ;;
304     'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean" $2 $refoutp ; yy=$? ;;
305     'Qntsd') testoutput_var $1 "forcing_qnet_sd" "Qnet Std.Dev" $2 $refoutp ; yy=$? ;;
306     'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $refoutp ; yy=$? ;;
307     'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $refoutp ; yy=$? ;;
308     'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $refoutp ; yy=$? ;;
309     'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $refoutp ; yy=$? ;;
310     'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $refoutp ; yy=$? ;;
311     'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Heff max" $2 $refoutp ; yy=$? ;;
312     'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Heff mean" $2 $refoutp ; yy=$? ;;
313     'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Heff StDv" $2 $refoutp ; yy=$? ;;
314     'uSImn') testoutput_var $1 "seaice_uice_min" "SIce Uice min" $2 $refoutp ; yy=$? ;;
315     'uSImx') testoutput_var $1 "seaice_uice_max" "SIce Uice max" $2 $refoutp ; yy=$? ;;
316     'uSIav') testoutput_var $1 "seaice_uice_mean" "SIce Uice mean" $2 $refoutp ; yy=$? ;;
317     'uSIsd') testoutput_var $1 "seaice_uice_sd" "SIce Uice StDv" $2 $refoutp ; yy=$? ;;
318     'vSImn') testoutput_var $1 "seaice_vice_min" "SIce Vice min" $2 $refoutp ; yy=$? ;;
319     'vSImx') testoutput_var $1 "seaice_vice_max" "SIce Vice max" $2 $refoutp ; yy=$? ;;
320     'vSIav') testoutput_var $1 "seaice_vice_mean" "SIce Vice mean" $2 $refoutp ; yy=$? ;;
321     'vSIsd') testoutput_var $1 "seaice_vice_sd" "SIce Vice StDv" $2 $refoutp ; yy=$? ;;
322     'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $refoutp ; yy=$? ;;
323     'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $refoutp ; yy=$? ;;
324     'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $refoutp ; yy=$? ;;
325 jmc 1.182 'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Glob-ave" $2 $refoutp ; yy=$? ;;
326     'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South-av" $2 $refoutp ; yy=$? ;;
327     'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North-av" $2 $refoutp ; yy=$? ;;
328     'HthMxS') testoutput_var $1 "thSI_IceH_max_S" "thSIc H South-max" $2 $refoutp ; yy=$? ;;
329     'HthMxN') testoutput_var $1 "thSI_IceH_max_N" "thSIc H North-max" $2 $refoutp ; yy=$? ;;
330 jmc 1.184 'StrmIc') testoutput_var $1 "STREAMICE_FP_ERR" "StreamIce Solver" $2 $refoutp ; yy=$? ;;
331 jmc 1.175 *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;;
332     esac
333     if test $xx = $sVar
334     then allargs="$allargs > $yy <"
335     else allargs="$allargs $yy"
336     fi
337     done
338 jmc 1.94
339     nbVar=`echo $listVar | awk '{print NF}'`
340 jmc 1.175 if [ $nbVar -lt $LEN_CHECK_LIST ] ; then
341     #-- fill line (up to standard length) with dot:
342     adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'`
343 jmc 1.94 echo $allargs $adNul
344 jmc 1.175 else
345 jmc 1.94 echo $allargs
346 jmc 1.135 fi
347 jmc 1.103 # <-- same processing for adjoint & forward test
348 edhill 1.1 }
349    
350     genmakemodel()
351     {
352     # genmakemodel directory
353 edhill 1.10 if test "x$NOGENMAKE" = xt ; then
354     echo "genmake skipped!"
355     else
356 edhill 1.34 if test "x$BASH" = x ; then
357     GENMAKE2="../../../tools/genmake2"
358     else
359     GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH"
360     fi
361 edhill 1.10 (
362     cd $1;
363 edhill 1.24 command="$GENMAKE2 -ds -m $MAKE"
364 jmc 1.141 if test "x$MKDEPEND" != x ; then
365     command="$command -makedepend=$MKDEPEND"
366     fi
367 jmc 1.175 if test $KIND = 1 -o $KIND = 2 ; then
368 utke 1.120 command="$command --mods=../code_ad"
369 jmc 1.175 elif test $KIND = 4 ; then
370 utke 1.120 command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'"
371     else
372 jmc 1.124 command="$command -mods=../code"
373 edhill 1.24 fi
374 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
375 jmc 1.124 command="$command -optfile=$OPTFILE"
376 edhill 1.10 fi
377 jmc 1.164 if test $OptLev = 1 ; then
378 jmc 1.175 command="$command -ieee"
379     fi
380 jmc 1.164 if test $OptLev = 0 ; then
381     command="$command -devel"
382 edhill 1.10 fi
383 jmc 1.109 if test "x$GSL" = xt ; then
384     command="$command -gsl"
385     fi
386 jmc 1.154 if test "x$MPI" != x0 ; then
387 edhill 1.66 command="$command -mpi"
388     fi
389 jmc 1.122 if test "x$MULTI_THREAD" = xt ; then
390     #- run multi-threaded using OpenMP:
391     command="$command -omp"
392     fi
393 jmc 1.135 if test "x$USE_R4" = xt ; then
394     command="$command -use_r4"
395     fi
396 jmc 1.165 if test "x$EXTRFLG" != x ; then
397     command="$command -extra_flag $EXTRFLG"
398     fi
399 ce107 1.90 if test "x$TS" = xt ; then
400     command="$command -ts"
401     fi
402     if test "x$PAPIS" = xt ; then
403     command="$command -papis"
404 jmc 1.135 else
405 ce107 1.90 if test "x$PCLS" = xt ; then
406     command="$command -pcls"
407     fi
408     fi
409 jmc 1.139 printf 'genmake ... '
410 jmc 1.146 eval $command > genmake.tr_log 2>&1
411 edhill 1.10 RETVAL=$?
412 edhill 1.44 # Reduce the size of the testing emails!
413 edhill 1.47 head -100 Makefile > $CDIR/Makefile_head
414 edhill 1.10 if test "x$RETVAL" != x0 ; then
415 jmc 1.146 tail genmake.tr_log
416 jmc 1.139 echo "genmakemodel: genmake failed"
417 jmc 1.148 cp genmake.log genmake_* genmake.tr_log $CDIR
418 edhill 1.10 return 1
419     else
420 jmc 1.139 echo "successful"
421 edhill 1.1 fi
422 edhill 1.10 )
423     fi
424 edhill 1.1 }
425    
426     makeclean()
427     {
428     # makeclean directory
429 jmc 1.146 if test "x$NODEPEND" = xf ; then rm -f $1/make.tr_log ; fi
430 edhill 1.10 if test "x$NOCLEAN" = xt ; then
431 jmc 1.83 echo "make Clean skipped!"
432 edhill 1.10 else
433     (
434     cd $1;
435 jmc 1.87 #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi
436 edhill 1.10 if test -r Makefile ; then
437 jmc 1.139 printf 'clean build-dir: make Clean ... '
438 jmc 1.146 $MAKE Clean >> make.tr_log 2>&1
439 edhill 1.10 RETVAL=$?
440     if test "x$RETVAL" != x0 ; then
441 jmc 1.146 tail make.tr_log
442 jmc 1.139 echo "makeclean: \"make Clean\" failed"
443 jmc 1.148 cp make.tr_log genmake.log genmake.tr_log $CDIR
444 edhill 1.10 return 1
445     fi
446 jmc 1.139 echo successful
447 jmc 1.127 else
448     echo ''
449 edhill 1.10 fi
450     exit 0
451     )
452     fi
453     }
454    
455 jmc 1.83 run_clean()
456 edhill 1.68 {
457 jmc 1.83 # run_clean directory
458     if test "x$NOCLEAN" = xt ; then
459     echo "run_clean skipped!"
460     else
461 edhill 1.68 (
462     cd $1;
463 jmc 1.139 printf 'clean run-dir ... '
464 jmc 1.83 # part of what is done after "make clean" when doing "make CLEAN"
465     find . -name "*.meta" -exec rm {} \;
466     find . -name "*.data" -exec rm {} \;
467     find . -name "fort.*" -exec rm {} \;
468     find . -type l -exec rm {} \;
469 jmc 1.161 #- should remove executable only if sym-link (alredy done above)
470 jmc 1.172 rm -f $RUNLOG *.txt STD* w2_tile_topology.????.log *diagnostics.log datetime
471 jmc 1.83 rm -rf mnc_test_*
472 jmc 1.175 rm -f *_MIT_CE_000.opt0000 costfunction*0000
473 jmc 1.185 rm -f oad_cp.000.[0-9][0-9][0-9][0-9][0-9]
474 jmc 1.139 echo successful
475 edhill 1.68 exit 0
476     )
477     fi
478     }
479    
480 edhill 1.10 makedependmodel()
481     {
482     # makedependmodel directory
483     if test "x$NODEPEND" = xt ; then
484     echo "make depend skipped!"
485     else
486     (
487     cd $1;
488 jmc 1.139 printf 'make depend ... '
489 jmc 1.146 $MAKE depend >> make.tr_log 2>&1
490 edhill 1.1 RETVAL=$?
491     if test "x$RETVAL" != x0 ; then
492 jmc 1.146 tail make.tr_log
493 jmc 1.139 echo "makedependmodel: make depend failed"
494 jmc 1.148 cp make.tr_log genmake.log genmake.tr_log $CDIR
495 edhill 1.1 return 1
496 edhill 1.10 else
497 jmc 1.139 echo successful
498 edhill 1.1 fi
499 edhill 1.10 )
500     fi
501 edhill 1.1 }
502    
503     makemodel()
504     {
505     # makemodel directory
506     (
507 jmc 1.159 mk_fail=0
508 jmc 1.136 if test "x$NOMAKE" = xt ; then
509     cd $1;
510     if test -x $EXECUTABLE ; then
511     echo "make skipped!"
512     else
513     echo "no executable!"
514 jmc 1.159 mk_fail=3
515 jmc 1.136 fi
516     else
517 edhill 1.1 cd $1;
518     if test -r Makefile ; then
519 jmc 1.139 printf 'make ... '
520 jmc 1.171 if test "x$REPLMAKE" = x ; then
521     $MAKE $TARG >> make.tr_log 2>&1
522     else
523     $REPLMAKE $TARG >> make.tr_log 2>&1
524     fi
525 edhill 1.1 RETVAL=$?
526     if test "x$RETVAL" != x0 ; then
527 jmc 1.146 tail make.tr_log
528 jmc 1.139 echo failed
529 jmc 1.163 cp genmake.log genmake.tr_log $CDIR
530     tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail"
531 jmc 1.131 rm -f $EXECUTABLE
532 jmc 1.159 mk_fail=1
533 edhill 1.1 else
534 jmc 1.139 echo successful
535 edhill 1.1 fi
536 jmc 1.159 else
537     echo "no Makefile !"
538     mk_fail=2
539 edhill 1.1 fi
540 jmc 1.136 fi
541 jmc 1.177 if test $KIND = 1 -a -f taf_ftl.log ; then
542 jmc 1.186 head -1 taf_ftl.log >> $CDIR"/summary.txt"
543 jmc 1.177 fi
544 jmc 1.175 if test $KIND = 2 -a -f taf_ad.log ; then
545 jmc 1.186 head -1 taf_ad.log >> $CDIR"/summary.txt"
546     nerr=`grep -c 'TAF *.* ERROR ' taf_ad.log`
547     nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' taf_ad.log`
548     if test -f taf_output ; then
549     n2er=`grep -c 'TAF *.* ERROR ' taf_output`
550     nerr=`expr $nerr + $n2er`
551     fi
552     echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \
553 jmc 1.158 >> $CDIR"/summary.txt"
554     fi
555 jmc 1.159 if test $mk_fail != 0 ; then return $mk_fail ; fi
556 edhill 1.1 )
557     }
558    
559 jmc 1.154 mk_mpi_size()
560     {
561 jmc 1.173 # mk_mpi_size input_file output_file proc_Nb threads_Nb_X threads_Nb_Y
562 jmc 1.154 #
563 jmc 1.173 # make new SIZE.h (=output_file) from SIZE.h_mpi (=input_file)
564 jmc 1.154 # for an MPI build with no more than proc_Nb processors ;
565 jmc 1.173 # ensure that enough tiles per proc (nSx,nSy) remain for the given
566     # number of threads (nTx,nTy) ;
567     # return the effective number of processors.
568 jmc 1.154
569     inp=$1
570     out=$2
571     np=$3
572 jmc 1.173 tx=$4
573     ty=$5
574 jmc 1.154 tmp=TTT.$$
575    
576 jmc 1.173 # dirX : select with direction to favor in MPI process repartition
577     # dirX=1 : prefer to put more proc in X direction
578     # dirX=0 : prefer to put more proc in Y direction
579     dirX=0
580    
581 jmc 1.154 px=`grep "^ & *nPx *=" $inp | sed "s/^ & *nPx *= *//" | sed 's/, *$//'`
582     py=`grep "^ & *nPy *=" $inp | sed "s/^ & *nPy *= *//" | sed 's/, *$//'`
583     sx=`grep "^ & *nSx *=" $inp | sed "s/^ & *nSx *= *//" | sed 's/, *$//'`
584     sy=`grep "^ & *nSy *=" $inp | sed "s/^ & *nSy *= *//" | sed 's/, *$//'`
585    
586 jmc 1.173 #- for each direction, assume # of threads is a multiple of total number of tiles
587     nx=$px
588     if [ `expr $sx % $tx` -ne 0 -a `expr $sx \* $px % $tx` -eq 0 ] ; then
589     nx=`expr $sx \* $px / $tx`
590     if [ $verbose -gt 1 ]; then
591     echo " change px from $px to $nx to accommodate $tx threads"
592     fi
593     fi
594     ny=$py
595     if [ `expr $sy % $ty` -ne 0 -a `expr $sy \* $py % $ty` -eq 0 ] ; then
596     ny=`expr $sy \* $py / $ty`
597     if [ $verbose -gt 1 ]; then
598     echo " change py from $py to $ny to accommodate $ty threads"
599     fi
600     fi
601 jmc 1.154 #- find the largest divisor of input_file proc Nb, but not larger than $np
602     pp=0
603 jmc 1.155 i=1
604 jmc 1.173 while [ $i -le $nx ] ; do
605     if [ `expr $nx % $i` -eq 0 ] ; then
606 jmc 1.155 j=1
607 jmc 1.173 while [ $j -le $ny ] ; do
608     if [ `expr $ny % $j` -eq 0 ] ; then
609 jmc 1.155 ij=`expr $i \* $j`
610     if [ $ij -gt $pp ] ; then
611 jmc 1.161 flag=1
612     elif [ $ij -eq $pp ] ; then
613     flag=$dirX
614     else
615     flag=0
616     fi
617     if test $flag = 1 ; then
618 jmc 1.155 if [ $ij -le $np ] ; then
619     ix=$i ; jy=$j ; pp=$ij
620     #echo " ix,jy= $ix,$jy"
621     fi
622     fi
623     fi
624     j=`expr $j + 1`
625     done
626 jmc 1.154 fi
627 jmc 1.155 i=`expr $i + 1`
628 jmc 1.154 done
629    
630     #- create new SIZE.h type file:
631     sx=`expr $sx \* $px / $ix`
632     sy=`expr $sy \* $py / $jy`
633     if [ $verbose -gt 1 ]; then
634     echo " px,py,np= $px,$py,$np : New MPI size: px,py= $ix,$jy : sx,sy= $sx,$sy"
635     fi
636     sed "/^ \& *nPx *=/s/[0-9]*,/$ix,/" $inp > $tmp
637     sed "/^ \& *nPy *=/s/[0-9]*,/$jy,/" $tmp > $out
638     sed "/^ \& *nSx *=/s/[0-9]*,/$sx,/" $out > $tmp
639     sed "/^ \& *nSy *=/s/[0-9]*,/$sy,/" $tmp > $out
640     rm -f $tmp
641     return $pp
642     }
643    
644 edhill 1.27 symlink_mpifiles()
645     {
646     # Put special links so that MPI specific files are used
647     # This MUST be invoked between makeclean and makelinks because
648     # the Makefile will link to non-mpi files by default
649    
650     dir=$1
651     code_dir=$2
652 jmc 1.154 build_dir=$dir/$3
653 jmc 1.135
654 edhill 1.27 # These are files that should replace their counter-part when using -mpi
655 jmc 1.154 MPI_FILES=`(cd $dir/$code_dir; find . -name "*_mpi" -print)`
656 edhill 1.27
657 jmc 1.154 for ii in $MPI_FILES ; do
658     i=`echo $ii | sed 's:^\./::'`
659     name=`echo $i | sed 's:_mpi::'`
660     file="../$code_dir/$i"
661     if test $name = 'SIZE.h' ; then file="SIZE.h.mpi" ; fi
662    
663     # Is this an MPI run?
664     if test "x$MPI" = x0 ; then
665     # NO: We undo any _mpi symbolically linked files
666     if test -L $build_dir/$name ; then
667     ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 )
668     RETVAL=$?
669     if test "x$RETVAL" = x0 ; then
670     if [ $verbose -gt 1 ]; then
671     echo " Un-linking $name from ../$code_dir" ; fi
672     rm -f $build_dir/$name
673     fi
674     fi
675     else
676     # YES: We symbolically link these files to the build
677     # dir so long as there is no real file in place
678     ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 )
679 edhill 1.27 RETVAL=$?
680 jmc 1.154 if [ $verbose -gt 1 ]; then echo " cmp $name $file returns: $RETVAL" ; fi
681 edhill 1.27 if test "x$RETVAL" != x0 ; then
682 jmc 1.154 if test -h $build_dir/$name ; then rm -f $build_dir/$name ; fi
683     if test ! -r $build_dir/$name ; then
684     if [ $verbose -gt 1 ]; then echo " Linking $name to $file" ; fi
685     (cd $build_dir; ln -sf $file $name)
686 edhill 1.27 fi
687     fi
688 jmc 1.154 fi
689     done
690 edhill 1.27 }
691    
692 edhill 1.1 linkdata()
693     {
694 jmc 1.83 # linkdata run_dir input_dir_1 input_dir_2 ...
695 edhill 1.1 #
696     # symbolically link data files to run directory
697 jmc 1.83 if test -d $1 ; then
698 edhill 1.12 (
699 jmc 1.83 cd $1 ; shift
700 jmc 1.139 echo 'linkdata from dirs:' $*
701 jmc 1.175 inpMPI=`(cd ../$1 ; find . -name "*.mpi" -print | sed 's:^\./::')`
702 jmc 1.125 for xx in $inpMPI ; do
703     if test -r "../"$1"/"$xx ; then
704 jmc 1.175 # found 1 _mpi sfx file in 1rst input dir and it is readable
705 jmc 1.125 yy=`echo $xx | sed 's:\.mpi$::'`
706 jmc 1.154 if test "x$MPI" = "x0" ; then
707     # not mpi test: remove symbolic link
708     if test -h $yy ; then rm -f $yy ; fi
709     else
710     # mpi test: remove symbolic link & link .mpi sfx file
711 jmc 1.125 if test -h $yy ; then rm -f $yy ; fi
712 jmc 1.135 if test ! -r $yy ; then
713 jmc 1.125 ln -sf "../"$1"/"$xx $yy ;
714 jmc 1.139 printf " $xx" 1>&2
715 jmc 1.125 fi
716     fi
717     fi
718     done
719 jmc 1.83 if test -r "../"$1"/eedata.mth" ; then
720     # found eedata.mth in 1rst input dir and it is readable
721     if test "x$MULTI_THREAD" = "xt" ; then
722     # multi-threaded test: remove symbolic link & link eedata.mth
723     if test -h eedata ; then rm -f eedata ; fi
724 jmc 1.135 if test ! -r eedata ; then
725 jmc 1.83 ln -sf "../"$1"/eedata.mth" eedata ;
726 jmc 1.139 printf ' eedata.mth' 1>&2
727 edhill 1.24 fi
728 jmc 1.83 else
729     # not multi-threaded test: remove eedata symbolic link
730     if test -h eedata ; then rm -f eedata ; fi
731     fi
732     fi
733 jmc 1.108 prevDir='NONE'
734 jmc 1.83 for ldir in $* ; do
735 jmc 1.108 if test -d "../"$ldir -a $ldir != $prevDir ; then
736 jmc 1.175 printf " ldir=${ldir}:" 1>&2
737 jmc 1.83 files=`( cd "../"$ldir ; ls -1 | grep -v CVS )`
738     for i in $files ; do
739     if test ! -d "../"$ldir/$i ; then
740     if test ! -r $i ; then
741     printf ' '$i 1>&2
742     ln -sf "../"$ldir"/"$i $i
743     fi
744     fi
745     done
746 jmc 1.139 printf ' ;' 1>&2
747 jmc 1.83 if test -x "../"$ldir"/"prepare_run ; then
748 jmc 1.139 "../"$ldir"/"prepare_run 1>&2
749     else
750     echo '' 1>&2
751 edhill 1.24 fi
752 jmc 1.82 fi
753 jmc 1.108 prevDir=$ldir
754 jmc 1.83 done
755 edhill 1.12 )
756 edhill 1.1 fi
757     }
758    
759     runmodel()
760     {
761 edhill 1.6 # runmodel directory
762 edhill 1.1 #
763 jmc 1.154 # runs $COMMAND in "directory"
764 edhill 1.6 # (where "$COMMAND" is relative to "directory")
765 edhill 1.1 (
766     cd $1
767 jmc 1.139 printf 'runmodel in %s ... ' $1
768 jmc 1.158 if test "x$MPI" != x0 ; then
769     #- adjust the MPI run command with the right number of Procs
770     #echo '' ; echo " COMMAND='$COMMAND'"
771     COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"`
772     if test "x$MPI_MFILE" != x ; then
773     COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"`
774     fi
775     #echo " COMMAND='$COMMAND'"
776     fi
777 jmc 1.134 if test -L $EXECUTABLE ; then
778     if test -x "../"$builddir"/"$EXECUTABLE ; then
779 jmc 1.154 cmp $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1
780     outD=$? ; if test "x$outD" != x0 ; then rm -f $EXECUTABLE ; fi
781 jmc 1.134 else rm -f $EXECUTABLE
782     fi
783 jmc 1.106 fi
784 jmc 1.175 if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then
785 jmc 1.119 echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp
786 jmc 1.83 ln -sf "../"$builddir"/"$EXECUTABLE .
787     fi
788 jmc 1.175 if test ! -x $EXECUTABLE ; then
789 jmc 1.156 rm -f $RUNLOG ; touch $RUNLOG
790     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
791     echo " no executable:" $EXECUTABLE >> $RUNLOG
792     RETVAL=8
793     ENDVAL=-1
794 jmc 1.75 else
795 jmc 1.175 if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then
796     # output do not exist or is older than executable:
797 jmc 1.162 rm -f $OUTPUTFILE $RUNLOG ; touch $RUNLOG
798 jmc 1.156 if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
799 jmc 1.187 #--a quick and dirty trick to run with Divided Adjoint (DIVA):
800     if test $KIND = 2 -a -x dorun.sh ; then
801     ./dorun.sh >> $RUNLOG 2>&1
802     else
803 jmc 1.156 ( eval $COMMAND ) >> $RUNLOG 2>&1
804 jmc 1.187 fi
805     #----------
806 jmc 1.156 RETVAL=$?
807     ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'`
808     if [ $POSTCLEAN -eq 1 -a $ENDVAL -gt 0 ] ; then
809     find . -name "*.meta" -exec rm {} \;
810     find . -name "*.data" -exec rm {} \;
811     rm -rf mnc_test_*
812 jmc 1.83 fi
813 jmc 1.156 else
814     RETVAL=0
815 jmc 1.153 ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'`
816 jmc 1.156 touch $RUNLOG
817     if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi
818 jmc 1.175 echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1
819 jmc 1.156 fi
820 jmc 1.75 fi
821 jmc 1.119 rm -f run.log_tmp
822 jmc 1.177 #- in all cases where OutputFile exists, report SIZE and time
823 jmc 1.158 if test -f $OUTPUTFILE ; then
824     grep '(PID\.TID 0000\.0001) n.. =' $OUTPUTFILE \
825 jmc 1.177 | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt"
826     grep -A3 'Seconds in section "ALL' $OUTPUTFILE \
827     | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt"
828 jmc 1.158 fi
829     if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi
830 jmc 1.88 if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then
831 jmc 1.139 echo successful
832     printf '=> output from running in %s :\n' $1 1>&2
833     tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
834 edhill 1.6 return 0
835 jmc 1.132 elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then
836     #-- for some weird cases (run is finihed but with error code)
837 jmc 1.139 echo 'finished with error (run:' $RETVAL ' end:' $ENDVAL ')'
838     printf '=> output from running in %s :\n' $1 1>&2
839     tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
840 jmc 1.132 return 0
841 edhill 1.6 else
842 jmc 1.139 echo 'failed (run:' $RETVAL ' end:' $ENDVAL ')'
843     printf '=> output from running in %s :\n' $1 1>&2
844     tail $RUNLOG | sed 's/^.*/> &/g' 1>&2
845 jmc 1.119 cp $RUNLOG $CDIR"/"$RUNLOG
846 edhill 1.6 return 1
847 edhill 1.1 fi
848     )
849     }
850    
851     createcodelet()
852     {
853     # create codelet for comparing model output
854    
855 jmc 1.112 printf "creating the comparison code (using CC=$CC)... "
856 jmc 1.91 cat > tr_cmpnum.c <<EOF
857 edhill 1.22 #include <stdio.h>
858     #include <math.h>
859     int main( int argc, char** argv ) {
860 jmc 1.91 int linnum,cmplin,best,lncnt;
861 edhill 1.52 double a,b,abave,relerr;
862 jmc 1.57 best = -22;
863 adcroft 1.33 lncnt = 0;
864 jmc 1.96 while( 1 & ( (lncnt+=1) < 999 ) ) {
865 edhill 1.22 scanf("%d", &linnum);
866     if (linnum == -1) break;
867     scanf("%lf", &a); scanf("%lf", &b);
868 edhill 1.52 abave = 0.5*(fabs(a)+fabs(b));
869 jmc 1.91 if ( abave == abave ) {
870     if (abave > 0.0) {
871     relerr=fabs(a-b)/abave;
872     if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); }
873     else { cmplin = -16 ; }
874     best = (best > cmplin) ? best : cmplin; }
875     else { cmplin = -22 ; }
876     /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */
877     }
878     else {
879     /* printf("%lf ; %lf ; %lf\n",abave,a,b); */
880     break; }
881 edhill 1.22 }
882 adcroft 1.33 if (lncnt == 999) best=-29;
883 jmc 1.91 if (linnum != -1) best=-99;
884 edhill 1.22 printf("%d\n", -best);
885     return 0;
886     }
887     EOF
888 jmc 1.91 $CC -o tr_cmpnum tr_cmpnum.c -lm
889 edhill 1.1
890 jmc 1.91 if [ -x ./tr_cmpnum ]; then
891 edhill 1.1 echo "OK"
892     return 0
893     else
894     echo
895 jmc 1.135 echo "ERROR: failed to compile comparison code -- please specify"
896 edhill 1.58 echo " a C compiler using the CC environment variable."
897 edhill 1.1 exit 1
898     fi
899     }
900    
901     formatresults()
902     {
903     # formatresults expt genmake depend make run results*
904    
905     nm=$1
906     printf '%s %s %s %s' $2 $3 $4 $5
907     shift; shift; shift; shift; shift;
908 jmc 1.94 listPrt=$@
909     listRes=`echo $listPrt | sed 's/>//' | sed 's/<//'`
910     xx=`echo $listPrt | sed 's/.*>//' | sed 's/<.*//' | awk '{print $1}'`
911 jmc 1.97 printf '%3s' $listPrt
912 jmc 1.98 # line below does not work on hp-ux_ia64 : do those substitutions later on
913 jmc 1.95 # printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</'
914 jmc 1.135
915 jmc 1.94 if [ $xx = '..' ]; then
916     printf ' N/O '
917     elif [ $xx = '--' ]; then
918     printf ' N/O '
919     elif [ $xx = 99 ]; then
920 edhill 1.1 printf ' N/O '
921     else
922 jmc 1.94 if [ $xx -ge $MATCH_CRIT ]; then
923 edhill 1.1 printf ' pass'
924     else
925     printf ' FAIL'
926     fi
927     fi
928     printf ' %s' $nm
929 jmc 1.186 if test $KIND = 2 ; then
930     #-- apppend taf repport summary:
931     tafrep=`grep -c '^ TAF reports ' $CDIR/summary.txt`
932     if test $tafrep = 1 ; then
933     grep '^ TAF reports ' $CDIR/summary.txt | awk '{printf " (e=%i, w=%i)",$3,$6}'
934     fi
935     fi
936 edhill 1.1 printf '\n'
937 jmc 1.135
938 edhill 1.1 }
939    
940     scandirs()
941     {
942 jmc 1.81 if [ $# -eq 1 ]; then
943     for arg in * ; do
944 jmc 1.175 #test -f $arg/$1 && echo $arg
945     test -f $arg/$1 -o -f $arg/$1.gz && echo $arg
946 jmc 1.81 done
947 edhill 1.1 else
948 jmc 1.81 echo $*
949 edhill 1.1 fi
950     }
951    
952    
953 jmc 1.123 check_eedata()
954     {
955 jmc 1.161 # check_eedata eedata size.h
956 jmc 1.123 if [ $# -eq 2 ] ; then
957 jmc 1.161 if test -f $1 -a -f $2 ; then
958     nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
959     sx=`grep "^ & *nSx *=" $2 | sed "s/^ & *nSx *=//" | sed 's/, *$//'`
960 jmc 1.123 if test "x$nx" = x ; then
961     rx=10
962     else
963     rx=`expr $sx % $nx`
964     fi
965 jmc 1.161 ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
966     sy=`grep "^ & *nSy *=" $2 | sed "s/^ & *nSy *=//" | sed 's/, *$//'`
967 jmc 1.123 if test "x$ny" = x ; then
968     ry=20
969     else
970     ry=`expr $sy % $ny`
971     fi
972     echo `expr $rx + $ry`
973 jmc 1.161 else
974     echo '-1'
975     fi
976     elif [ $# -eq 1 ] ; then
977     if test -f $1 ; then
978     nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
979     if test "x$nx" = x ; then nx=1 ; fi
980     ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
981     if test "x$ny" = x ; then ny=1 ; fi
982     #echo $nx $ny
983     echo $nx
984     else
985 jmc 1.123 echo '-1'
986 jmc 1.161 fi
987 jmc 1.123 else
988     echo '-2'
989     fi
990    
991     }
992    
993 edhill 1.1 ###############################################################################
994    
995    
996     # Default properties
997     debug=0
998     verbose=1
999 jmc 1.163 NBLINES_MKLOG=16000
1000 edhill 1.10
1001 jmc 1.164 OptLev=1
1002 jmc 1.109 GSL=f
1003 edhill 1.10
1004     CLEANUP=f
1005 jmc 1.133 NORUN=f
1006 edhill 1.10 QUICK=f
1007 jmc 1.136 NOMAKE=f
1008 edhill 1.10 NOGENMAKE=f
1009     NOCLEAN=f
1010     NODEPEND=f
1011 jmc 1.156 POSTCLEAN=0
1012 edhill 1.1
1013 edhill 1.4 BASH=
1014 edhill 1.10 OPTFILE=NONE
1015     ADDRESSES=
1016 edhill 1.1 TESTDIRS=
1017 jmc 1.89 SKIPDIRS=
1018 edhill 1.1 MPACKDIR="../tools/mpack-1.6"
1019 edhill 1.20 HAVE_MPACK=
1020 jmc 1.113 MPACK=
1021 edhill 1.24 COMMAND=
1022 jmc 1.141 MKDEPEND=
1023 edhill 1.59 if test "x$MAKE" = x ; then
1024     MAKE=make
1025     fi
1026 jmc 1.171 REPLMAKE=
1027 edhill 1.59 if test "x$CC" = x ; then
1028     CC=cc
1029     fi
1030 edhill 1.43 JOBS=
1031 jmc 1.165 TARG=
1032 jmc 1.154 MPI=0
1033 jmc 1.156 MPI_MFILE=
1034 jmc 1.83 MULTI_THREAD=f
1035 jmc 1.70 OUTDIR=
1036 edhill 1.40 DELDIR=
1037 jmc 1.135 USE_R4=
1038 jmc 1.165 EXTRFLG=
1039 edhill 1.1
1040 jmc 1.175 #- type of testing (KIND):
1041     # KIND=0 : forward (= default) ; KIND=1 : Tangent Linear with TAF ;
1042     # KIND=2 : Adjoint with TAF ; KIND=4 : Adjoint with OpenAD
1043     KIND=0
1044 edhill 1.24
1045 jmc 1.117 # list of pTracers to check for monitor output
1046 edhill 1.50 PTRACERS_NUM="1 2 3 4 5"
1047 edhill 1.49
1048 jmc 1.147 MATCH_CRIT=10
1049 jmc 1.92
1050 edhill 1.34 printf "parsing options... "
1051 edhill 1.1
1052     ac_prev=
1053     for ac_option ; do
1054    
1055     # If the previous option needs an argument, assign it.
1056     if test -n "$ac_prev"; then
1057     eval "$ac_prev=\$ac_option"
1058     ac_prev=
1059     continue
1060     fi
1061    
1062     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
1063 jmc 1.135
1064 edhill 1.1 case $ac_option in
1065    
1066 jmc 1.156 -help | --help | -h | --h) usage ;;
1067    
1068     -optfile | --optfile | -of | --of) ac_prev=OPTFILE ;;
1069     -optfile=* | --optfile=* | -of=* | --of=*) OPTFILE=$ac_optarg ;;
1070    
1071     -addr | --addr | -a | --a) ac_prev=ADDRESSES ;;
1072     -addr=* | --addr=* | -a=* | --a=*) ADDRESSES=$ac_optarg ;;
1073     -mpackdir | --mpackdir | -mpd | --mpd) ac_prev=MPACKDIR ;;
1074     -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*) MPACKDIR=$ac_optarg ;;
1075    
1076     -tdir | --tdir | -t | --t) ac_prev=TESTDIRS ;;
1077     -tdir=* | --tdir=* | -t=* | --t=*) TESTDIRS=$ac_optarg ;;
1078     -skipdir | --skipdir | -skd | --skd) ac_prev=SKIPDIRS ;;
1079     -skipdir=* | --skipdir=* | -skd=* | --skd=*) SKIPDIRS=$ac_optarg ;;
1080    
1081     -bash | --bash | -b | --b) ac_prev=BASH ;;
1082     -bash=* | --bash=* | -b=* | --b=*) BASH=$ac_optarg ;;
1083    
1084     -command | --command | -c | --c) ac_prev=COMMAND ;;
1085     -command=* | --command=* | -c=* | --c=*) COMMAND=$ac_optarg ;;
1086    
1087     -makedepend | --makedepend | -md | --md) ac_prev=MKDEPEND ;;
1088     -makedepend=* | --makedepend=* | -md=* | --md=*) MKDEPEND=$ac_optarg ;;
1089    
1090     -make | --make | -m | --m) ac_prev=MAKE ;;
1091     -make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;;
1092    
1093 jmc 1.171 -repl_mk | --repl_mk ) ac_prev=REPLMAKE ;;
1094     -repl_mk=* | --repl_mk=*) REPLMAKE=$ac_optarg ;;
1095    
1096 jmc 1.156 -odir | --odir) ac_prev=OUTDIR ;;
1097     -odir=* | --odir=*) OUTDIR=$ac_optarg ;;
1098    
1099     -ptracers | --ptracers | -ptr | --ptr) ac_prev=PTRACERS_NUM ;;
1100     -ptracers=* | --ptracers=* | -ptr=* | --ptr=*) PTRACERS_NUM=$ac_optarg ;;
1101 edhill 1.50
1102 jmc 1.92 -match | --match ) ac_prev=MATCH_CRIT ;;
1103     -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;;
1104    
1105 jmc 1.136 -j | --j) ac_prev=JOBS ;;
1106     -j=* | --j=*) JOBS=$ac_optarg ;;
1107 edhill 1.43
1108 jmc 1.165 -ef | --ef) ac_prev=EXTRFLG ;;
1109     -ef=* | --ef=*) EXTRFLG=$ac_optarg ;;
1110    
1111 jmc 1.156 -clean | --clean) CLEANUP=t ; DELDIR=t ;;
1112 edhill 1.10
1113 jmc 1.156 -norun | --norun | -nr | --nr) NORUN=t ;;
1114 jmc 1.165 -obj | --obj ) TARG='obj' ; NORUN=t ;;
1115 jmc 1.156 -runonly | --runonly | -ro | --ro) QUICK=t ; NOMAKE=t ;;
1116     -quick | --quick | -q | --q) QUICK=t ;;
1117     -nogenmake | --nogenmake | -ng | --ng) NOGENMAKE=t ;;
1118     -noclean | --noclean | -nc | --nc) NOCLEAN=t ;;
1119     -nodepend | --nodepend | -nd | --nd) NODEPEND=t ;;
1120 edhill 1.10
1121 jmc 1.156 -postclean | --postclean | -pc | --pc) POSTCLEAN=2 ;;
1122     -deloutp | --deloutp | -do | --do) POSTCLEAN=1 ;;
1123 edhill 1.68
1124 jmc 1.154 -mpi | --mpi) MPI=2 ;;
1125     -MPI | --MPI) ac_prev=MPI ;;
1126     -MPI=* | --MPI=*) MPI=$ac_optarg ;;
1127 edhill 1.10
1128 jmc 1.156 -mfile | --mfile | -mf | --mf) ac_prev=MPI_MFILE ;;
1129     -mfile=* | --mfile=* | -mf=* | --mf=*) MPI_MFILE=$ac_optarg ;;
1130    
1131 jmc 1.83 -mth) MULTI_THREAD=t ;;
1132    
1133 jmc 1.175 -tlm) if test $KIND = 0 ; then KIND=1 ; else
1134     echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage
1135     fi ;;
1136     -adm | -ad) if test $KIND = 0 ; then KIND=2 ; else
1137     echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage
1138     fi ;;
1139     -oad) if test $KIND = 0 ; then KIND=4 ; NODEPEND=t ; else
1140     echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage
1141     fi ;;
1142 utke 1.120
1143 jmc 1.164 -ieee) echo "Warning: ignore option '-ieee' (already the default)"
1144 jmc 1.175 printf " ... " ;;
1145 jmc 1.164 -noieee) echo "Warning: will use option '-fast' instead of '-noieee' (obsolete)"
1146 jmc 1.175 printf " ... " ; OptLev=`expr $OptLev \* 2` ;;
1147 jmc 1.164 -fast) OptLev=`expr $OptLev \* 2` ;;
1148     -devel) OptLev=0 ;;
1149 jmc 1.156 -gsl) GSL=t ;;
1150 edhill 1.10
1151 edhill 1.1 -verbose) verbose=2 ;;
1152     -debug) debug=1 ;;
1153     -quiet) verbose=0 ;;
1154    
1155 edhill 1.40 -deldir | -dd) DELDIR=t ;;
1156    
1157 jmc 1.136 -use_r4|-ur4) USE_R4=t ;;
1158 jmc 1.135
1159 ce107 1.90 -ts) TS=t;;
1160     -papis) PAPIS=t;;
1161     -pcls) PCL=t;;
1162    
1163 jmc 1.156 -*) echo "Error: unrecognized option: "$ac_option
1164     usage ;;
1165     *) echo "Error: unrecognized argument: "$ac_option
1166     usage ;;
1167 jmc 1.135
1168 edhill 1.1 esac
1169 jmc 1.135
1170 edhill 1.1 done
1171    
1172 edhill 1.10 if test "x$QUICK" = xt ; then
1173     NOGENMAKE=t
1174     NOCLEAN=t
1175     NODEPEND=t
1176     fi
1177    
1178 jmc 1.156 #- check length of MPI machine file:
1179     if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then
1180     if test -r $MPI_MFILE ; then
1181     nl=`wc -l $MPI_MFILE | awk '{print $1}'`
1182     if [ $nl -lt $MPI ] ; then
1183 jmc 1.175 echo "Error: need at least $MPI nodes (currently only $nl) in MPI_MFILE=$MPI_FILE"
1184     usage
1185 jmc 1.156 fi
1186     if [ $verbose -gt 1 ]; then
1187     echo " MPI_MFILE=$MPI_MFILE : $nl procs for MPI=$MPI run"
1188     fi
1189     else
1190 jmc 1.175 echo "Error: cannot access MPI_MFILE=$MPI_FILE"
1191     usage
1192 jmc 1.156 fi
1193     fi
1194    
1195 jmc 1.99 #- setting for forward or ADM testing
1196 jmc 1.175 if test $KIND = 1 ; then
1197     TARG=ftlall
1198     code_dir=code_ad
1199     inputdir=input_ad
1200     ref_outp="output_tlm.txt"
1201     EXECUTABLE="mitgcmuv_ftl"
1202     elif test $KIND = 2 ; then
1203 jmc 1.168 if test "x$TARG" = xobj ; then TARG=adobj ; else TARG=adall ; fi
1204 jmc 1.99 code_dir=code_ad
1205 jmc 1.108 inputdir=input_ad
1206 jmc 1.99 ref_outp="output_adm.txt"
1207     EXECUTABLE="mitgcmuv_ad"
1208 jmc 1.175 elif test $KIND = 4 ; then
1209 jmc 1.168 TARG=adAll
1210 utke 1.120 code_dir=code_oad
1211     inputdir=input_oad
1212     ref_outp="output_oadm.txt"
1213     EXECUTABLE="mitgcmuv_ad"
1214 jmc 1.99 else
1215 jmc 1.168 if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi
1216 jmc 1.99 code_dir=code
1217 jmc 1.108 inputdir=input
1218 jmc 1.99 ref_outp="output.txt"
1219     EXECUTABLE="mitgcmuv"
1220     fi
1221    
1222 jmc 1.142 xx=`echo $TESTDIRS | awk '{print $1}'`
1223 edhill 1.1 if test "x$TESTDIRS" = x ; then
1224 jmc 1.103 LIST=`scandirs results/$ref_outp`
1225 jmc 1.142 elif test $xx = 'start_from' ; then
1226     xx=`echo $TESTDIRS | awk '{print $2}'`
1227     LIST=`scandirs results/$ref_outp | sed -n "/$xx/,$ p"`
1228 jmc 1.84 else
1229     #- expand group of experiments:
1230     LIST=" "
1231     for xx in $TESTDIRS
1232     do
1233     case $xx in
1234 jmc 1.86 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce"
1235     LIST=${LIST}" lab_sea tutorial_baroclinic_gyre"
1236     LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope"
1237 jmc 1.84 ;;
1238     'tutorials')
1239     LIST=${LIST}" "`ls | grep 'tutorial_'` ;;
1240     *) LIST=${LIST}" "$xx ;;
1241     esac
1242 jmc 1.135 done
1243 jmc 1.89 fi
1244     #echo 'LIST='${LIST}'<'
1245     #- skip dirs, remove duplicate and non-directory:
1246     TESTDIRS=" "
1247     count=0
1248     for xx in $LIST
1249     do
1250     yy=`echo $SKIPDIRS | grep -c $xx`
1251     if test $yy = 0 ; then
1252 jmc 1.84 if test -d $xx ; then
1253     yy=`echo $TESTDIRS | grep -c $xx`
1254     if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi
1255 jmc 1.89 else count=1 ;
1256     echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
1257     fi
1258     else
1259 jmc 1.175 if test $count = 1 ; then echo -n ", \"$xx\""
1260 jmc 1.89 else count=1 ; echo "" ; echo -n " skip: \"$xx\""
1261 jmc 1.84 fi
1262 jmc 1.89 fi
1263 jmc 1.135 done
1264 jmc 1.89 if test $count = 1 ; then echo "" ; echo -n " ... " ; fi
1265 jmc 1.84 #echo 'TESTDIRS='${TESTDIRS}'<'
1266 edhill 1.1
1267 edhill 1.10 if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then
1268     OPTFILE=$MITGCM_OF
1269     fi
1270    
1271 jmc 1.156 LOC_MFILE='tr_mpi_mfile'
1272 jmc 1.146 RUNLOG="run.tr_log"
1273 jmc 1.99 OUTPUTFILE=$ref_outp
1274 jmc 1.87 if test "x$COMMAND" = x ; then
1275 jmc 1.77 COMMAND="./$EXECUTABLE > $OUTPUTFILE"
1276 edhill 1.24 fi
1277 jmc 1.154 if test "x$MPI" != x0 ; then
1278 jmc 1.87 OUTPUTFILE="STDOUT.0000"
1279 edhill 1.24 fi
1280    
1281 jmc 1.76 echo "OK (COMMAND= $COMMAND )"
1282 edhill 1.1
1283 jmc 1.94 # set the Default List of output variables to be checked:
1284     # (use default or load experiment-specific list from file "tr_checklist")
1285     # content : 1rst = main variable used to decide if it pass or FAIL
1286     # others = number of matching digits to be printed in summary.txt
1287 jmc 1.175 if test $KIND = 1 ; then
1288     DEF_CHECK_LIST='tlmGrd tlmCst tlmGrd fwdGrd'
1289     EMPTY_RESULTS='.. .. ..'
1290     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1291     elif test $KIND = 2 -o $KIND = 4 ; then
1292     DEF_CHECK_LIST='AdGrd Cost AdGrd FDGrd'
1293     EMPTY_RESULTS='.. .. ..'
1294     LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1295     else
1296 jmc 1.94 DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+'
1297     EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..'
1298 jmc 1.97 LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'`
1299 jmc 1.94 ii=`echo $EMPTY_RESULTS | awk '{print NF}'`
1300 jmc 1.97 EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'`
1301 jmc 1.94 fi
1302    
1303 edhill 1.1 # create the FORTRAN comparison code
1304 jmc 1.178 if test "x$CLEANUP" = xt -o -x tr_cmpnum ; then
1305 jmc 1.133 echo "skipping comparison code build"
1306     else
1307     createcodelet
1308     fi
1309 edhill 1.1
1310 jmc 1.112 # build the mpack utility (if ADDRESSES = NONE, do it to test the build)
1311     if test "x$ADDRESSES" = x ; then
1312 edhill 1.32 echo "skipping mpack build"
1313     else
1314 edhill 1.31 build_mpack
1315     fi
1316 edhill 1.1
1317     # Create a uniquely named directory to store results
1318 jmc 1.110 CMDLINE=$0
1319 jmc 1.118 for xx in "$@" ; do nw=`echo $xx | wc -w`
1320 jmc 1.180 if test $nw = '1' ; then CMDLINE="$CMDLINE $xx" ; else
1321     nb=`echo $xx | grep -c '='`
1322     if test $nb = 0 ; then CMDLINE="$CMDLINE '$xx'"
1323     else yy=`echo "$xx'" | sed "s/=/='/"` ;
1324     CMDLINE="$CMDLINE $yy" ; fi
1325     fi
1326 jmc 1.118 done
1327     #for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done
1328 edhill 1.1 MACH=`hostname`
1329 edhill 1.2 UNAMEA=`uname -a`
1330 edhill 1.1 DATE=`date +%Y%m%d`
1331 edhill 1.25 BASE="tr_"$MACH"_"$DATE"_"
1332 jmc 1.70 if test "x$OUTDIR" != x ; then
1333     BASE="tr_"$OUTDIR"_"$DATE"_"
1334 jmc 1.121 else
1335     short_name=`hostname | sed 's/\..*$//'`
1336     BASE="tr_"$short_name"_"$DATE"_"
1337 jmc 1.70 fi
1338 edhill 1.1 DNUM=0
1339     DRESULTS="$BASE$DNUM"
1340     while test -e $DRESULTS ; do
1341     DNUM=$(( $DNUM + 1 ))
1342     DRESULTS="$BASE$DNUM"
1343     done
1344     mkdir $DRESULTS
1345     RETVAL=$?
1346     if test "x$RETVAL" != x0 ; then
1347 edhill 1.20 echo "ERROR: Can't create results directory \"./$DRESULTS\""
1348 edhill 1.1 exit 1
1349     fi
1350     SUMMARY="$DRESULTS/summary.txt"
1351 edhill 1.16 start_date=`date`
1352 edhill 1.17 echo $start_date > $SUMMARY
1353 jmc 1.110 echo 'run:' $CMDLINE >> $SUMMARY
1354     echo 'on :' $UNAMEA >> $SUMMARY
1355 edhill 1.1
1356 edhill 1.11 of_path=
1357 edhill 1.10 if test "x$OPTFILE" != xNONE ; then
1358     if test -r $OPTFILE ; then
1359 edhill 1.11 # get the path
1360     path=${OPTFILE%/*}
1361     if test "x$path" = x ; then
1362     of_path=`pwd`
1363     else
1364     of_path=`( cd $path > /dev/null 2>&1 ; pwd )`
1365     fi
1366     file=${OPTFILE##*/}
1367     OPTFILE=$of_path/$file
1368 edhill 1.21 cp $OPTFILE $DRESULTS
1369     echo >> $SUMMARY
1370     echo " OPTFILE=$OPTFILE" >> $SUMMARY
1371 edhill 1.11 else
1372 jmc 1.179 echo | tee -a $SUMMARY
1373     echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee -a $SUMMARY
1374 edhill 1.21 exit 1
1375 edhill 1.10 fi
1376 edhill 1.21 else
1377     echo >> $SUMMARY
1378 jmc 1.152 echo "No \"OPTFILE\" was specified ; genmake2 found and uses:" >> $SUMMARY
1379     #-note: to be filled later after 1rst run
1380 edhill 1.10 fi
1381     echo
1382     echo >> $SUMMARY
1383 jmc 1.175 if test $KIND = 0 ; then
1384     line_0=`printf '%s %2i' 'default' $MATCH_CRIT`
1385 jmc 1.94 line_0="$line_0 ----T----- ----S----- ----U----- ----V-----"
1386 edhill 1.49 line_1="G D M c m s m s m s m s"
1387 jmc 1.167 line_2="e p a R g m m e . m m e . m m e . m m e ."
1388     line_3="n n k u 2 i a a d i a a d i a a d i a a d"
1389 edhill 1.49 line_4="2 d e n d n x n . n x n . n x n . n x n ."
1390 edhill 1.50 for ii in $PTRACERS_NUM ; do
1391 edhill 1.49 line_0="$line_0 --PTR 0"$ii"--"
1392     line_1="$line_1 m s"
1393     line_2="$line_2 m m e ."
1394     line_3="$line_3 i a a d"
1395     line_4="$line_4 n x n ."
1396     done
1397 jmc 1.175 else
1398     line_0=`printf '%s %2i' 'default ' $MATCH_CRIT`
1399     if test $KIND = 1 ; then
1400 jmc 1.181 #echo "TANGLIN=true" >> $SUMMARY
1401     echo "TangLin generated by TAF" >> $SUMMARY
1402     elif test $KIND = 3 ; then
1403     echo "TangLin generated by OpenAD" >> $SUMMARY
1404     elif test $KIND = 2 ; then
1405     #echo "ADJOINT=true" >> $SUMMARY
1406     echo "Adjoint generated by TAF" >> $SUMMARY
1407     else
1408     echo "Adjoint generated by OpenAD" >> $SUMMARY
1409     fi
1410     if test $KIND = 1 -o $KIND = 3 ; then
1411 jmc 1.175 line_1="G D M C T F"
1412     line_2="e p a R o L D"
1413     else
1414     line_1="G D M C A F"
1415     line_2="e p a R o d D"
1416     fi
1417     line_3="n n k u s G G"
1418     line_4="2 d e n t r r"
1419     echo >> $SUMMARY
1420     fi
1421 jmc 1.178 if test "x$CLEANUP" != xt ; then
1422 edhill 1.49 echo "$line_0" | tee -a $SUMMARY
1423     echo "$line_1" | tee -a $SUMMARY
1424     echo "$line_2" | tee -a $SUMMARY
1425     echo "$line_3" | tee -a $SUMMARY
1426     echo "$line_4" | tee -a $SUMMARY
1427 jmc 1.175 echo "" | tee -a $SUMMARY
1428 jmc 1.178 fi
1429 jmc 1.139 echo "-------------------------------------------------------------------------------"
1430 edhill 1.1
1431 edhill 1.10 # ...and each test directory...
1432     for dir in $TESTDIRS ; do
1433 jmc 1.135
1434 jmc 1.106 # set builddir & rundir:
1435     builddir="build"
1436     if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi
1437     rundir="run"
1438 jmc 1.143 pfxdir="tr_$rundir"
1439 jmc 1.106 if test ! -d $dir/$rundir ; then
1440     rundir=$builddir
1441     fi
1442     CODE_DIR=$dir/$code_dir
1443     BUILD_DIR=$dir/$builddir
1444    
1445 edhill 1.10 # Cleanup only!
1446     if test "x$CLEANUP" = xt ; then
1447 jmc 1.127 echo -n ' --- dir:' $BUILD_DIR ': '
1448     makeclean $BUILD_DIR
1449 jmc 1.145 ( cd $BUILD_DIR
1450     rm -f $EXECUTABLE *.bak
1451 jmc 1.146 rm -f genmake_state genmake_*optfile genmake.log
1452 jmc 1.154 rm -f SIZE.h.mpi genmake.tr_log make.tr_log
1453 jmc 1.145 )
1454 jmc 1.106 if test -d $dir/$rundir/CVS ; then
1455 jmc 1.175 echo -n ' --- dir:' $dir/$rundir ': '
1456 jmc 1.106 run_clean $dir/$rundir
1457 edhill 1.10 fi
1458 jmc 1.143 trdir=`( cd $dir ; find . -type d -name "$pfxdir.*" -print | sed 's/^.\///')`
1459 jmc 1.127 ttd=`echo $trdir | wc -w`
1460     if test $ttd != 0 ; then
1461     echo ' --- rm dir:' $trdir
1462     ( cd $dir ; rm -rf $trdir )
1463     fi
1464 edhill 1.10 continue
1465 edhill 1.1 fi
1466 edhill 1.3
1467 jmc 1.135 # Verify that the testdir exists and contains previous
1468 edhill 1.10 # results in the correct location--or skip this directory!
1469 jmc 1.104 fout=$dir"/results/"$ref_outp
1470 jmc 1.175 #if test ! -r $fout ; then
1471     if test ! -r $fout -a ! -r ${fout}.gz ; then
1472 edhill 1.24 echo "can't read \"$fout\" -- skipping $dir"
1473 edhill 1.10 continue
1474     fi
1475 edhill 1.7
1476 jmc 1.117 # Check for specific files for particular type of run
1477 edhill 1.49
1478 jmc 1.161 if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" != "x0" ; then
1479     echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir"
1480     continue
1481     fi
1482     if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then
1483     echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir"
1484     continue
1485     fi
1486    
1487 jmc 1.154 if test "x$MPI" != "x0" ; then
1488 jmc 1.173 ntx=1 ; nty=1
1489 jmc 1.161 if test "x$MULTI_THREAD" = "xt" ; then
1490 jmc 1.174 ff=$dir"/input/eedata.mth"
1491     ntx=`grep "^ *nTx *=" $ff | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"`
1492     nty=`grep "^ *nTy *=" $ff | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"`
1493     if test "x$ntx" = x ; then ntx=1 ; fi
1494     if test "x$nty" = x ; then nty=1 ; fi
1495 jmc 1.161 fi
1496     #- create new SIZE.h with no more than '$MPI' Procs
1497 jmc 1.173 mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI $ntx $nty
1498 jmc 1.161 LOC_NPROC=$?
1499     ( cd $BUILD_DIR
1500     if test -r SIZE.h.mpi ; then
1501     cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$?
1502     else RETVAL=1
1503     fi
1504     if test "x$RETVAL" = x0 ; then
1505     rm -f tr_size.mpi
1506     else
1507     rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi
1508     fi
1509     )
1510 jmc 1.156 if test "x$MPI_MFILE" != x ; then
1511     #- create new MPI machine-file with the right number of Procs
1512     rm -f $LOC_MFILE
1513     cat $MPI_MFILE | sort | uniq | head -$LOC_NPROC > $LOC_MFILE
1514     nl=`wc -l $LOC_MFILE | awk '{print $1}'`
1515     if [ $nl -lt $LOC_NPROC ] ; then
1516     rm -f $LOC_MFILE
1517     cat $MPI_MFILE | head -$LOC_NPROC > $LOC_MFILE
1518     #sed -n "1,$LOC_NPROC p" $MPI_MFILE > $LOC_MFILE
1519     fi
1520     if [ $verbose -gt 1 ]; then
1521     nl=`wc -l $LOC_MFILE | awk '{print $1}'`
1522     echo " new LOC_MFILE=$LOC_MFILE : $nl procs for LOC_NPROC=$LOC_NPROC"
1523     fi
1524     fi
1525 jmc 1.161 if test "x$MULTI_THREAD" = "xt" ; then
1526     retv=`check_eedata $dir"/input/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"`
1527     if test $retv != 0 ; then
1528     echo "input/eedata.mth tiling misfit -- skipping $dir"
1529     continue
1530     fi
1531 jmc 1.154 fi
1532 jmc 1.123 fi
1533 jmc 1.83
1534     # Check whether there are "extra runs" for this testdir
1535     extra_runs=
1536 jmc 1.133 if test "x$NORUN" = xf ; then
1537 jmc 1.183 ex_run_dirs=`( cd $dir ; ls -d $inputdir.* 2> /dev/null )`
1538 jmc 1.133 fi
1539 jmc 1.108 #echo "ex_run_dirs='$ex_run_dirs'"
1540     for exd in $ex_run_dirs ; do
1541     name=`echo $exd | sed -e "s/$inputdir\.//"`
1542     refExOut=`echo $ref_outp | sed "s/\./.${name}./"`
1543     outf="$dir/results/$refExOut"
1544 jmc 1.175 if test -r $outf -o -r ${outf}.gz ; then
1545 jmc 1.135 if test "x$MULTI_THREAD" = "xt" ; then
1546 jmc 1.123 if test -r $dir"/"$exd"/eedata.mth" ; then
1547 jmc 1.154 if test "x$MPI" = "x0" ; then
1548     extra_runs="$extra_runs $name"
1549     else
1550 jmc 1.161 retv=`check_eedata $dir"/"$exd"/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"`
1551 jmc 1.123 if test $retv = 0 ; then
1552 jmc 1.83 extra_runs="$extra_runs $name"
1553 jmc 1.123 else
1554     echo $exd"/eedata.mth tiling misfit -- skipping $dir"
1555 jmc 1.83 fi
1556 jmc 1.123 fi
1557     #else echo $dir"/"$exd"/eedata.mth: not found"
1558 jmc 1.83 fi
1559 jmc 1.123 else
1560     extra_runs="$extra_runs $name"
1561     fi
1562 jmc 1.108 fi
1563     done
1564 edhill 1.28
1565     echo
1566 jmc 1.135 if test "x$extra_runs" = "x" ; then
1567 jmc 1.175 echo "Experiment: $dir"
1568 jmc 1.83 else
1569 jmc 1.175 echo "Experiment: $dir ; extra_runs=$extra_runs"
1570 jmc 1.83 fi
1571 edhill 1.28 echo
1572     unset genmake makedepend make run
1573 jmc 1.94 results=$EMPTY_RESULTS
1574 edhill 1.10
1575 jmc 1.158 # Create an output dir & summary.txt file for each tested experiment (tdir)
1576 jmc 1.138 locDIR=$DRESULTS"/"$dir
1577     mkdir $locDIR
1578 jmc 1.158 #- report to this experiment local summary file ---
1579     echo "DATE='$DATE' ; tdir='$dir'" > $locDIR"/summary.txt"
1580     echo "MACH='$MACH'" >> $locDIR"/summary.txt"
1581     echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
1582 jmc 1.138 CDIR=`pwd`"/$locDIR"
1583 jmc 1.135
1584 jmc 1.133 if test "x$NORUN" = xt ; then
1585     run=N
1586     genmakemodel $dir/$builddir && genmake=Y \
1587     && makeclean $dir/$builddir \
1588 jmc 1.175 && symlink_mpifiles $dir $code_dir $builddir \
1589 jmc 1.133 && makedependmodel $dir/$builddir && makedepend=Y \
1590     && makemodel $dir/$builddir && make=Y
1591 edhill 1.10 else
1592 edhill 1.1 genmakemodel $dir/$builddir && genmake=Y \
1593     && makeclean $dir/$builddir \
1594 jmc 1.175 && symlink_mpifiles $dir $code_dir $builddir \
1595 edhill 1.1 && makedependmodel $dir/$builddir && makedepend=Y \
1596     && makemodel $dir/$builddir && make=Y \
1597 jmc 1.139 && run_clean $dir/$rundir \
1598 jmc 1.140 && linkdata $dir/$rundir $inputdir \
1599 edhill 1.12 && runmodel $dir/$rundir && run=Y \
1600 jmc 1.99 && results=`testoutput_run $dir $rundir $ref_outp`
1601 edhill 1.10 fi
1602 jmc 1.139 #echo "results='$results'"
1603 jmc 1.135
1604 edhill 1.24 fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1605 jmc 1.139 echo 1>&2
1606 jmc 1.175 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1607 jmc 1.138 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
1608 edhill 1.34
1609     for ex in $extra_runs ; do
1610 jmc 1.73 unset run
1611 jmc 1.175 results=$EMPTY_RESULTS
1612 jmc 1.106 # reference output file
1613     refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"`
1614 jmc 1.158 # Create an output dir & summary.txt file for each extra run (tdir.ex)
1615 jmc 1.138 locDIR=$DRESULTS"/"$dir"."$ex
1616     mkdir $locDIR
1617 jmc 1.158 #- report to this experiment local summary file ---
1618     echo "DATE='$DATE' ; tdir='$dir.$ex'" > $locDIR"/summary.txt"
1619     #echo "MACH='$MACH'" >> $locDIR"/summary.txt"
1620     #echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt"
1621 jmc 1.138 CDIR=`pwd`"/$locDIR"
1622 jmc 1.143 test ! -e "$dir/$pfxdir.$ex" && mkdir "$dir/$pfxdir.$ex"
1623     run_clean $dir/$pfxdir.$ex
1624     linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir
1625     runmodel $dir/$pfxdir.$ex && run=Y \
1626     && results=`testoutput_run $dir $pfxdir.$ex $refExOut`
1627 jmc 1.186 fres=`formatresults $dir.$ex ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results`
1628 jmc 1.139 echo 1>&2
1629 jmc 1.100 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /</' >> $SUMMARY
1630 jmc 1.138 echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt"
1631 jmc 1.156 if test "x$POSTCLEAN" = x2 ; then
1632 jmc 1.143 run_clean $dir/$pfxdir.$ex
1633 jmc 1.83 fi
1634 edhill 1.34 done
1635 edhill 1.68
1636 jmc 1.176 if test ! -f $DRESULTS"/"genmake_state ; then
1637 jmc 1.137 if test -f $dir/$builddir/Makefile ; then
1638     mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
1639     echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state
1640     sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state
1641 jmc 1.151 echo " $mkOpt" >> $DRESULTS/genmake_state
1642 jmc 1.152 if test "x$OPTFILE" = xNONE ; then
1643     eval $mkOpt
1644     sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\
1645 jmc 1.176 $SUMMARY > tmp.tr_log
1646 jmc 1.152 RETVAL=$?
1647     if test "x$RETVAL" = x0 ; then rm -f $SUMMARY
1648 jmc 1.176 cp tmp.tr_log $SUMMARY
1649     else rm -f tmp.tr_log
1650 jmc 1.152 fi
1651     fi
1652 jmc 1.150 gmkLog=$dir/$builddir/genmake.log
1653 jmc 1.176 if test -r $gmkLog ; then
1654     grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1
1655     RETVAL=$?
1656     if test "x$RETVAL" = x0 ; then
1657     echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state
1658     echo "extract compiler version:" >> $DRESULTS/genmake_state
1659     sed -n '/Get compiler version/,/<-- compiler version/p' \
1660     $gmkLog | grep -v '^... compiler version ' > tmp.tr_log
1661     sed -n '1p' tmp.tr_log >> $DRESULTS/genmake_state
1662     sed -n '2,/^$/p' tmp.tr_log | sed '/^$/d' | sed 's/^./ &/' \
1663     >> $DRESULTS/genmake_state
1664     rm -f tmp.tr_log
1665     fi
1666     echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state
1667     echo "get NETCDF & LAPACK settings:" >> $DRESULTS/genmake_state
1668     sed -n '/set HAVE_NETCDF=/p' $gmkLog >> $DRESULTS/genmake_state
1669     sed -n '/set HAVE_LAPACK=/p' $gmkLog >> $DRESULTS/genmake_state
1670 jmc 1.150 fi
1671 jmc 1.137 fi
1672     fi
1673 jmc 1.83 #postclean $dir/$builddir
1674 jmc 1.156 if test "x$POSTCLEAN" = x2 ; then
1675 jmc 1.175 makeclean $dir/$builddir \
1676     && run_clean $dir/$rundir
1677 jmc 1.83 fi
1678 jmc 1.156 if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then rm -f $LOC_MFILE ; fi
1679 jmc 1.135
1680 edhill 1.10 echo "-------------------------------------------------------------------------------"
1681 jmc 1.135
1682 edhill 1.1 done
1683    
1684 edhill 1.34 printf "Start time: " >> $SUMMARY
1685 jmc 1.114 echo "$start_date" >> $SUMMARY
1686 edhill 1.34 printf "End time: " >> $SUMMARY
1687 edhill 1.13 date >> $SUMMARY
1688 edhill 1.20
1689     # If addresses were supplied and mpack built successfully, then try
1690     # to send email using mpack.
1691     if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then
1692     echo "No results email was sent."
1693     else
1694     if test "x$HAVE_MPACK" = xt ; then
1695     tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \
1696     && gzip $DRESULTS".tar" \
1697 edhill 1.44 && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES
1698 edhill 1.20 RETVAL=$?
1699     if test "x$RETVAL" != x0 ; then
1700     echo
1701     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
1702     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
1703     echo " summary of results from the directory \"$DRESULTS\"."
1704     echo
1705     else
1706     echo
1707     echo "An email containing results was sent to the following addresses:"
1708     echo " \"$ADDRESSES\""
1709     echo
1710 jmc 1.129 test -f $DRESULTS".tar" && rm -f $DRESULTS".tar"
1711     test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz"
1712 edhill 1.20 fi
1713     fi
1714     fi
1715 edhill 1.13
1716 jmc 1.133 if test "x$QUICK" = xf -a "x$NORUN" = xf ; then
1717     rm -f tr_cmpnum.c tr_cmpnum
1718     fi
1719 edhill 1.1
1720 edhill 1.12 if test "x$CLEANUP" != xt ; then
1721 jmc 1.102 cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //'
1722 edhill 1.25 if test -e tr_out.txt ; then
1723 jmc 1.175 mv tr_out.txt tr_out.txt.old
1724 edhill 1.14 fi
1725 jmc 1.103 cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt
1726 edhill 1.12 fi
1727 edhill 1.1
1728 edhill 1.40 if test "x$DELDIR" = xt ; then
1729     rm -rf $DRESULTS
1730     fi
1731 jmc 1.178 echo "======== End of testreport execution ========"

  ViewVC Help
Powered by ViewVC 1.1.22