1 |
#!/bin/bash |
2 |
#SBATCH -J trintel |
3 |
#SBATCH -p sched_mit_darwin |
4 |
#SBATCH --mem-per-cpu 4000 |
5 |
#SBATCH -n 6 |
6 |
#SBATCH -N 2 |
7 |
NSLOTS=$SLURM_NTASKS |
8 |
echo '********************************************************************************' |
9 |
THEDATE=`date` |
10 |
echo 'Start job '$THEDATE |
11 |
echo 'NSLOTS = '$NSLOTS |
12 |
echo '======= NODELIST ===============================================================' |
13 |
echo $SLURM_NODELIST |
14 |
echo '======= env ====================================================================' |
15 |
env |
16 |
echo '================================================================================' |
17 |
|
18 |
source /etc/profile.d/modules.sh |
19 |
module add engaging/intel/2013.1.046 |
20 |
|
21 |
# to make sort order case-insensitive |
22 |
export LC_ALL=en_US.iso885915 |
23 |
|
24 |
doieee=0 |
25 |
dompi=0 |
26 |
dompidevel=1 |
27 |
dompifast=1 |
28 |
dodarwin=0 |
29 |
dodarwin2=0 |
30 |
dodarwin2mpi=0 |
31 |
|
32 |
dofast=0 |
33 |
dodarwinmpi=0 |
34 |
|
35 |
dirpre=engaging1-ifort |
36 |
pre=ifort |
37 |
defopts="-j 7" |
38 |
#cleanopts="-postclean" |
39 |
cleanopts= |
40 |
trmpiopts="-MPI $NSLOTS" |
41 |
optfile="../tools/build_options/linux_amd64_ifort+impi" |
42 |
mpioptfile="../tools/build_options/linux_amd64_ifort+impi" |
43 |
mpicommand="mpirun -env I_MPI_DEBUG 2 -n TR_NPROC ./mitgcmuv" |
44 |
|
45 |
# NONE will not send email, but keep rs_* directory |
46 |
#mailrcpt="-a NONE" |
47 |
mailrcpt="-a jmc@mitgcm.org" |
48 |
mailfw="$mailrcpt" |
49 |
mailrs="$mailrcpt" |
50 |
maildf="jahn@mitgcm.org" |
51 |
mailhost=eofe4 |
52 |
|
53 |
# relative to MITgcm/verification |
54 |
refdir=../../ref |
55 |
|
56 |
cpmodel() { |
57 |
newgcmDIR="$1" |
58 |
if test -e $newgcmDIR ; then |
59 |
rm -rf $newgcmDIR |
60 |
fi |
61 |
cp -a MITgcm $newgcmDIR |
62 |
} |
63 |
|
64 |
cpmodel1() { |
65 |
newgcmDIR="$1" |
66 |
if test -e $newgcmDIR ; then |
67 |
rm -rf $newgcmDIR |
68 |
fi |
69 |
cp -a MITgcm_darwin1 $newgcmDIR |
70 |
} |
71 |
|
72 |
cpmodel2() { |
73 |
newgcmDIR="$1" |
74 |
if test -e $newgcmDIR ; then |
75 |
rm -rf $newgcmDIR |
76 |
fi |
77 |
cp -a MITgcm_darwin2 $newgcmDIR |
78 |
} |
79 |
|
80 |
maildiff() { |
81 |
summaryref="$1" |
82 |
odir="$2" |
83 |
subject="$3" |
84 |
npassref=`grep -c -e ' pass ' $summaryref` |
85 |
ntotref=`grep -c -e ' \(pass\|FAIL\|N/O\) ' $summaryref` |
86 |
summary=`pwd`/`ls tr_${odir}_*/summary.txt` |
87 |
npass=`grep -c -e ' pass ' $summary` |
88 |
ntot=`grep -c -e ' \(pass\|FAIL\|N/O\) ' $summary` |
89 |
echo "npass=$npass" |
90 |
echo "ntot=$ntot" |
91 |
diff -I time: -I 'on :' -I run: $summaryref $summary | grep -v -e 'run:' -e 'on :' > diff.out |
92 |
difflen=`cat diff.out | wc -l` |
93 |
ssh $mailhost "cat $summary `pwd`/diff.out | mail -s '"$subject$odir" "$npass"/"$ntot" ["$npassref"/"$ntotref"] "$difflen"' $maildf" |
94 |
} |
95 |
|
96 |
maildiff2p2() { |
97 |
summaryref="$1" |
98 |
odir="$2" |
99 |
subject="$3" |
100 |
npassref=`grep -c -e ' pass ' $summaryref` |
101 |
ntotref=`grep -c -e ' \(pass\|FAIL\|N/O\) ' $summaryref` |
102 |
summary=`pwd`/`ls rs_${odir}_*/summary.txt` |
103 |
npass=`grep -c -e ' pass ' $summary` |
104 |
ntot=`grep -c -e ' \(pass\|FAIL\|N/O\) ' $summary` |
105 |
echo "npass=$npass" |
106 |
echo "ntot=$ntot" |
107 |
diff -I 'End time:' -I 'on :' -I run: "$summaryref" $summary | grep -v -e 'run:' -e 'on :' > diff.out |
108 |
difflen=`cat diff.out | wc -l` |
109 |
ssh $mailhost "cat $summary `pwd`/diff.out | mail -s '"$subject$odir" 2+2 "$npass"/"$ntot" ["$npassref"/"$ntotref"] "$difflen"' $maildf" |
110 |
} |
111 |
|
112 |
######################################################################## |
113 |
if test $dompi -ne 0; then |
114 |
|
115 |
sfx=mpi |
116 |
dirsfx=$pre-$sfx |
117 |
odir="$dirpre-$sfx" |
118 |
|
119 |
echo '--------------------------------------------------------------------------------' |
120 |
echo "$sfx: MITgcm_$dirsfx $odir" |
121 |
|
122 |
cpmodel MITgcm_$dirsfx |
123 |
|
124 |
cd MITgcm_$dirsfx/verification |
125 |
|
126 |
echo ./testreport $defopts $trmpiopts -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw |
127 |
./testreport $defopts $trmpiopts -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw |
128 |
|
129 |
maildiff "$refdir/summary_$dirsfx.txt" "$odir" |
130 |
|
131 |
echo ../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
132 |
../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
133 |
|
134 |
maildiff2p2 "$refdir/summary_2+2_$dirsfx.txt" "$odir" |
135 |
|
136 |
for d in */build; do make -C $d Clean; done |
137 |
|
138 |
cd ../.. |
139 |
|
140 |
THEDATE=`date` |
141 |
echo '--------------------------------------------------------------------------------' |
142 |
echo "End $dirsfx $THEDATE" |
143 |
echo '--------------------------------------------------------------------------------' |
144 |
|
145 |
fi |
146 |
|
147 |
######################################################################## |
148 |
if test $doieee -ne 0; then |
149 |
|
150 |
sfx= |
151 |
dirsfx=$pre |
152 |
odir="$dirpre" |
153 |
|
154 |
echo '--------------------------------------------------------------------------------' |
155 |
echo "$sfx: MITgcm_$dirsfx $odir" |
156 |
|
157 |
cpmodel MITgcm_$dirsfx |
158 |
|
159 |
cd MITgcm_$dirsfx/verification |
160 |
|
161 |
echo ./testreport $defopts -optfile "$optfile" -odir "$odir" $mailfw |
162 |
./testreport $defopts $cleanopts -optfile "$optfile" -odir "$odir" $mailfw |
163 |
|
164 |
maildiff "$refdir/summary_$dirsfx.txt" "$odir" |
165 |
|
166 |
cd ../.. |
167 |
|
168 |
THEDATE=`date` |
169 |
echo '--------------------------------------------------------------------------------' |
170 |
echo "End $dirsfx $THEDATE" |
171 |
echo '--------------------------------------------------------------------------------' |
172 |
|
173 |
fi |
174 |
|
175 |
######################################################################## |
176 |
if test $dompidevel -ne 0; then |
177 |
|
178 |
sfx=mpi-dvlp |
179 |
options="$defopts $trmpiopts -devel" |
180 |
dirsfx=$pre-$sfx |
181 |
odir="$dirpre-dvlp" |
182 |
|
183 |
echo '--------------------------------------------------------------------------------' |
184 |
echo "$sfx: MITgcm_$dirsfx $odir" |
185 |
|
186 |
cpmodel MITgcm_$dirsfx |
187 |
|
188 |
cd MITgcm_$dirsfx/verification |
189 |
|
190 |
echo ./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw |
191 |
./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw |
192 |
|
193 |
maildiff "$refdir/summary_$dirsfx.txt" "$odir" |
194 |
|
195 |
echo ../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
196 |
../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
197 |
|
198 |
maildiff2p2 "$refdir/summary_2+2_$dirsfx.txt" "$odir" |
199 |
|
200 |
for d in */build; do make -C $d Clean; done |
201 |
|
202 |
cd ../.. |
203 |
|
204 |
THEDATE=`date` |
205 |
echo '--------------------------------------------------------------------------------' |
206 |
echo "End $dirsfx $THEDATE" |
207 |
echo '--------------------------------------------------------------------------------' |
208 |
|
209 |
fi |
210 |
|
211 |
######################################################################## |
212 |
if test $dompifast -ne 0; then |
213 |
|
214 |
sfx=mpi-fast |
215 |
options="$defopts $trmpiopts -fast" |
216 |
dirsfx=$pre-$sfx |
217 |
odir="$dirpre-fast" |
218 |
|
219 |
echo '--------------------------------------------------------------------------------' |
220 |
echo "$sfx: MITgcm_$dirsfx $odir" |
221 |
|
222 |
cpmodel MITgcm_$dirsfx |
223 |
|
224 |
cd MITgcm_$dirsfx/verification |
225 |
|
226 |
echo ./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw |
227 |
./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw |
228 |
|
229 |
maildiff "$refdir/summary_$dirsfx.txt" "$odir" |
230 |
|
231 |
echo ../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
232 |
../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
233 |
|
234 |
maildiff2p2 "$refdir/summary_2+2_$dirsfx.txt" "$odir" |
235 |
|
236 |
for d in */build; do make -C $d Clean; done |
237 |
|
238 |
cd ../.. |
239 |
|
240 |
THEDATE=`date` |
241 |
echo '--------------------------------------------------------------------------------' |
242 |
echo "End $dirsfx $THEDATE" |
243 |
echo '--------------------------------------------------------------------------------' |
244 |
|
245 |
fi |
246 |
|
247 |
######################################################################## |
248 |
if test $dofast -ne 0; then |
249 |
|
250 |
sfx=fast |
251 |
|
252 |
options="$defopts -fast $cleanopts" |
253 |
dirsfx=$pre-$sfx |
254 |
odir="$dirpre-$sfx" |
255 |
|
256 |
echo '--------------------------------------------------------------------------------' |
257 |
echo "$sfx: MITgcm_$dirsfx $odir" |
258 |
|
259 |
cpmodel MITgcm_$dirsfx |
260 |
|
261 |
cd MITgcm_$dirsfx/verification |
262 |
|
263 |
echo ./testreport $options -optfile "$optfile" -odir "$odir" $mailfw |
264 |
./testreport $options -optfile "$optfile" -odir "$odir" $mailfw |
265 |
|
266 |
maildiff "$refdir/summary_$dirsfx.txt" "$odir" |
267 |
|
268 |
cd ../.. |
269 |
|
270 |
THEDATE=`date` |
271 |
echo '--------------------------------------------------------------------------------' |
272 |
echo "End $dirsfx $THEDATE" |
273 |
echo '--------------------------------------------------------------------------------' |
274 |
|
275 |
fi |
276 |
|
277 |
######################################################################## |
278 |
if test $dodarwinmpi -ne 0; then |
279 |
|
280 |
sfx=mpi |
281 |
|
282 |
options="$defopts $trmpiopts $cleanopts" |
283 |
dirsfx=$pre-$sfx |
284 |
odir="$dirpre-darwin-$sfx" |
285 |
|
286 |
echo '--------------------------------------------------------------------------------' |
287 |
echo "$sfx: MITgcm_$dirsfx $odir" |
288 |
|
289 |
gcmDIR="MITgcm_darwin_$dirsfx" |
290 |
cpmodel $gcmDIR |
291 |
|
292 |
cd $gcmDIR/verification |
293 |
|
294 |
exps="`ls -d darwin_*`" |
295 |
|
296 |
echo ./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw -t "$exps" |
297 |
./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw -t "$exps" |
298 |
|
299 |
maildiff "$refdir/summary_darwin_$dirsfx.txt" "$odir" |
300 |
|
301 |
#echo ../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
302 |
#../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
303 |
# |
304 |
#maildiff2p2 "$refdir/summary_2+2_$dirsfx.txt" "$odir" |
305 |
|
306 |
cd ../.. |
307 |
|
308 |
THEDATE=`date` |
309 |
echo '--------------------------------------------------------------------------------' |
310 |
echo "End $dirsfx $THEDATE" |
311 |
echo '--------------------------------------------------------------------------------' |
312 |
|
313 |
fi |
314 |
|
315 |
######################################################################## |
316 |
if test $dodarwin -ne 0; then |
317 |
|
318 |
sfx= |
319 |
|
320 |
options="$defopts" |
321 |
dirsfx=$pre |
322 |
odir="$dirpre-darwin" |
323 |
|
324 |
echo '--------------------------------------------------------------------------------' |
325 |
echo "$sfx: MITgcm_$dirsfx $odir" |
326 |
|
327 |
gcmDIR="MITgcm_darwin_$dirsfx" |
328 |
cpmodel1 $gcmDIR |
329 |
|
330 |
cd $gcmDIR/verification |
331 |
|
332 |
exps="`ls -d darwin_*`" |
333 |
|
334 |
echo ./testreport $options -optfile "$optfile" -odir "$odir" $mailfw -t "$exps" |
335 |
./testreport $options -optfile "$optfile" -odir "$odir" $mailfw -t "$exps" |
336 |
|
337 |
maildiff "$refdir/summary_darwin_$dirsfx.txt" "$odir" |
338 |
|
339 |
echo ../tools/do_tst_2+2 -o "$odir" $mailrs |
340 |
../tools/do_tst_2+2 -o "$odir" $mailrs |
341 |
|
342 |
maildiff2p2 "$refdir/summary_2+2_darwin_$dirsfx.txt" "$odir" |
343 |
|
344 |
for d in */build; do make -C $d Clean; done |
345 |
|
346 |
cd ../.. |
347 |
|
348 |
THEDATE=`date` |
349 |
echo '--------------------------------------------------------------------------------' |
350 |
echo 'End job '$THEDATE |
351 |
echo '--------------------------------------------------------------------------------' |
352 |
|
353 |
fi |
354 |
|
355 |
######################################################################## |
356 |
if test $dodarwin2mpi -ne 0; then |
357 |
|
358 |
sfx=mpi |
359 |
|
360 |
options="$defopts $trmpiopts $cleanopts" |
361 |
dirsfx=$pre-$sfx |
362 |
odir="$dirpre-darwin2-$sfx" |
363 |
|
364 |
echo '--------------------------------------------------------------------------------' |
365 |
echo "$sfx: MITgcm_$dirsfx $odir" |
366 |
|
367 |
gcmDIR="MITgcm_darwin2_$dirsfx" |
368 |
cpmodel2 $gcmDIR |
369 |
|
370 |
cd $gcmDIR/verification |
371 |
|
372 |
exps="`ls -d monod_* quota_*`" |
373 |
|
374 |
echo ./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw -t "$exps" |
375 |
./testreport $options -optfile "$mpioptfile" -command "$mpicommand" -odir "$odir" $mailfw -t "$exps" |
376 |
|
377 |
#maildiff "$refdir/summary_darwin2_$dirsfx.txt" "$odir" "darwin2 " |
378 |
maildiff "$refdir/summary_darwin2_$dirsfx.txt" "$odir" |
379 |
|
380 |
#echo ../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
381 |
#../tools/do_tst_2+2 -mpi -exe "$mpicommand" -o "$odir" $mailrs |
382 |
# |
383 |
#maildiff2p2 "$refdir/summary_2+2_$dirsfx.txt" "$odir" |
384 |
|
385 |
cd ../.. |
386 |
|
387 |
THEDATE=`date` |
388 |
echo '--------------------------------------------------------------------------------' |
389 |
echo "End $dirsfx $THEDATE" |
390 |
echo '--------------------------------------------------------------------------------' |
391 |
|
392 |
fi |
393 |
|
394 |
######################################################################## |
395 |
if test $dodarwin2 -ne 0; then |
396 |
|
397 |
sfx= |
398 |
|
399 |
options="$defopts" |
400 |
dirsfx=$pre |
401 |
odir="$dirpre-darwin2" |
402 |
|
403 |
echo '--------------------------------------------------------------------------------' |
404 |
echo "$sfx: MITgcm_$dirsfx $odir" |
405 |
|
406 |
gcmDIR="MITgcm_darwin2_$dirsfx" |
407 |
cpmodel2 $gcmDIR |
408 |
|
409 |
cd $gcmDIR/verification |
410 |
|
411 |
exps="`ls -d monod_* quota_*`" |
412 |
|
413 |
echo ./testreport $options -optfile "$optfile" -odir "$odir" $mailfw -t "$exps" |
414 |
./testreport $options -optfile "$optfile" -odir "$odir" $mailfw -t "$exps" |
415 |
|
416 |
maildiff "$refdir/summary_darwin2_$dirsfx.txt" "$odir" |
417 |
|
418 |
echo ../tools/do_tst_2+2 -o "$odir" $mailrs |
419 |
../tools/do_tst_2+2 -o "$odir" $mailrs |
420 |
|
421 |
maildiff2p2 "$refdir/summary_2+2_darwin2_$dirsfx.txt" "$odir" |
422 |
|
423 |
for d in */build; do make -C $d Clean; done |
424 |
|
425 |
cd ../.. |
426 |
|
427 |
fi |
428 |
|
429 |
THEDATE=`date` |
430 |
echo '================================================================================' |
431 |
echo 'End job '$THEDATE |
432 |
echo '********************************************************************************' |