1 |
#! /usr/bin/env bash |
2 |
|
3 |
# $Header: /u/gcmpack/MITgcm/tools/example_scripts/csail/test_baudelaire,v 1.36 2017/05/14 15:59:39 jmc Exp $ |
4 |
|
5 |
# Test script for MITgcm that should work on most of the csail.mit.edu Linux machines. |
6 |
|
7 |
tst_grp=0 ; if test $# = 1 ; then tst_grp=$1 ; fi |
8 |
if test $tst_grp != 'a' -a $tst_grp != 'b' ; then |
9 |
echo "missing or invalid argument (expect: 'a' or 'b') ==> exit" |
10 |
exit 1 |
11 |
fi |
12 |
|
13 |
# defaults |
14 |
#export PATH="$PATH:/usr/local/bin" |
15 |
if [ -d ~/bin ]; then export PATH=$PATH:~/bin ; fi |
16 |
#- to get case insensitive "ls" (and order of tested experiments) |
17 |
export LC_ALL="en_US.UTF-8" |
18 |
# Turn off stack limit for FIZHI & AD-tests |
19 |
ulimit -s unlimited |
20 |
# MPI test (for now, only with gfortran) |
21 |
export MPI_GCC_DIR=/srv/software/gcc/gcc-packages/gcc-4.4.5/mpich2/mpich2-1.3 |
22 |
export MPI_INC_DIR=$MPI_GCC_DIR/include |
23 |
export PATH="$PATH:$MPI_GCC_DIR/bin" |
24 |
|
25 |
#- method to acces CVS: |
26 |
# export CVSROOT='/u/gcmpack' |
27 |
# export CVSROOT=':ext:@mitgcm.org:/u/gcmpack' |
28 |
# export CVS_RSH='ssh' ; cvs co -P MITgcm > /dev/null |
29 |
# cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co -P MITgcm > /dev/null |
30 |
cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack' |
31 |
cmdCVS='cvs -d /u/gcmpack' |
32 |
|
33 |
# checkOut=2 : download new code ; |
34 |
# =3 : skip download but, if sepDir, use a new copy |
35 |
# =1 : update code (if no existing code -> swith to 2) |
36 |
# =0 : use existing code (if no existing code -> swith to 2) |
37 |
dInWeek=`date +%a` |
38 |
|
39 |
dNam=`hostname -s | tr '[:upper:]' '[:lower:]'` |
40 |
TESTDIR="/scratch/jmc/test_${dNam}" |
41 |
outDir="${dNam}-${tst_grp}" |
42 |
#outDir=`hostname -s` |
43 |
MC=13 |
44 |
sepDir=1 |
45 |
option= |
46 |
|
47 |
#tst_list='g7a adm mpa g77 gfo+rs mth mp2+rs mpi ifc' |
48 |
#if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm oad" ; fi |
49 |
if test $tst_grp = 'a' ; then |
50 |
checkOut=2 |
51 |
tst_list='g7a mpa mth mp2+rs mpi oad' |
52 |
else |
53 |
checkOut=3 |
54 |
tst_list='adm g77 gfo+rs ifc' |
55 |
if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm" ; fi |
56 |
fi |
57 |
echo " test: $outDir ; list='$tst_list'" |
58 |
|
59 |
#option="-nc" ; checkOut=1 |
60 |
#option="-q" ; checkOut=1 |
61 |
|
62 |
TODAY=`date +%d` |
63 |
tdir=$TESTDIR |
64 |
updFile='updated_code' ; today=`date +%Y%m%d` |
65 |
if test $checkOut != 2 ; then |
66 |
if test -e $tdir/MITgcm/CVS ; then |
67 |
echo $tdir/MITgcm/CVS 'exist' |
68 |
if test $sepDir = 0 -a $checkOut = 1 ; then |
69 |
#- remove date/lock-file: |
70 |
if test -f $tdir/$updFile ; then rm -f $tdir/$updFile ; sleep 2 ; fi |
71 |
echo -n "Update the MITgcm code using: $cmdCVS ..." |
72 |
cd $tdir/MITgcm |
73 |
$cmdCVS update -P -d |
74 |
echo " done" |
75 |
#- update date/lock-file: |
76 |
( cd $tdir ; echo $today > $updFile |
77 |
sleep 2 ; ls -l $updFile ) |
78 |
fi |
79 |
else |
80 |
echo -n $tdir/MITgcm 'missing ; ' |
81 |
checkOut=2 |
82 |
fi |
83 |
fi |
84 |
if test $checkOut = 2 ; then |
85 |
if test -e $tdir ; then |
86 |
echo -n "Removing working copy: $tdir/MITgcm ..." |
87 |
test -e $tdir/MITgcm && rm -rf $tdir/MITgcm |
88 |
else |
89 |
echo -n "Creating a working dir: $tdir ..." |
90 |
mkdir $tdir |
91 |
fi |
92 |
echo " done" |
93 |
cd $tdir |
94 |
#- remove date/lock-file: |
95 |
if test -f $updFile ; then rm -f $updFile ; sleep 2 ; fi |
96 |
echo -n "Downloading the MITgcm code using: $cmdCVS ..." |
97 |
$cmdCVS co -P MITgcm > /dev/null |
98 |
echo " done" |
99 |
if test -d other_input ; then |
100 |
list_dirs=`(cd other_input ; ls 2> /dev/null )` |
101 |
echo "Updating ( $cmdCVS update -P -d ) 'other_input' extra dirs:" |
102 |
for exd in $list_dirs ; do |
103 |
if test -d other_input/$exd/CVS ; then echo " $exd" |
104 |
( cd other_input/$exd ; $cmdCVS update -P -d ) |
105 |
fi |
106 |
done |
107 |
echo " <-- update of 'other_input' dirs done" |
108 |
fi |
109 |
#- update date/lock-file: |
110 |
echo $today > $updFile ; sleep 2 ; ls -l $updFile |
111 |
else |
112 |
cd $tdir |
113 |
fi |
114 |
|
115 |
#------------------------------------------------------------------------ |
116 |
|
117 |
firstTst=`echo $tst_list | awk '{print $1}'` |
118 |
last_Tst=`echo $tst_list | awk '{print $NF}'` |
119 |
for tt in $tst_list |
120 |
do |
121 |
|
122 |
echo "================================================================" |
123 |
typ=`echo $tt | sed 's/+rs//'` |
124 |
#- define list of additional experiences to test: |
125 |
addExp='' |
126 |
if test $typ = 'mp2' -o $typ = 'ifc' ; then |
127 |
addExp="offline_cheapaml atm_gray" |
128 |
fi |
129 |
if test $typ = 'gfo' -o $typ = 'ifc' ; then |
130 |
addExp="$addExp global_oce_biogeo_bling" |
131 |
addExp="$addExp shelfice_remeshing" |
132 |
fi |
133 |
if test $typ = 'mpi' ; then |
134 |
addExp="$addExp global_oce_cs32" |
135 |
fi |
136 |
#- check day and time: |
137 |
curDay=`date +%d` ; curHour=`date +%H` |
138 |
if [ $curDay -ne $TODAY ] ; then |
139 |
date ; echo "day is over => skip test $typ" |
140 |
continue |
141 |
fi |
142 |
if [ $curHour -ge 18 ] ; then |
143 |
date ; echo "too late to run test $typ" |
144 |
continue |
145 |
fi |
146 |
if test $sepDir = 0 -a "x$option" != x -a $tt != $last_Tst ; then |
147 |
echo "using option='$option' prevent multi-tests => skip test $typ" |
148 |
continue |
149 |
fi |
150 |
#- clean-up old output files |
151 |
rm -f $tdir/output_${typ}* |
152 |
touch $tdir/output_$tt |
153 |
if test $sepDir = 1 ; then |
154 |
new_dir="MITgcm_$typ" |
155 |
reUse=1 ; if [ $checkOut -ge 2 ] ; then reUse=0 ; fi |
156 |
if test -d $new_dir/CVS -a $reUse = 1 ; then |
157 |
cd $tdir/$new_dir |
158 |
if test $tt != $typ ; then |
159 |
( cd verification ; ../tools/do_tst_2+2 -clean ) |
160 |
fi |
161 |
if test $checkOut = 1 ; then |
162 |
echo -n "Update the MITgcm code using: $cmdCVS ..." |
163 |
$cmdCVS update -P -d |
164 |
echo " done" |
165 |
fi |
166 |
else |
167 |
if test -d prev ; then |
168 |
#-- save previous summary: |
169 |
oldS=`ls -t ${new_dir}/verification/tr_${outDir}_*/summary.txt 2> /dev/null | head -1` |
170 |
if test "x$oldS" != x ; then |
171 |
cat $oldS | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > prev/tr_out.$typ |
172 |
touch -r $oldS prev/tr_out.$typ |
173 |
fi |
174 |
if test $tt != $typ ; then |
175 |
oldS=`ls -t ${new_dir}/verification/rs_${outDir}_*/summary.txt 2> /dev/null | head -1` |
176 |
if test "x$oldS" != x ; then cp -p -f $oldS prev/rs_out.$typ ; fi |
177 |
fi |
178 |
fi |
179 |
test -e $new_dir && rm -rf $new_dir |
180 |
mkdir $new_dir |
181 |
#- before making a copy, check that code has been updated |
182 |
nCount=0; today=`date +%Y%m%d` |
183 |
updDate=0 ; test -f $updFile && updDate=`cat $updFile` |
184 |
while [ $today -gt $updDate ] ; do |
185 |
nCount=`expr $nCount + 1` |
186 |
if [ $nCount -gt 40 ] ; then |
187 |
echo " waiting too long (nCount=$nCount) for updated code" |
188 |
echo " today=$today , updDate=$updDate " |
189 |
ls -l $updFile |
190 |
exit |
191 |
fi |
192 |
sleep 60 |
193 |
updDate=0 ; test -f $updFile && updDate=`cat $updFile` |
194 |
done |
195 |
ls -l $updFile | tee -a $tdir/output_$tt |
196 |
echo " waited nCount=$nCount for updated code ($updDate) to copy" | tee -a $tdir/output_$tt |
197 |
#----------------------------- |
198 |
if test -d MITgcm -a -d $new_dir ; then |
199 |
echo " copy main code from MITgcm to $new_dir" | tee -a $tdir/output_$tt |
200 |
cp -ra MITgcm/* $new_dir 2>&1 | tee -a $tdir/output_$tt |
201 |
else |
202 |
if test -d MITgcm ; then |
203 |
echo " missing dir $new_dir --> end test $tt" | tee -a $tdir/output_$tt |
204 |
else |
205 |
echo " missing dir MITgcm --> end test $tt" | tee -a $tdir/output_$tt |
206 |
fi |
207 |
continue |
208 |
fi |
209 |
echo -n " cd $tdir/$new_dir " | tee -a $tdir/output_$tt |
210 |
cd $tdir/$new_dir |
211 |
retVal=$? |
212 |
( echo "(retVal= $retVal )" ; pwd ) | tee -a $tdir/output_$tt |
213 |
if test -d verification ; then |
214 |
echo " check: dir verification exist" | tee -a $tdir/output_$tt |
215 |
else |
216 |
echo " missing dir verification --> end test $tt" | tee -a $tdir/output_$tt |
217 |
continue |
218 |
fi |
219 |
#-- download additional experience from Contrib: |
220 |
for exp2add in $addExp ; do |
221 |
echo " add dir: $exp2add (from Contrib:verification_other)" |
222 |
( cd verification ; $cmdCVS co -P -d $exp2add \ |
223 |
MITgcm_contrib/verification_other/$exp2add > /dev/null ) |
224 |
if test $exp2add = 'global_oce_cs32' ; then |
225 |
echo " link dir 'other_input/core2_cnyf' in here" |
226 |
( cd verification/${exp2add} |
227 |
ln -s ../../../other_input/core2_cnyf . ) |
228 |
fi |
229 |
if test $exp2add = 'shelfice_remeshing' ; then |
230 |
echo " link dir 'other_input/remeshing_code' to 'extra_code'" |
231 |
( cd verification/${exp2add} |
232 |
ln -s ../../../other_input/remeshing_code extra_code ) |
233 |
fi |
234 |
done |
235 |
fi |
236 |
else |
237 |
cd $tdir/MITgcm |
238 |
fi |
239 |
cd verification |
240 |
|
241 |
#-- set the testreport command: |
242 |
comm="./testreport" |
243 |
if test $typ = 'g7a' -o $typ = 'adm' -o $typ = 'mpa' ; then |
244 |
comm="$comm -adm" |
245 |
elif test $typ = 'oad' ; then |
246 |
comm="$comm -oad" |
247 |
elif test $typ = 'tlm' ; then |
248 |
comm="$comm -tlm" |
249 |
elif test $typ = 'mth' -o $typ = 'mp2' ; then |
250 |
export GOMP_STACKSIZE=400m |
251 |
export OMP_NUM_THREADS=2 |
252 |
comm="$comm -mth" |
253 |
else |
254 |
comm="$comm -md cyrus-makedepend" |
255 |
fi |
256 |
comm="$comm -odir $outDir -a jm_c@mitgcm.org" |
257 |
#-- set the optfile (+ mpi & match-precision) |
258 |
MPI=0 |
259 |
case $typ in |
260 |
'g77'|'g7a') OPTFILE='../tools/build_options/linux_amd64_g77' ;; |
261 |
'gfo'|'adm'|'oad'|'tlm'|'mth') comm="$comm -match $MC -devel" |
262 |
OPTFILE='../tools/build_options/linux_amd64_gfortran' ;; |
263 |
'ifc') comm="$comm -devel" |
264 |
OPTFILE='../tools/build_options/linux_amd64_ifort11' ;; |
265 |
'pgi') OPTFILE='../tools/build_options/linux_amd64_pgf77' ;; |
266 |
'mpa'|'mpi'|'mp2') comm="$comm -match $MC -devel" ; MPI=6 |
267 |
OPTFILE='../tools/build_options/linux_amd64_gfortran' ;; |
268 |
*) OPTFILE= ;; |
269 |
esac |
270 |
#-- set MPI command: |
271 |
if test $MPI != 0 ; then |
272 |
if test $typ = 'mp2' ; then MPI=3 ; fi |
273 |
if test $typ = 'mpa' ; then |
274 |
EXE="mpirun -np TR_NPROC ./mitgcmuv_ad" |
275 |
else |
276 |
EXE="mpirun -np TR_NPROC ./mitgcmuv" |
277 |
fi |
278 |
fi |
279 |
|
280 |
#-- set specific Env Vars: |
281 |
if test $typ = 'oad' ; then |
282 |
source ~jmc/mitgcm/bin/setenv_OpenAD.sh |
283 |
fi |
284 |
if test $typ = 'ifc' ; then |
285 |
source /srv/software/intel/intel-11.1.073/bin/ifortvars.sh intel64 |
286 |
fi |
287 |
if test $typ = 'pgi' ; then |
288 |
#listT='fizhi-cs-32x32x40 fizhi-cs-aqualev20' |
289 |
export PGI=/srv/software/pgi/pgi-10.9 |
290 |
export PATH="$PATH:$PGI/linux86-64/10.9/bin" |
291 |
export LM_LICENSE_FILE=$PGI/license.dat |
292 |
fi |
293 |
|
294 |
if test $sepDir = 0 -a "x$option" = x -a $tt = $firstTst -a $checkOut != 2 ; then |
295 |
#-- cleaning: |
296 |
echo "======================" |
297 |
echo "Cleaning test directories:" | tee -a $tdir/output_$tt |
298 |
cmdCLN="./testreport -clean" |
299 |
echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt |
300 |
$cmdCLN >> $tdir/output_$tt 2>&1 |
301 |
echo "======================" |
302 |
echo "" | tee -a $tdir/output_$tt |
303 |
fi |
304 |
|
305 |
#-- run the testreport command: |
306 |
echo -n "Running testreport using:" | tee -a $tdir/output_$tt |
307 |
if test "x$OPTFILE" != x ; then |
308 |
comm="$comm -of=$OPTFILE" |
309 |
fi |
310 |
if test $MPI = 0 ; then echo '' | tee -a $tdir/output_$tt |
311 |
else echo " (EXE='$EXE')" | tee -a $tdir/output_$tt |
312 |
comm="$comm -MPI $MPI -command \"\$EXE\"" |
313 |
fi |
314 |
if test "x$option" != x ; then comm="$comm $option" ; fi |
315 |
#if test $typ = 'pgi' ; then comm="$comm -t \"\$listT\"" ; fi |
316 |
echo " \"eval $comm\"" | tee -a $tdir/output_$tt |
317 |
echo "======================" |
318 |
eval $comm >> $tdir/output_$tt 2>&1 |
319 |
sed -n "/^An email /,/^======== End of testreport / p" $tdir/output_$tt |
320 |
echo "" | tee -a $tdir/output_$tt |
321 |
|
322 |
#-- also test restart (test 2+2=4) |
323 |
if test $tt != $typ |
324 |
then |
325 |
echo "testing restart using:" | tee -a $tdir/output_$tt |
326 |
comm="../tools/do_tst_2+2 -o $outDir -a jm_c@mitgcm.org" |
327 |
if test $MPI = 0 ; then |
328 |
echo " \"$comm\"" | tee -a $tdir/output_$tt |
329 |
echo "======================" |
330 |
$comm >> $tdir/output_$tt 2>&1 |
331 |
else |
332 |
echo " \"$comm -mpi -exe $EXE\"" | tee -a $tdir/output_$tt |
333 |
echo "======================" |
334 |
$comm -mpi -exe "$EXE" >> $tdir/output_$tt 2>&1 |
335 |
fi |
336 |
echo ; cat tst_2+2_out.txt |
337 |
echo |
338 |
fi |
339 |
export OMP_NUM_THREADS=1 |
340 |
|
341 |
if test $sepDir = 0 ; then |
342 |
#-- cleaning: |
343 |
echo "======================" |
344 |
echo "Cleaning test directories:" | tee -a $tdir/output_$tt |
345 |
if test $tt != $typ ; then |
346 |
cmdCLN="../tools/do_tst_2+2 -clean" |
347 |
echo " clean tst_2+2 running: $cmdCLN" | tee -a $tdir/output_$tt |
348 |
$cmdCLN >> $tdir/output_$tt 2>&1 |
349 |
fi |
350 |
if test $tt != $last_Tst ; then |
351 |
cmdCLN="./testreport -clean" |
352 |
echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt |
353 |
$cmdCLN >> $tdir/output_$tt 2>&1 |
354 |
fi |
355 |
echo "======================" |
356 |
echo |
357 |
fi |
358 |
cd $tdir |
359 |
|
360 |
done |