/[MITgcm]/MITgcm/tools/example_scripts/csail/test_baudelaire
ViewVC logotype

Annotation of /MITgcm/tools/example_scripts/csail/test_baudelaire

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.38 - (hide annotations) (download)
Mon Jan 29 21:59:01 2018 UTC (6 years, 3 months ago) by jmc
Branch: MAIN
CVS Tags: checkpoint66o, HEAD
Changes since 1.37: +38 -27 lines
download today code into "MITgcm_today" (instead of just "MITgcm")

1 jmc 1.1 #! /usr/bin/env bash
2    
3 jmc 1.38 # $Header: /u/gcmpack/MITgcm/tools/example_scripts/csail/test_baudelaire,v 1.37 2018/01/06 15:19:55 jmc Exp $
4 jmc 1.1
5 jmc 1.23 # Test script for MITgcm that should work on most of the csail.mit.edu Linux machines.
6    
7     tst_grp=0 ; if test $# = 1 ; then tst_grp=$1 ; fi
8     if test $tst_grp != 'a' -a $tst_grp != 'b' ; then
9     echo "missing or invalid argument (expect: 'a' or 'b') ==> exit"
10     exit 1
11     fi
12 jmc 1.1
13     # defaults
14     #export PATH="$PATH:/usr/local/bin"
15     if [ -d ~/bin ]; then export PATH=$PATH:~/bin ; fi
16     #- to get case insensitive "ls" (and order of tested experiments)
17     export LC_ALL="en_US.UTF-8"
18     # Turn off stack limit for FIZHI & AD-tests
19     ulimit -s unlimited
20 jmc 1.4 # MPI test (for now, only with gfortran)
21     export MPI_GCC_DIR=/srv/software/gcc/gcc-packages/gcc-4.4.5/mpich2/mpich2-1.3
22     export MPI_INC_DIR=$MPI_GCC_DIR/include
23     export PATH="$PATH:$MPI_GCC_DIR/bin"
24 jmc 1.1
25     #- method to acces CVS:
26     # export CVSROOT='/u/gcmpack'
27     # export CVSROOT=':ext:@mitgcm.org:/u/gcmpack'
28     # export CVS_RSH='ssh' ; cvs co -P MITgcm > /dev/null
29     # cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co -P MITgcm > /dev/null
30     cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack'
31     cmdCVS='cvs -d /u/gcmpack'
32    
33 jmc 1.19 # checkOut=2 : download new code ;
34 jmc 1.20 # =3 : skip download but, if sepDir, use a new copy
35 jmc 1.19 # =1 : update code (if no existing code -> swith to 2)
36     # =0 : use existing code (if no existing code -> swith to 2)
37 jmc 1.11 dInWeek=`date +%a`
38 jmc 1.34
39     dNam=`hostname -s | tr '[:upper:]' '[:lower:]'`
40 jmc 1.36 TESTDIR="/scratch/jmc/test_${dNam}"
41 jmc 1.35 outDir="${dNam}-${tst_grp}"
42     #outDir=`hostname -s`
43 jmc 1.3 MC=13
44 jmc 1.1 sepDir=1
45     option=
46 jmc 1.34
47 jmc 1.20 #tst_list='g7a adm mpa g77 gfo+rs mth mp2+rs mpi ifc'
48     #if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm oad" ; fi
49 jmc 1.23 if test $tst_grp = 'a' ; then
50     checkOut=2
51     tst_list='g7a mpa mth mp2+rs mpi oad'
52     else
53     checkOut=3
54     tst_list='adm g77 gfo+rs ifc'
55     if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm" ; fi
56     fi
57     echo " test: $outDir ; list='$tst_list'"
58 jmc 1.1
59 jmc 1.19 #option="-nc" ; checkOut=1
60     #option="-q" ; checkOut=1
61 jmc 1.1
62     TODAY=`date +%d`
63     tdir=$TESTDIR
64 jmc 1.34 updFile='updated_code' ; today=`date +%Y%m%d`
65 jmc 1.19 if test $checkOut != 2 ; then
66 jmc 1.38 if test -e $tdir/MITgcm_today/CVS ; then
67     echo $tdir/MITgcm_today/CVS 'exist'
68 jmc 1.19 if test $sepDir = 0 -a $checkOut = 1 ; then
69 jmc 1.34 #- remove date/lock-file:
70     if test -f $tdir/$updFile ; then rm -f $tdir/$updFile ; sleep 2 ; fi
71 jmc 1.15 echo -n "Update the MITgcm code using: $cmdCVS ..."
72 jmc 1.38 cd $tdir/MITgcm_today
73 jmc 1.7 $cmdCVS update -P -d
74 jmc 1.15 echo " done"
75 jmc 1.34 #- update date/lock-file:
76     ( cd $tdir ; echo $today > $updFile
77     sleep 2 ; ls -l $updFile )
78 jmc 1.7 fi
79 jmc 1.1 else
80 jmc 1.38 echo -n $tdir/MITgcm_today 'missing ; '
81 jmc 1.19 checkOut=2
82 jmc 1.1 fi
83     fi
84 jmc 1.19 if test $checkOut = 2 ; then
85 jmc 1.1 if test -e $tdir ; then
86 jmc 1.38 echo -n "Removing working copy: $tdir/MITgcm_today ..."
87     test -e $tdir/MITgcm_today && rm -rf $tdir/MITgcm_today
88 jmc 1.1 else
89     echo -n "Creating a working dir: $tdir ..."
90     mkdir $tdir
91     fi
92     echo " done"
93 jmc 1.34 cd $tdir
94     #- remove date/lock-file:
95     if test -f $updFile ; then rm -f $updFile ; sleep 2 ; fi
96 jmc 1.1 echo -n "Downloading the MITgcm code using: $cmdCVS ..."
97 jmc 1.38 $cmdCVS co -P -d MITgcm_today MITgcm > /dev/null
98     retVal=$?
99     if test "x$retVal" != x0 ; then echo ''
100     echo -n "'$cmdCVS co -P -d MITgcm_today MITgcm'"
101     echo " on '"`hostname`"' fail (return val=$retVal) => exit"
102     exit
103     else echo " done"
104     #echo "'$cmdCVS co -P -d MITgcm_today MITgcm' on '"`hostname`"' work"
105     fi
106 jmc 1.28 if test -d other_input ; then
107     list_dirs=`(cd other_input ; ls 2> /dev/null )`
108 jmc 1.29 echo "Updating ( $cmdCVS update -P -d ) 'other_input' extra dirs:"
109 jmc 1.28 for exd in $list_dirs ; do
110 jmc 1.29 if test -d other_input/$exd/CVS ; then echo " $exd"
111 jmc 1.28 ( cd other_input/$exd ; $cmdCVS update -P -d )
112 jmc 1.29 fi
113 jmc 1.28 done
114 jmc 1.29 echo " <-- update of 'other_input' dirs done"
115 jmc 1.28 fi
116 jmc 1.34 #- update date/lock-file:
117     echo $today > $updFile ; sleep 2 ; ls -l $updFile
118 jmc 1.1 else
119     cd $tdir
120     fi
121    
122     #------------------------------------------------------------------------
123    
124     firstTst=`echo $tst_list | awk '{print $1}'`
125     last_Tst=`echo $tst_list | awk '{print $NF}'`
126     for tt in $tst_list
127     do
128    
129 jmc 1.5 echo "================================================================"
130     typ=`echo $tt | sed 's/+rs//'`
131 jmc 1.22 #- define list of additional experiences to test:
132     addExp=''
133     if test $typ = 'mp2' -o $typ = 'ifc' ; then
134 jmc 1.24 addExp="offline_cheapaml atm_gray"
135     fi
136 jmc 1.27 if test $typ = 'gfo' -o $typ = 'ifc' ; then
137 jmc 1.26 addExp="$addExp global_oce_biogeo_bling"
138 jmc 1.29 addExp="$addExp shelfice_remeshing"
139 jmc 1.26 fi
140 jmc 1.27 if test $typ = 'mpi' ; then
141     addExp="$addExp global_oce_cs32"
142     fi
143 jmc 1.5 #- check day and time:
144     curDay=`date +%d` ; curHour=`date +%H`
145     if [ $curDay -ne $TODAY ] ; then
146     date ; echo "day is over => skip test $typ"
147     continue
148     fi
149     if [ $curHour -ge 18 ] ; then
150     date ; echo "too late to run test $typ"
151     continue
152     fi
153 jmc 1.19 if test $sepDir = 0 -a "x$option" != x -a $tt != $last_Tst ; then
154     echo "using option='$option' prevent multi-tests => skip test $typ"
155     continue
156     fi
157 jmc 1.5 #- clean-up old output files
158     rm -f $tdir/output_${typ}*
159 jmc 1.12 touch $tdir/output_$tt
160 jmc 1.38 echo -n "-- Starting test: $tt at: " >> $tdir/output_$tt
161     date >> $tdir/output_$tt
162     echo " typ='$typ', addExp='$addExp'" >> $tdir/output_$tt
163 jmc 1.5 if test $sepDir = 1 ; then
164     new_dir="MITgcm_$typ"
165 jmc 1.20 reUse=1 ; if [ $checkOut -ge 2 ] ; then reUse=0 ; fi
166     if test -d $new_dir/CVS -a $reUse = 1 ; then
167 jmc 1.34 cd $tdir/$new_dir
168 jmc 1.14 if test $tt != $typ ; then
169     ( cd verification ; ../tools/do_tst_2+2 -clean )
170     fi
171 jmc 1.19 if test $checkOut = 1 ; then
172 jmc 1.38 echo -n "Update the MITgcm code using: $cmdCVS ..." | tee -a $tdir/output_$tt
173     $cmdCVS update -P -d | tee -a $tdir/output_$tt
174     echo " done" | tee -a $tdir/output_$tt
175 jmc 1.19 fi
176 jmc 1.5 else
177 jmc 1.21 if test -d prev ; then
178     #-- save previous summary:
179     oldS=`ls -t ${new_dir}/verification/tr_${outDir}_*/summary.txt 2> /dev/null | head -1`
180 jmc 1.33 if test "x$oldS" != x ; then
181     cat $oldS | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > prev/tr_out.$typ
182     touch -r $oldS prev/tr_out.$typ
183     fi
184 jmc 1.21 if test $tt != $typ ; then
185     oldS=`ls -t ${new_dir}/verification/rs_${outDir}_*/summary.txt 2> /dev/null | head -1`
186     if test "x$oldS" != x ; then cp -p -f $oldS prev/rs_out.$typ ; fi
187     fi
188     fi
189 jmc 1.38 echo " remove dir: $new_dir and make new one" >> $tdir/output_$tt
190 jmc 1.5 test -e $new_dir && rm -rf $new_dir
191     mkdir $new_dir
192 jmc 1.34 #- before making a copy, check that code has been updated
193     nCount=0; today=`date +%Y%m%d`
194     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
195     while [ $today -gt $updDate ] ; do
196     nCount=`expr $nCount + 1`
197     if [ $nCount -gt 40 ] ; then
198     echo " waiting too long (nCount=$nCount) for updated code"
199     echo " today=$today , updDate=$updDate "
200     ls -l $updFile
201     exit
202     fi
203     sleep 60
204     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
205     done
206     ls -l $updFile | tee -a $tdir/output_$tt
207     echo " waited nCount=$nCount for updated code ($updDate) to copy" | tee -a $tdir/output_$tt
208     #-----------------------------
209 jmc 1.38 if test -d MITgcm_today -a -d $new_dir ; then
210     echo " copy main code from MITgcm_today to $new_dir" | tee -a $tdir/output_$tt
211     cp -ra MITgcm_today/* $new_dir 2>&1 | tee -a $tdir/output_$tt
212 jmc 1.34 else
213 jmc 1.38 if test -d MITgcm_today ; then
214     echo " missing dir $new_dir --> end test $tt" | tee -a $tdir/output_$tt
215 jmc 1.34 else
216 jmc 1.38 echo " missing dir MITgcm_today --> end test $tt" | tee -a $tdir/output_$tt
217 jmc 1.34 fi
218     continue
219     fi
220 jmc 1.38 echo -n " cd $tdir/$new_dir " | tee -a $tdir/output_$tt
221 jmc 1.34 cd $tdir/$new_dir
222     retVal=$?
223 jmc 1.38 ( echo "(retVal= $retVal )" ; pwd ) | tee -a $tdir/output_$tt
224 jmc 1.34 if test -d verification ; then
225 jmc 1.38 echo " check: dir verification exist" | tee -a $tdir/output_$tt
226 jmc 1.34 else
227 jmc 1.38 echo " missing dir verification --> end test $tt" | tee -a $tdir/output_$tt
228 jmc 1.34 continue
229     fi
230 jmc 1.22 #-- download additional experience from Contrib:
231     for exp2add in $addExp ; do
232 jmc 1.38 echo " add dir: $exp2add (from Contrib:verification_other)" | tee -a $tdir/output_$tt
233 jmc 1.22 ( cd verification ; $cmdCVS co -P -d $exp2add \
234     MITgcm_contrib/verification_other/$exp2add > /dev/null )
235 jmc 1.27 if test $exp2add = 'global_oce_cs32' ; then
236 jmc 1.38 echo " link dir 'other_input/core2_cnyf' in here" | tee -a $tdir/output_$tt
237 jmc 1.30 ( cd verification/${exp2add}
238     ln -s ../../../other_input/core2_cnyf . )
239 jmc 1.27 fi
240 jmc 1.28 if test $exp2add = 'shelfice_remeshing' ; then
241 jmc 1.38 echo " link dir 'other_input/remeshing_code' to 'extra_code'" | tee -a $tdir/output_$tt
242 jmc 1.28 ( cd verification/${exp2add}
243     ln -s ../../../other_input/remeshing_code extra_code )
244     fi
245 jmc 1.22 done
246 jmc 1.5 fi
247 jmc 1.1 else
248 jmc 1.38 cd $tdir/MITgcm_today
249 jmc 1.1 fi
250 jmc 1.5 cd verification
251 jmc 1.1
252 jmc 1.5 #-- set the testreport command:
253 jmc 1.7 comm="./testreport"
254     if test $typ = 'g7a' -o $typ = 'adm' -o $typ = 'mpa' ; then
255 jmc 1.12 comm="$comm -adm"
256 jmc 1.18 elif test $typ = 'oad' ; then
257     comm="$comm -oad"
258 jmc 1.11 elif test $typ = 'tlm' ; then
259 jmc 1.12 comm="$comm -tlm"
260 jmc 1.5 elif test $typ = 'mth' -o $typ = 'mp2' ; then
261 jmc 1.4 export GOMP_STACKSIZE=400m
262 jmc 1.5 export OMP_NUM_THREADS=2
263     comm="$comm -mth"
264 jmc 1.17 else
265     comm="$comm -md cyrus-makedepend"
266 jmc 1.5 fi
267 jmc 1.37 comm="$comm -odir $outDir -a jm_c@mitgcm.org"
268 jmc 1.5 #-- set the optfile (+ mpi & match-precision)
269     MPI=0
270     case $typ in
271     'g77'|'g7a') OPTFILE='../tools/build_options/linux_amd64_g77' ;;
272 jmc 1.18 'gfo'|'adm'|'oad'|'tlm'|'mth') comm="$comm -match $MC -devel"
273 jmc 1.11 OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
274 jmc 1.20 'ifc') comm="$comm -devel"
275     OPTFILE='../tools/build_options/linux_amd64_ifort11' ;;
276 jmc 1.5 'pgi') OPTFILE='../tools/build_options/linux_amd64_pgf77' ;;
277 jmc 1.11 'mpa'|'mpi'|'mp2') comm="$comm -match $MC -devel" ; MPI=6
278     OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
279 jmc 1.5 *) OPTFILE= ;;
280     esac
281     #-- set MPI command:
282 jmc 1.8 if test $MPI != 0 ; then
283     if test $typ = 'mp2' ; then MPI=3 ; fi
284 jmc 1.7 if test $typ = 'mpa' ; then
285 jmc 1.6 EXE="mpirun -np TR_NPROC ./mitgcmuv_ad"
286 jmc 1.5 else
287 jmc 1.6 EXE="mpirun -np TR_NPROC ./mitgcmuv"
288 jmc 1.5 fi
289     fi
290 jmc 1.1
291 jmc 1.7 #-- set specific Env Vars:
292 jmc 1.18 if test $typ = 'oad' ; then
293     source ~jmc/mitgcm/bin/setenv_OpenAD.sh
294     fi
295 jmc 1.7 if test $typ = 'ifc' ; then
296     source /srv/software/intel/intel-11.1.073/bin/ifortvars.sh intel64
297     fi
298     if test $typ = 'pgi' ; then
299 jmc 1.8 #listT='fizhi-cs-32x32x40 fizhi-cs-aqualev20'
300 jmc 1.7 export PGI=/srv/software/pgi/pgi-10.9
301     export PATH="$PATH:$PGI/linux86-64/10.9/bin"
302     export LM_LICENSE_FILE=$PGI/license.dat
303     fi
304    
305 jmc 1.19 if test $sepDir = 0 -a "x$option" = x -a $tt = $firstTst -a $checkOut != 2 ; then
306 jmc 1.1 #-- cleaning:
307 jmc 1.5 echo "======================"
308 jmc 1.12 echo "Cleaning test directories:" | tee -a $tdir/output_$tt
309 jmc 1.1 cmdCLN="./testreport -clean"
310 jmc 1.12 echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt
311     $cmdCLN >> $tdir/output_$tt 2>&1
312 jmc 1.5 echo "======================"
313 jmc 1.12 echo "" | tee -a $tdir/output_$tt
314 jmc 1.1 fi
315    
316 jmc 1.5 #-- run the testreport command:
317 jmc 1.12 echo -n "Running testreport using:" | tee -a $tdir/output_$tt
318 jmc 1.1 if test "x$OPTFILE" != x ; then
319     comm="$comm -of=$OPTFILE"
320     fi
321 jmc 1.12 if test $MPI = 0 ; then echo '' | tee -a $tdir/output_$tt
322     else echo " (EXE='$EXE')" | tee -a $tdir/output_$tt
323 jmc 1.8 comm="$comm -MPI $MPI -command \"\$EXE\""
324     fi
325 jmc 1.1 if test "x$option" != x ; then comm="$comm $option" ; fi
326 jmc 1.8 #if test $typ = 'pgi' ; then comm="$comm -t \"\$listT\"" ; fi
327 jmc 1.12 echo " \"eval $comm\"" | tee -a $tdir/output_$tt
328 jmc 1.1 echo "======================"
329 jmc 1.12 eval $comm >> $tdir/output_$tt 2>&1
330     sed -n "/^An email /,/^======== End of testreport / p" $tdir/output_$tt
331     echo "" | tee -a $tdir/output_$tt
332 jmc 1.1
333 jmc 1.5 #-- also test restart (test 2+2=4)
334     if test $tt != $typ
335     then
336 jmc 1.12 echo "testing restart using:" | tee -a $tdir/output_$tt
337 jmc 1.37 comm="../tools/do_tst_2+2 -o $outDir -a jm_c@mitgcm.org"
338 jmc 1.8 if test $MPI = 0 ; then
339 jmc 1.12 echo " \"$comm\"" | tee -a $tdir/output_$tt
340 jmc 1.8 echo "======================"
341 jmc 1.12 $comm >> $tdir/output_$tt 2>&1
342 jmc 1.8 else
343 jmc 1.12 echo " \"$comm -mpi -exe $EXE\"" | tee -a $tdir/output_$tt
344 jmc 1.5 echo "======================"
345 jmc 1.12 $comm -mpi -exe "$EXE" >> $tdir/output_$tt 2>&1
346 jmc 1.5 fi
347     echo ; cat tst_2+2_out.txt
348     echo
349 jmc 1.1 fi
350 jmc 1.5 export OMP_NUM_THREADS=1
351 jmc 1.1
352 jmc 1.5 if test $sepDir = 0 ; then
353     #-- cleaning:
354 jmc 1.7 echo "======================"
355 jmc 1.38 echo "Cleaning test directories:" | tee -a $tdir/output_$tt
356 jmc 1.7 if test $tt != $typ ; then
357     cmdCLN="../tools/do_tst_2+2 -clean"
358 jmc 1.12 echo " clean tst_2+2 running: $cmdCLN" | tee -a $tdir/output_$tt
359     $cmdCLN >> $tdir/output_$tt 2>&1
360 jmc 1.7 fi
361     if test $tt != $last_Tst ; then
362     cmdCLN="./testreport -clean"
363 jmc 1.12 echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt
364     $cmdCLN >> $tdir/output_$tt 2>&1
365 jmc 1.7 fi
366     echo "======================"
367     echo
368     fi
369 jmc 1.34 cd $tdir
370 jmc 1.1
371     done

  ViewVC Help
Powered by ViewVC 1.1.22