/[MITgcm]/MITgcm_contrib/test_scripts/ref_machine/test_baudelaire
ViewVC logotype

Annotation of /MITgcm_contrib/test_scripts/ref_machine/test_baudelaire

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.12 - (hide annotations) (download)
Mon Sep 16 23:38:47 2019 UTC (5 years, 10 months ago) by jmc
Branch: MAIN
Changes since 1.11: +4 -3 lines
update testing script on villon (new Ubuntu testing platform)
 + tie-up batsi and baudelaire testing script

1 jmc 1.1 #! /usr/bin/env bash
2    
3 jmc 1.12 # $Header: /u/gcmpack/MITgcm_contrib/test_scripts/ref_machine/test_baudelaire,v 1.11 2019/08/28 14:43:23 jmc Exp $
4 jmc 1.1
5     # Test script for MITgcm that should work on most of the x86_64 Linux machines.
6    
7     tst_grp=0 ; if test $# = 1 ; then tst_grp=$1 ; fi
8     if test $tst_grp != 'a' -a $tst_grp != 'b' ; then
9     echo "missing or invalid argument (expect: 'a' or 'b') ==> exit"
10     exit 1
11     fi
12    
13     #- defaults
14     #export PATH="$PATH:/usr/local/bin"
15     if [ -d ~/bin ]; then export PATH=$PATH:~/bin ; fi
16     #- to get case insensitive "ls" (and order of tested experiments)
17     export LC_ALL="en_US.UTF-8"
18     # Turn off stack limit for FIZHI & AD-tests
19     ulimit -s unlimited
20     # MPI test (for now, only with gfortran)
21     export MPI_GCC_DIR=/srv/software/gcc/gcc-packages/gcc-4.4.5/mpich2/mpich2-1.3
22     export MPI_INC_DIR=$MPI_GCC_DIR/include
23     export PATH="$PATH:$MPI_GCC_DIR/bin"
24    
25 jmc 1.5 #- method to access CVS:
26 jmc 1.1 cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack'
27     cmdCVS='cvs -d /u/gcmpack'
28     # export CVS_RSH=ssh
29     # cmdCVS='cvs -q -d :ext:jmc@mitgcm.org:/u/gcmpack'
30    
31 jmc 1.5 #- which GitHub repository to use and how to access it:
32 jmc 1.7 git_repo='MITgcm'; git_code='MITgcm' ; git_other='verification_other'
33 jmc 1.3 #git_repo='altMITgcm'; #git_code='MITgcm66h'
34 jmc 1.5 #--
35     #git_repo="https://github.com/$git_repo"
36     git_repo="git://github.com/$git_repo"
37     #git_repo="git@github.com:$git_repo"
38 jmc 1.1
39 jmc 1.12 #-------------------------------------------------------------------
40 jmc 1.1 # checkOut=3 : clone from GitHub and make a new copy (if sepDir)
41 jmc 1.12 # =2 : update (git pull) repo and make a new copy code (switch to 3 if no repo)
42 jmc 1.1 # =1 : skip update but use a new copy (if sepDir)
43     # =0 : use existing test code ( switch to 1 if missing test code )
44     dInWeek=`date +%a`
45    
46     dNam=`hostname -s | tr '[:upper:]' '[:lower:]'`
47     #TESTDIR="$HOME/test_${dNam}"
48     #outDir=$dNam
49     #MC=11
50     TESTDIR="/scratch/jmc/test_${dNam}"
51     outDir="${dNam}-${tst_grp}"
52     MC=13
53     sepDir=1
54     option=
55    
56     if test $tst_grp = 'a' ; then
57     checkOut=2
58 jmc 1.11 tst_list='mpa mth mp2+rs mpi oad'
59 jmc 1.1 #tst_list='mpa adm mpi gfo+rs mth+rs'
60     #tst_list="$tst_list oad"
61     else
62     checkOut=1
63     tst_list='adm g77 gfo+rs ifc'
64     if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm" ; fi
65     fi
66     echo " test: $outDir ; list='$tst_list'"
67    
68     #option="-nc" ; checkOut=0
69     #option="-q" ; checkOut=0
70    
71     TODAY=`date +%d`
72     tmpFil="/tmp/"`basename $0`".$$"
73     tdir=$TESTDIR
74     updFile='updated_code' ; today=`date +%Y%m%d`
75    
76     if [ $checkOut -le 1 ] ; then
77     if test -e $tdir/MITgcm_today/doc ; then
78     echo $tdir/MITgcm_today/doc 'exist'
79     else
80     echo -n $tdir/MITgcm_today 'missing ; '
81     checkOut=2
82     echo "will make a new copy ( checkOut=$checkOut )"
83     fi
84     fi
85    
86     if [ $checkOut -ge 2 ] ; then
87     #---- cleaning:
88     if test -e $tdir ; then
89     #- remove date/lock-file:
90     if test -f $tdir/$updFile ; then rm -f $tdir/$updFile ; sleep 2 ; fi
91     echo -n "Removing working copy: $tdir/MITgcm_today ..."
92     test -e $tdir/MITgcm_today && rm -rf $tdir/MITgcm_today
93     else
94     echo -n "Creating a working dir: $tdir ..."
95     mkdir $tdir
96     fi
97     echo " done"
98     cd $tdir
99    
100     #---- Making a new clone or updating existing one:
101 jmc 1.4 if [ $checkOut -eq 2 ] ; then
102 jmc 1.7 if test -e $git_code/.git/config ; then
103     echo $git_code/.git/config 'exist'
104 jmc 1.4 else
105 jmc 1.7 echo -n $git_code/.git/config 'missing ; '
106     checkOut=3
107     echo "will get new clone ( checkOut=$checkOut )"
108     fi
109     if test -e $git_other/.git/config ; then
110     echo $git_other/.git/config 'exist'
111     else
112     echo -n $git_other/.git/config 'missing ; '
113 jmc 1.4 checkOut=3
114     echo "will get new clone ( checkOut=$checkOut )"
115     fi
116 jmc 1.1 fi
117     if [ $checkOut -eq 3 ] ; then
118 jmc 1.4 test -e $git_code && rm -rf $git_code
119 jmc 1.7 echo "Make a clone of $git_code from repo: $git_repo ..."
120 jmc 1.5 git clone $git_repo/${git_code}.git 2> $tmpFil
121 jmc 1.1 retVal=$?
122     if test $retVal = 0 ; then
123     echo ' --> done!'
124     rm -f $tmpFil
125     else
126     echo " Error: 'git clone' returned: $retVal"
127     cat $tmpFil
128     rm -f $tmpFil
129     exit 2
130     fi
131 jmc 1.7 test -e $git_other && rm -rf $git_other
132     echo "Make a clone of $git_other from repo: $git_repo ..."
133     git clone $git_repo/${git_other}.git 2> $tmpFil
134     retVal=$?
135     if test $retVal = 0 ; then
136     echo ' --> done!'
137     rm -f $tmpFil
138     else
139     echo " Error: 'git clone' returned: $retVal"
140     cat $tmpFil
141     rm -f $tmpFil
142     exit 2
143     fi
144 jmc 1.1 else
145     echo "Updating current clone ( $git_code ) ..."
146     ( cd $git_code ; git pull )
147 jmc 1.4 retVal=$?
148     if test $retVal = 0 ; then
149     echo ' --> done!'
150     else
151     echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
152     exit 3
153     fi
154 jmc 1.8 ( cd $git_code ; git checkout master -- . )
155 jmc 1.7 echo "Updating current clone ( $git_other ) ..."
156     ( cd $git_other ; git pull )
157     retVal=$?
158     if test $retVal = 0 ; then
159     echo ' --> done!'
160     else
161     echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
162     exit 3
163     fi
164 jmc 1.8 ( cd $git_other ; git checkout master -- . )
165 jmc 1.1 fi
166     #---- making a new working copy: MITgcm_today
167     rsync -a $git_code/ MITgcm_today --exclude '.git'
168 jmc 1.7 if test $sepDir = 0 ; then
169     echo -n " make a local copy of $git_other in MITgcm_today ..." | tee -a $tdir/output_$tt
170     ( cd MITgcm_today
171     rsync -a ../$git_other/ $git_other --exclude '.git' )
172     echo " done" | tee -a $tdir/output_$tt
173     fi
174 jmc 1.1 #---- updating "other_input" dir
175     if test -d other_input ; then
176     list_dirs=`(cd other_input ; ls 2> /dev/null )`
177     echo "Updating ( $cmdCVS update -P -d ) 'other_input' extra dirs:"
178     for exd in $list_dirs ; do
179     if test -d other_input/$exd/CVS ; then echo " $exd"
180     ( cd other_input/$exd ; $cmdCVS update -P -d )
181     fi
182     done
183     echo " <-- update of 'other_input' dirs done"
184     fi
185     #---- update date/lock-file:
186     echo $today > $updFile ; sleep 2 ; ls -l $updFile
187     else
188     cd $tdir
189     fi
190    
191     #------------------------------------------------------------------------
192    
193     firstTst=`echo $tst_list | awk '{print $1}'`
194     last_Tst=`echo $tst_list | awk '{print $NF}'`
195     for tt in $tst_list
196     do
197    
198     echo "================================================================"
199     typ=`echo $tt | sed 's/+rs//'`
200     #- define list of additional experiences to test:
201     addExp=''
202     if test $typ = 'mp2' -o $typ = 'ifc' ; then
203 jmc 1.9 addExp="offline_cheapaml atm_gray atm_strato"
204 jmc 1.1 fi
205 jmc 1.10 if test $typ = 'gfo' -o $typ = 'ifc' -o $typ = 'adm' ; then
206     addExp="$addExp global_oce_biogeo_bling"
207     fi
208 jmc 1.1 if test $typ = 'gfo' -o $typ = 'ifc' ; then
209     addExp="$addExp shelfice_remeshing"
210     fi
211     if test $typ = 'mpi' ; then
212     #addExp="$addExp global_ocean.gm_k3d"
213     addExp="$addExp global_oce_cs32"
214     fi
215     #- check day and time:
216     curDay=`date +%d` ; curHour=`date +%H`
217     if [ $curDay -ne $TODAY ] ; then
218     date ; echo "day is over => skip test $typ"
219     continue
220     fi
221     if [ $curHour -ge 18 ] ; then
222     date ; echo "too late to run test $typ"
223     continue
224     fi
225     if test $sepDir = 0 -a "x$option" != x -a $tt != $last_Tst ; then
226     echo "using option='$option' prevent multi-tests => skip test $typ"
227     continue
228     fi
229     #- clean-up old output files
230     if test -d $tdir/prev ; then
231     mv -f $tdir/output_${typ}* $tdir/prev
232     else
233     rm -f $tdir/output_${typ}*
234     fi
235     touch $tdir/output_$tt
236     echo -n "-- Starting test: $tt at: " >> $tdir/output_$tt
237     date >> $tdir/output_$tt
238     echo " typ='$typ', addExp='$addExp'" >> $tdir/output_$tt
239     if test $sepDir = 1 ; then
240     new_dir="MITgcm_$typ"
241     reUse=0 ; if [ $checkOut -le 0 ] ; then reUse=1 ; fi
242 jmc 1.12 if test -d $new_dir/doc -a $reUse = 1 ; then
243 jmc 1.1 cd $tdir/$new_dir
244     if test $tt != $typ ; then
245     ( cd verification ; ../tools/do_tst_2+2 -clean )
246     fi
247     else
248     if test -d prev ; then
249     #-- save previous summary:
250     oldS=`ls -t ${new_dir}/verification/tr_${outDir}_*/summary.txt 2> /dev/null | head -1`
251     if test "x$oldS" != x ; then
252     cat $oldS | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > prev/tr_out.$typ
253     touch -r $oldS prev/tr_out.$typ
254     fi
255     if test $tt != $typ ; then
256     oldS=`ls -t ${new_dir}/verification/rs_${outDir}_*/summary.txt 2> /dev/null | head -1`
257     if test "x$oldS" != x ; then cp -p -f $oldS prev/rs_out.$typ ; fi
258     fi
259     fi
260     echo " remove dir: $new_dir and make new one" >> $tdir/output_$tt
261     test -e $new_dir && rm -rf $new_dir
262     mkdir $new_dir
263     #- before making a copy, check that code has been updated
264     nCount=0; today=`date +%Y%m%d`
265     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
266     while [ $today -gt $updDate ] ; do
267     nCount=`expr $nCount + 1`
268     if [ $nCount -gt 40 ] ; then
269     echo " waiting too long (nCount=$nCount) for updated code"
270     echo " today=$today , updDate=$updDate "
271     ls -l $updFile
272     exit
273     fi
274     sleep 60
275     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
276     done
277     ls -l $updFile | tee -a $tdir/output_$tt
278     echo " waited nCount=$nCount for updated code ($updDate) to copy" | tee -a $tdir/output_$tt
279     #-----------------------------
280     if test -d MITgcm_today -a -d $new_dir ; then
281     echo " copy main code from MITgcm_today to $new_dir" | tee -a $tdir/output_$tt
282     cp -ra MITgcm_today/* $new_dir 2>&1 | tee -a $tdir/output_$tt
283     else
284     if test -d MITgcm_today ; then
285     echo " missing dir $new_dir --> end test $tt" | tee -a $tdir/output_$tt
286     else
287     echo " missing dir MITgcm_today --> end test $tt" | tee -a $tdir/output_$tt
288     fi
289     continue
290     fi
291     echo -n " cd $tdir/$new_dir " | tee -a $tdir/output_$tt
292     cd $tdir/$new_dir
293     retVal=$?
294     ( echo "(retVal= $retVal )" ; pwd ) | tee -a $tdir/output_$tt
295     if test -d verification ; then
296     echo " check: dir verification exist" | tee -a $tdir/output_$tt
297     else
298     echo " missing dir verification --> end test $tt" | tee -a $tdir/output_$tt
299     continue
300     fi
301 jmc 1.7 if test "x$addExp" != x ; then
302     echo -n " make a local copy of $git_other ..." | tee -a $tdir/output_$tt
303     rsync -a $tdir/$git_other/ $git_other --exclude '.git'
304     echo " done" | tee -a $tdir/output_$tt
305     fi
306     #-- add additional experience from $git_other
307 jmc 1.1 for exp2add in $addExp ; do
308 jmc 1.7 echo " add link: $exp2add (from $git_other )" | tee -a $tdir/output_$tt
309     ( cd verification ; ln -s ../$git_other/$exp2add . )
310 jmc 1.1 if test $exp2add = 'global_oce_cs32' ; then
311 jmc 1.7 echo " link dir 'other_input/core2_cnyf' in here" | tee -a $tdir/output_$tt
312 jmc 1.1 ( cd verification/${exp2add}
313     ln -s ../../../other_input/core2_cnyf . )
314     fi
315 jmc 1.6 #if test $exp2add = 'shelfice_remeshing' ; then
316     # echo " link dir 'other_input/remeshing_code' to 'extra_code'" | tee -a $tdir/output_$tt
317     # ( cd verification/${exp2add}
318     # ln -s ../../../other_input/remeshing_code extra_code )
319     #fi
320 jmc 1.1 done
321     fi
322     else
323     cd $tdir/MITgcm_today
324     fi
325     cd verification
326    
327     #-- set the testreport command:
328     comm="./testreport"
329     if test $typ = 'g7a' -o $typ = 'adm' -o $typ = 'mpa' ; then
330     comm="$comm -adm"
331     elif test $typ = 'oad' ; then
332     comm="$comm -oad"
333     elif test $typ = 'tlm' ; then
334     comm="$comm -tlm"
335     elif test $typ = 'mth' -o $typ = 'mp2' ; then
336     export GOMP_STACKSIZE=400m
337     export OMP_NUM_THREADS=2
338     comm="$comm -mth"
339     else
340     comm="$comm -md cyrus-makedepend"
341     fi
342     comm="$comm -odir $outDir -a jm_c@mitgcm.org"
343     #-- set the optfile (+ mpi & match-precision)
344     MPI=0
345     case $typ in
346     'g77'|'g7a') OPTFILE='../tools/build_options/linux_amd64_g77' ;;
347     'gfo'|'adm'|'oad'|'tlm'|'mth') comm="$comm -match $MC -devel"
348     OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
349     'ifc') comm="$comm -devel"
350     OPTFILE='../tools/build_options/linux_amd64_ifort11' ;;
351     'pgi') OPTFILE='../tools/build_options/linux_amd64_pgf77' ;;
352     'mpa'|'mpi'|'mp2') comm="$comm -match $MC -devel" ; MPI=6
353     OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
354     *) OPTFILE= ;;
355     esac
356     #-- set MPI command:
357     if test $MPI != 0 ; then
358     if test $typ = 'mp2' ; then MPI=3 ; fi
359     if test $typ = 'mpa' ; then
360     EXE="mpirun -np TR_NPROC ./mitgcmuv_ad"
361     else
362     EXE="mpirun -np TR_NPROC ./mitgcmuv"
363     fi
364     fi
365    
366     #-- set specific Env Vars:
367     if test $typ = 'oad' ; then
368 jmc 1.2 #- for some reasons, "source ScriptFile | tee -a LogFile"
369     # does run the script but does not keep the env-var settings
370     source $HOME/mitgcm/bin/setenv_OpenAD.sh >> $tdir/output_$tt
371 jmc 1.1 fi
372     if test $typ = 'ifc' ; then
373     source /srv/software/intel/intel-11.1.073/bin/ifortvars.sh intel64
374     fi
375     if test $typ = 'pgi' ; then
376     #listT='fizhi-cs-32x32x40 fizhi-cs-aqualev20'
377     export PGI=/srv/software/pgi/pgi-10.9
378     export PATH="$PATH:$PGI/linux86-64/10.9/bin"
379     export LM_LICENSE_FILE=$PGI/license.dat
380     fi
381    
382     if test $sepDir = 0 -a "x$option" = x -a $tt = $firstTst -a $checkOut = 0 ; then
383     #-- cleaning:
384     echo "======================"
385     echo "Cleaning test directories:" | tee -a $tdir/output_$tt
386     cmdCLN="./testreport -clean"
387     echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt
388     $cmdCLN >> $tdir/output_$tt 2>&1
389     echo "======================"
390     echo "" | tee -a $tdir/output_$tt
391     fi
392    
393     #-- run the testreport command:
394     echo -n "Running testreport using:" | tee -a $tdir/output_$tt
395     if test "x$OPTFILE" != x ; then
396     comm="$comm -of=$OPTFILE"
397     fi
398     if test $MPI = 0 ; then echo '' | tee -a $tdir/output_$tt
399     else echo " (EXE='$EXE')" | tee -a $tdir/output_$tt
400     comm="$comm -MPI $MPI -command \"\$EXE\""
401     fi
402     if test "x$option" != x ; then comm="$comm $option" ; fi
403     #if test $typ = 'pgi' ; then comm="$comm -t \"\$listT\"" ; fi
404     echo " \"eval $comm\"" | tee -a $tdir/output_$tt
405     echo "======================"
406     eval $comm >> $tdir/output_$tt 2>&1
407     sed -n "/^An email /,/^======== End of testreport / p" $tdir/output_$tt
408     echo "" | tee -a $tdir/output_$tt
409    
410     #-- also test restart (test 2+2=4)
411     if test $tt != $typ
412     then
413     echo "testing restart using:" | tee -a $tdir/output_$tt
414     comm="../tools/do_tst_2+2 -o $outDir -a jm_c@mitgcm.org"
415     if test $MPI = 0 ; then
416     echo " \"$comm\"" | tee -a $tdir/output_$tt
417     echo "======================"
418     $comm >> $tdir/output_$tt 2>&1
419     else
420     echo " \"$comm -mpi -exe $EXE\"" | tee -a $tdir/output_$tt
421     echo "======================"
422     $comm -mpi -exe "$EXE" >> $tdir/output_$tt 2>&1
423     fi
424     echo ; cat tst_2+2_out.txt
425     echo
426     fi
427     export OMP_NUM_THREADS=1
428    
429     if test $sepDir = 0 ; then
430     #-- cleaning:
431     echo "======================"
432     echo "Cleaning test directories:" | tee -a $tdir/output_$tt
433     if test $tt != $typ ; then
434     cmdCLN="../tools/do_tst_2+2 -clean"
435     echo " clean tst_2+2 running: $cmdCLN" | tee -a $tdir/output_$tt
436     $cmdCLN >> $tdir/output_$tt 2>&1
437     fi
438     if test $tt != $last_Tst ; then
439     cmdCLN="./testreport -clean"
440     echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt
441     $cmdCLN >> $tdir/output_$tt 2>&1
442     fi
443     echo "======================"
444     echo
445     fi
446     cd $tdir
447    
448     done

  ViewVC Help
Powered by ViewVC 1.1.22