/[MITgcm]/MITgcm_contrib/test_scripts/ref_machine/test_baudelaire
ViewVC logotype

Annotation of /MITgcm_contrib/test_scripts/ref_machine/test_baudelaire

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.17 - (hide annotations) (download)
Sat Dec 7 16:22:32 2019 UTC (5 years, 7 months ago) by jmc
Branch: MAIN
Changes since 1.16: +1 -4 lines
remove additional experiment "global_oce_biogeo_bling" (now part of the
 main set of experiment in verification)

1 jmc 1.1 #! /usr/bin/env bash
2    
3 jmc 1.17 # $Header: /u/gcmpack/MITgcm_contrib/test_scripts/ref_machine/test_baudelaire,v 1.16 2019/10/21 23:53:20 jmc Exp $
4 jmc 1.1
5     # Test script for MITgcm that should work on most of the x86_64 Linux machines.
6    
7     tst_grp=0 ; if test $# = 1 ; then tst_grp=$1 ; fi
8     if test $tst_grp != 'a' -a $tst_grp != 'b' ; then
9     echo "missing or invalid argument (expect: 'a' or 'b') ==> exit"
10     exit 1
11     fi
12    
13     #- defaults
14     #export PATH="$PATH:/usr/local/bin"
15     if [ -d ~/bin ]; then export PATH=$PATH:~/bin ; fi
16     #- to get case insensitive "ls" (and order of tested experiments)
17     export LC_ALL="en_US.UTF-8"
18     # Turn off stack limit for FIZHI & AD-tests
19     ulimit -s unlimited
20     # MPI test (for now, only with gfortran)
21     export MPI_GCC_DIR=/srv/software/gcc/gcc-packages/gcc-4.4.5/mpich2/mpich2-1.3
22     export MPI_INC_DIR=$MPI_GCC_DIR/include
23     export PATH="$PATH:$MPI_GCC_DIR/bin"
24    
25 jmc 1.5 #- method to access CVS:
26 jmc 1.1 cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack'
27     cmdCVS='cvs -d /u/gcmpack'
28     # export CVS_RSH=ssh
29     # cmdCVS='cvs -q -d :ext:jmc@mitgcm.org:/u/gcmpack'
30    
31 jmc 1.5 #- which GitHub repository to use and how to access it:
32 jmc 1.7 git_repo='MITgcm'; git_code='MITgcm' ; git_other='verification_other'
33 jmc 1.3 #git_repo='altMITgcm'; #git_code='MITgcm66h'
34 jmc 1.5 #--
35     #git_repo="https://github.com/$git_repo"
36     git_repo="git://github.com/$git_repo"
37     #git_repo="git@github.com:$git_repo"
38 jmc 1.1
39 jmc 1.12 #-------------------------------------------------------------------
40 jmc 1.1 # checkOut=3 : clone from GitHub and make a new copy (if sepDir)
41 jmc 1.12 # =2 : update (git pull) repo and make a new copy code (switch to 3 if no repo)
42 jmc 1.1 # =1 : skip update but use a new copy (if sepDir)
43     # =0 : use existing test code ( switch to 1 if missing test code )
44     dInWeek=`date +%a`
45    
46     dNam=`hostname -s | tr '[:upper:]' '[:lower:]'`
47     #TESTDIR="$HOME/test_${dNam}"
48     TESTDIR="/scratch/jmc/test_${dNam}"
49 jmc 1.13 MC=13 ; outDir="${dNam}-${tst_grp}"
50 jmc 1.15 MC=11 ;#outDir=$dNam
51 jmc 1.1 sepDir=1
52     option=
53    
54     if test $tst_grp = 'a' ; then
55     checkOut=2
56 jmc 1.16 tst_list='mth mp2+rs mpi oad g77'
57 jmc 1.1 #tst_list='mpa adm mpi gfo+rs mth+rs'
58     #tst_list="$tst_list oad"
59     else
60     checkOut=1
61 jmc 1.16 tst_list='adm gfo+rs ifc'
62     #if test "x$dInWeek" = xSun ; then tst_list="$tst_list tlm" ; fi
63 jmc 1.1 fi
64     echo " test: $outDir ; list='$tst_list'"
65    
66     #option="-nc" ; checkOut=0
67     #option="-q" ; checkOut=0
68    
69     TODAY=`date +%d`
70     tmpFil="/tmp/"`basename $0`".$$"
71     tdir=$TESTDIR
72     updFile='updated_code' ; today=`date +%Y%m%d`
73    
74     if [ $checkOut -le 1 ] ; then
75     if test -e $tdir/MITgcm_today/doc ; then
76     echo $tdir/MITgcm_today/doc 'exist'
77     else
78     echo -n $tdir/MITgcm_today 'missing ; '
79     checkOut=2
80     echo "will make a new copy ( checkOut=$checkOut )"
81     fi
82     fi
83    
84     if [ $checkOut -ge 2 ] ; then
85     #---- cleaning:
86     if test -e $tdir ; then
87     #- remove date/lock-file:
88     if test -f $tdir/$updFile ; then rm -f $tdir/$updFile ; sleep 2 ; fi
89     echo -n "Removing working copy: $tdir/MITgcm_today ..."
90     test -e $tdir/MITgcm_today && rm -rf $tdir/MITgcm_today
91     else
92     echo -n "Creating a working dir: $tdir ..."
93     mkdir $tdir
94     fi
95     echo " done"
96     cd $tdir
97    
98     #---- Making a new clone or updating existing one:
99 jmc 1.4 if [ $checkOut -eq 2 ] ; then
100 jmc 1.7 if test -e $git_code/.git/config ; then
101     echo $git_code/.git/config 'exist'
102 jmc 1.4 else
103 jmc 1.7 echo -n $git_code/.git/config 'missing ; '
104     checkOut=3
105     echo "will get new clone ( checkOut=$checkOut )"
106     fi
107     if test -e $git_other/.git/config ; then
108     echo $git_other/.git/config 'exist'
109     else
110     echo -n $git_other/.git/config 'missing ; '
111 jmc 1.4 checkOut=3
112     echo "will get new clone ( checkOut=$checkOut )"
113     fi
114 jmc 1.1 fi
115     if [ $checkOut -eq 3 ] ; then
116 jmc 1.4 test -e $git_code && rm -rf $git_code
117 jmc 1.7 echo "Make a clone of $git_code from repo: $git_repo ..."
118 jmc 1.5 git clone $git_repo/${git_code}.git 2> $tmpFil
119 jmc 1.1 retVal=$?
120     if test $retVal = 0 ; then
121     echo ' --> done!'
122     rm -f $tmpFil
123     else
124     echo " Error: 'git clone' returned: $retVal"
125     cat $tmpFil
126     rm -f $tmpFil
127     exit 2
128     fi
129 jmc 1.7 test -e $git_other && rm -rf $git_other
130     echo "Make a clone of $git_other from repo: $git_repo ..."
131     git clone $git_repo/${git_other}.git 2> $tmpFil
132     retVal=$?
133     if test $retVal = 0 ; then
134     echo ' --> done!'
135     rm -f $tmpFil
136     else
137     echo " Error: 'git clone' returned: $retVal"
138     cat $tmpFil
139     rm -f $tmpFil
140     exit 2
141     fi
142 jmc 1.1 else
143     echo "Updating current clone ( $git_code ) ..."
144     ( cd $git_code ; git pull )
145 jmc 1.4 retVal=$?
146     if test $retVal = 0 ; then
147     echo ' --> done!'
148     else
149     echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
150     exit 3
151     fi
152 jmc 1.8 ( cd $git_code ; git checkout master -- . )
153 jmc 1.7 echo "Updating current clone ( $git_other ) ..."
154     ( cd $git_other ; git pull )
155     retVal=$?
156     if test $retVal = 0 ; then
157     echo ' --> done!'
158     else
159     echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
160     exit 3
161     fi
162 jmc 1.8 ( cd $git_other ; git checkout master -- . )
163 jmc 1.1 fi
164     #---- making a new working copy: MITgcm_today
165     rsync -a $git_code/ MITgcm_today --exclude '.git'
166 jmc 1.7 if test $sepDir = 0 ; then
167     echo -n " make a local copy of $git_other in MITgcm_today ..." | tee -a $tdir/output_$tt
168     ( cd MITgcm_today
169     rsync -a ../$git_other/ $git_other --exclude '.git' )
170     echo " done" | tee -a $tdir/output_$tt
171     fi
172 jmc 1.1 #---- updating "other_input" dir
173     if test -d other_input ; then
174     list_dirs=`(cd other_input ; ls 2> /dev/null )`
175     echo "Updating ( $cmdCVS update -P -d ) 'other_input' extra dirs:"
176     for exd in $list_dirs ; do
177     if test -d other_input/$exd/CVS ; then echo " $exd"
178     ( cd other_input/$exd ; $cmdCVS update -P -d )
179     fi
180     done
181     echo " <-- update of 'other_input' dirs done"
182     fi
183     #---- update date/lock-file:
184     echo $today > $updFile ; sleep 2 ; ls -l $updFile
185     else
186     cd $tdir
187     fi
188    
189     #------------------------------------------------------------------------
190    
191     firstTst=`echo $tst_list | awk '{print $1}'`
192     last_Tst=`echo $tst_list | awk '{print $NF}'`
193     for tt in $tst_list
194     do
195    
196     echo "================================================================"
197     typ=`echo $tt | sed 's/+rs//'`
198     #- define list of additional experiences to test:
199     addExp=''
200     if test $typ = 'mp2' -o $typ = 'ifc' ; then
201 jmc 1.9 addExp="offline_cheapaml atm_gray atm_strato"
202 jmc 1.1 fi
203     if test $typ = 'gfo' -o $typ = 'ifc' ; then
204     addExp="$addExp shelfice_remeshing"
205     fi
206     if test $typ = 'mpi' ; then
207     #addExp="$addExp global_ocean.gm_k3d"
208     addExp="$addExp global_oce_cs32"
209     fi
210     #- check day and time:
211     curDay=`date +%d` ; curHour=`date +%H`
212     if [ $curDay -ne $TODAY ] ; then
213     date ; echo "day is over => skip test $typ"
214     continue
215     fi
216     if [ $curHour -ge 18 ] ; then
217     date ; echo "too late to run test $typ"
218     continue
219     fi
220     if test $sepDir = 0 -a "x$option" != x -a $tt != $last_Tst ; then
221     echo "using option='$option' prevent multi-tests => skip test $typ"
222     continue
223     fi
224     #- clean-up old output files
225     if test -d $tdir/prev ; then
226     mv -f $tdir/output_${typ}* $tdir/prev
227     else
228     rm -f $tdir/output_${typ}*
229     fi
230     touch $tdir/output_$tt
231     echo -n "-- Starting test: $tt at: " >> $tdir/output_$tt
232     date >> $tdir/output_$tt
233     echo " typ='$typ', addExp='$addExp'" >> $tdir/output_$tt
234     if test $sepDir = 1 ; then
235     new_dir="MITgcm_$typ"
236     reUse=0 ; if [ $checkOut -le 0 ] ; then reUse=1 ; fi
237 jmc 1.12 if test -d $new_dir/doc -a $reUse = 1 ; then
238 jmc 1.1 cd $tdir/$new_dir
239     if test $tt != $typ ; then
240     ( cd verification ; ../tools/do_tst_2+2 -clean )
241     fi
242     else
243     if test -d prev ; then
244     #-- save previous summary:
245     oldS=`ls -t ${new_dir}/verification/tr_${outDir}_*/summary.txt 2> /dev/null | head -1`
246     if test "x$oldS" != x ; then
247     cat $oldS | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > prev/tr_out.$typ
248     touch -r $oldS prev/tr_out.$typ
249     fi
250     if test $tt != $typ ; then
251     oldS=`ls -t ${new_dir}/verification/rs_${outDir}_*/summary.txt 2> /dev/null | head -1`
252     if test "x$oldS" != x ; then cp -p -f $oldS prev/rs_out.$typ ; fi
253     fi
254     fi
255     echo " remove dir: $new_dir and make new one" >> $tdir/output_$tt
256     test -e $new_dir && rm -rf $new_dir
257     mkdir $new_dir
258     #- before making a copy, check that code has been updated
259     nCount=0; today=`date +%Y%m%d`
260     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
261     while [ $today -gt $updDate ] ; do
262     nCount=`expr $nCount + 1`
263     if [ $nCount -gt 40 ] ; then
264     echo " waiting too long (nCount=$nCount) for updated code"
265     echo " today=$today , updDate=$updDate "
266     ls -l $updFile
267     exit
268     fi
269     sleep 60
270     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
271     done
272     ls -l $updFile | tee -a $tdir/output_$tt
273     echo " waited nCount=$nCount for updated code ($updDate) to copy" | tee -a $tdir/output_$tt
274     #-----------------------------
275     if test -d MITgcm_today -a -d $new_dir ; then
276     echo " copy main code from MITgcm_today to $new_dir" | tee -a $tdir/output_$tt
277     cp -ra MITgcm_today/* $new_dir 2>&1 | tee -a $tdir/output_$tt
278     else
279     if test -d MITgcm_today ; then
280     echo " missing dir $new_dir --> end test $tt" | tee -a $tdir/output_$tt
281     else
282     echo " missing dir MITgcm_today --> end test $tt" | tee -a $tdir/output_$tt
283     fi
284     continue
285     fi
286     echo -n " cd $tdir/$new_dir " | tee -a $tdir/output_$tt
287     cd $tdir/$new_dir
288     retVal=$?
289     ( echo "(retVal= $retVal )" ; pwd ) | tee -a $tdir/output_$tt
290     if test -d verification ; then
291     echo " check: dir verification exist" | tee -a $tdir/output_$tt
292     else
293     echo " missing dir verification --> end test $tt" | tee -a $tdir/output_$tt
294     continue
295     fi
296 jmc 1.7 if test "x$addExp" != x ; then
297     echo -n " make a local copy of $git_other ..." | tee -a $tdir/output_$tt
298     rsync -a $tdir/$git_other/ $git_other --exclude '.git'
299     echo " done" | tee -a $tdir/output_$tt
300     fi
301     #-- add additional experience from $git_other
302 jmc 1.1 for exp2add in $addExp ; do
303 jmc 1.7 echo " add link: $exp2add (from $git_other )" | tee -a $tdir/output_$tt
304     ( cd verification ; ln -s ../$git_other/$exp2add . )
305 jmc 1.1 if test $exp2add = 'global_oce_cs32' ; then
306 jmc 1.7 echo " link dir 'other_input/core2_cnyf' in here" | tee -a $tdir/output_$tt
307 jmc 1.14 ( cd ${git_other}/${exp2add}
308 jmc 1.1 ln -s ../../../other_input/core2_cnyf . )
309     fi
310 jmc 1.6 #if test $exp2add = 'shelfice_remeshing' ; then
311     # echo " link dir 'other_input/remeshing_code' to 'extra_code'" | tee -a $tdir/output_$tt
312     # ( cd verification/${exp2add}
313     # ln -s ../../../other_input/remeshing_code extra_code )
314     #fi
315 jmc 1.1 done
316     fi
317     else
318     cd $tdir/MITgcm_today
319     fi
320     cd verification
321    
322     #-- set the testreport command:
323     comm="./testreport"
324     if test $typ = 'g7a' -o $typ = 'adm' -o $typ = 'mpa' ; then
325     comm="$comm -adm"
326     elif test $typ = 'oad' ; then
327     comm="$comm -oad"
328     elif test $typ = 'tlm' ; then
329     comm="$comm -tlm"
330     elif test $typ = 'mth' -o $typ = 'mp2' ; then
331     export GOMP_STACKSIZE=400m
332     export OMP_NUM_THREADS=2
333     comm="$comm -mth"
334     else
335     comm="$comm -md cyrus-makedepend"
336     fi
337     comm="$comm -odir $outDir -a jm_c@mitgcm.org"
338     #-- set the optfile (+ mpi & match-precision)
339     MPI=0
340     case $typ in
341     'g77'|'g7a') OPTFILE='../tools/build_options/linux_amd64_g77' ;;
342     'gfo'|'adm'|'oad'|'tlm'|'mth') comm="$comm -match $MC -devel"
343     OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
344     'ifc') comm="$comm -devel"
345     OPTFILE='../tools/build_options/linux_amd64_ifort11' ;;
346     'pgi') OPTFILE='../tools/build_options/linux_amd64_pgf77' ;;
347     'mpa'|'mpi'|'mp2') comm="$comm -match $MC -devel" ; MPI=6
348     OPTFILE='../tools/build_options/linux_amd64_gfortran' ;;
349     *) OPTFILE= ;;
350     esac
351     #-- set MPI command:
352     if test $MPI != 0 ; then
353     if test $typ = 'mp2' ; then MPI=3 ; fi
354     if test $typ = 'mpa' ; then
355     EXE="mpirun -np TR_NPROC ./mitgcmuv_ad"
356     else
357     EXE="mpirun -np TR_NPROC ./mitgcmuv"
358     fi
359     fi
360    
361     #-- set specific Env Vars:
362     if test $typ = 'oad' ; then
363 jmc 1.2 #- for some reasons, "source ScriptFile | tee -a LogFile"
364     # does run the script but does not keep the env-var settings
365     source $HOME/mitgcm/bin/setenv_OpenAD.sh >> $tdir/output_$tt
366 jmc 1.1 fi
367     if test $typ = 'ifc' ; then
368     source /srv/software/intel/intel-11.1.073/bin/ifortvars.sh intel64
369     fi
370     if test $typ = 'pgi' ; then
371     #listT='fizhi-cs-32x32x40 fizhi-cs-aqualev20'
372     export PGI=/srv/software/pgi/pgi-10.9
373     export PATH="$PATH:$PGI/linux86-64/10.9/bin"
374     export LM_LICENSE_FILE=$PGI/license.dat
375     fi
376    
377     if test $sepDir = 0 -a "x$option" = x -a $tt = $firstTst -a $checkOut = 0 ; then
378     #-- cleaning:
379     echo "======================"
380     echo "Cleaning test directories:" | tee -a $tdir/output_$tt
381     cmdCLN="./testreport -clean"
382     echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt
383     $cmdCLN >> $tdir/output_$tt 2>&1
384     echo "======================"
385     echo "" | tee -a $tdir/output_$tt
386     fi
387    
388     #-- run the testreport command:
389     echo -n "Running testreport using:" | tee -a $tdir/output_$tt
390     if test "x$OPTFILE" != x ; then
391     comm="$comm -of=$OPTFILE"
392     fi
393     if test $MPI = 0 ; then echo '' | tee -a $tdir/output_$tt
394     else echo " (EXE='$EXE')" | tee -a $tdir/output_$tt
395     comm="$comm -MPI $MPI -command \"\$EXE\""
396     fi
397     if test "x$option" != x ; then comm="$comm $option" ; fi
398     #if test $typ = 'pgi' ; then comm="$comm -t \"\$listT\"" ; fi
399     echo " \"eval $comm\"" | tee -a $tdir/output_$tt
400     echo "======================"
401     eval $comm >> $tdir/output_$tt 2>&1
402     sed -n "/^An email /,/^======== End of testreport / p" $tdir/output_$tt
403     echo "" | tee -a $tdir/output_$tt
404    
405     #-- also test restart (test 2+2=4)
406     if test $tt != $typ
407     then
408     echo "testing restart using:" | tee -a $tdir/output_$tt
409     comm="../tools/do_tst_2+2 -o $outDir -a jm_c@mitgcm.org"
410     if test $MPI = 0 ; then
411     echo " \"$comm\"" | tee -a $tdir/output_$tt
412     echo "======================"
413     $comm >> $tdir/output_$tt 2>&1
414     else
415     echo " \"$comm -mpi -exe $EXE\"" | tee -a $tdir/output_$tt
416     echo "======================"
417     $comm -mpi -exe "$EXE" >> $tdir/output_$tt 2>&1
418     fi
419     echo ; cat tst_2+2_out.txt
420     echo
421     fi
422     export OMP_NUM_THREADS=1
423    
424     if test $sepDir = 0 ; then
425     #-- cleaning:
426     echo "======================"
427     echo "Cleaning test directories:" | tee -a $tdir/output_$tt
428     if test $tt != $typ ; then
429     cmdCLN="../tools/do_tst_2+2 -clean"
430     echo " clean tst_2+2 running: $cmdCLN" | tee -a $tdir/output_$tt
431     $cmdCLN >> $tdir/output_$tt 2>&1
432     fi
433     if test $tt != $last_Tst ; then
434     cmdCLN="./testreport -clean"
435     echo " clean dir running: $cmdCLN" | tee -a $tdir/output_$tt
436     $cmdCLN >> $tdir/output_$tt 2>&1
437     fi
438     echo "======================"
439     echo
440     fi
441     cd $tdir
442    
443     done

  ViewVC Help
Powered by ViewVC 1.1.22