/[MITgcm]/MITgcm_contrib/test_scripts/svante/test_svante_ifc_adm
ViewVC logotype

Annotation of /MITgcm_contrib/test_scripts/svante/test_svante_ifc_adm

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (hide annotations) (download)
Tue Apr 28 14:34:46 2020 UTC (5 years, 2 months ago) by jmc
Branch: MAIN
Add (TAF) Adjoint test using intel compiler and openmpi

1 jmc 1.1 #!/bin/bash
2     #SBATCH -J ifcAdm_tst
3     #SBATCH -p fdr
4     #SBATCH -t 23:30:00
5     #SBATCH --mem-per-cpu 4000
6     #SBATCH -N 1
7     #SBATCH --tasks-per-node 6
8     #SBATCH -x curly,larry,moe,shemp
9     #SBATCH -e /home/jm_c/test_svante/output/ifcAdm_tst.stderr
10     #SBATCH -o /home/jm_c/test_svante/output/ifcAdm_tst.stdout
11     #SBATCH --no-requeue
12    
13     # $Header: /u/gcmpack/MITgcm_contrib/test_scripts/svante/test_svante_ifc_mpi,v 1.15 2018/06/12 21:42:21 jmc Exp $
14     # $Name: $
15    
16     if test -f /etc/profile.d/modules.sh ; then . /etc/profile.d/modules.sh ; fi
17     if test -f /etc/profile.d/zz_modules.sh ; then . /etc/profile.d/zz_modules.sh ; fi
18     # Note: added "ulimit -s unlimited" in file "~/.bashrc"
19     # to pass big test (the 2 fizhi-cs-* test & adjoint tests) with MPI
20    
21     umask 0022
22     #- to get case insensitive "ls" (and order of tested experiments)
23     export LC_ALL="en_US.UTF-8"
24     echo " running on: "`hostname`
25     headNode='svante-login'
26    
27     dNam='svante'
28     HERE="$HOME/test_${dNam}"
29     OUTP="$HERE/output"; SavD="$HERE/send"
30     SEND="ssh $headNode $SavD/mpack"
31     #TST_DISK="/net/fs09/d1/jm_c"
32     TST_DISK="/scratch/jm_c"
33     TST_DIR="$TST_DISK/test_${dNam}"
34     #- where local copy of code is (need to be consistent with "test_submit_svante"):
35     #srcDIR='.'
36     srcDIR=$HERE
37     srcCode="MITgcm_today"
38    
39     #- following lines are not used here:
40     cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q'
41     #- which GitHub repository to use and how to access it:
42     git_repo='MITgcm'; git_code='MITgcm' ; git_other='verification_other'
43     #git_repo='altMITgcm'; #git_code='MITgcm66h'
44     #--
45     git_repo="https://github.com/$git_repo"
46     #git_repo="git://github.com/$git_repo"
47     #git_repo="git@github.com:$git_repo"
48    
49     typ='' ; addExp='' ; skipExp=''
50     sfx='ifcAdm'; typ='-adm'
51     module add intel/2017.0.1
52     module add openmpi
53     #module add netcdf
54     OPTFILE="../tools/build_options/linux_amd64_ifort11"
55     export GENERIC='on' # <-- to prevent the use of "-xHost" option
56     #- needed for DIVA with MPI:
57     export MPI_INC_DIR=$INC_MPI
58     options="$typ -ncad -MPI 6"
59     #options="-j 2 $options"
60     #- need this to get "staf":
61     export PATH="$PATH:$HOME/bin"
62    
63     gcmDIR="MITgcm_$sfx"
64     dAlt=`date +%d` ; dAlt=`expr $dAlt % 3`
65     if [ $dAlt -eq 1 ] ; then options="$options -fast"
66     else options="$options -devel" ; fi
67    
68     checkOut=2 ; #options="$options -do"
69     #options="$options -nc" ; checkOut=1
70     #options="$options -q" ; checkOut=0
71    
72     echo "cd $TST_DISK ; pwd (x2)"
73     cd $TST_DISK
74     pwd
75     if test ! -d $TST_DIR ; then
76     echo -n "Creating a working dir: $TST_DIR ..."
77     #/bin/rm -rf $TST_DIR
78     mkdir $TST_DIR
79     retVal=$?
80     if test "x$retVal" != x0 ; then
81     echo "Error: unable to make dir: $TST_DIR (err=$retVal ) --> Exit"
82     exit 1
83     fi
84     fi
85     echo "start from TST_DIR='$TST_DIR' at: "`date`
86     cd $TST_DIR
87     pwd
88    
89     NSLOTS=$SLURM_NTASKS
90     THEDATE=`date`
91     echo '********************************************************************************'
92     echo 'Start job '$THEDATE
93     echo 'NSLOTS = '$NSLOTS
94     echo '======= NODELIST ==============================================================='
95     echo $SLURM_NODELIST
96     cat /etc/redhat-release
97     echo '======= env ===================================================================='
98     env | grep SLURM
99     echo '======= modules ================================================================'
100     module list 2>&1
101     echo '================================================================================'
102    
103     #- check for disk space: relative space (99%) or absolute (10.G):
104     dsp=`df -P . | tail -1 | awk '{print $5}' | sed 's/%$//'`
105     if [ $dsp -gt 99 ] ; then
106     #dsp=`df -P . | tail -1 | awk '{print $4}'`
107     #if [ $dsp -le 100000000 ] ; then
108     echo 'Not enough space on this disk => do not run testreport.'
109     df .
110     exit
111     fi
112    
113     if [ $checkOut -eq 1 ] ; then
114     echo "cleaning output from $gcmDIR/verification :"
115     #- remove previous output tar files and tar & remove previous output-dir
116     /bin/rm -f $gcmDIR/verification/??_${dNam}-${sfx}_????????_?.tar.gz
117     ( cd $gcmDIR/verification
118     listD=`ls -1 -d ??_${dNam}-${sfx}_????????_? 2> /dev/null`
119     for dd in $listD
120     do
121     if test -d $dd ; then
122     tar -cf ${dd}".tar" $dd > /dev/null 2>&1 && gzip ${dd}".tar" && /bin/rm -rf $dd
123     retVal=$?
124     if test "x$retVal" != x0 ; then
125     echo "ERROR in tar+gzip prev outp-dir: $dd"
126     echo " on '"`hostname`"' (return val=$retVal) but continue"
127     fi
128     fi
129     done )
130     echo "clean testreport output (+ Makefile_syntax files)"
131     # ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
132     ( cd $gcmDIR/verification ; ./testreport $typ -clean )
133     ( cd $gcmDIR/verification ; rm -f */build/Makefile_syntax )
134     ( cd $gcmDIR/verification ; rm -f */build/port_rand.i */build/ptracers_set_iolabel.i )
135     if test "x$addExp" != x ; then
136     ( cd $gcmDIR/verification
137     listD=`ls -o | grep '^l' | awk '{print $8}' 2> /dev/null`
138     echo " + remove local links: $listD"
139     /bin/rm -f $listD
140     )
141     fi
142     echo "Update $git_code code in dir $gcmDIR :"
143     ( cd $gcmDIR ; git pull ) 2>&1
144     retVal=$?
145     if test "x$retVal" != x0 ; then
146     echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
147     exit
148     fi
149     echo " and checkout master:"
150     ( cd $gcmDIR ; git checkout master -- . ) 2>&1
151     if test "x$addExp" != x ; then
152     echo "Update $git_other code in dir $gcmDIR/$git_other :"
153     ( cd $gcmDIR/$git_other ; git pull ) 2>&1
154     retVal=$?
155     if test "x$retVal" != x0 ; then
156     echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
157     exit
158     fi
159     echo " and checkout master:"
160     ( cd $gcmDIR/$git_other ; git checkout master -- . ) 2>&1
161     fi
162     fi
163    
164     if [ $checkOut -eq 2 ] ; then
165     if test -e $gcmDIR ; then
166     echo -n "Removing working copy: $gcmDIR ..."
167     rm -rf $gcmDIR
168     echo " done"
169     fi
170     # make a local copy (instead of making a new clone):
171     today=`date +%Y%m%d`
172     nCount=0; updFile="${srcDIR}/updated_code"
173     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
174     while [ $today -gt $updDate ] ; do
175     nCount=`expr $nCount + 1`
176     if [ $nCount -gt 40 ] ; then
177     echo " waiting too long (nCount=$nCount) for updated code"
178     echo " today=$today , updDate=$updDate "
179     ls -l $updFile
180     exit
181     fi
182     sleep 60
183     updDate=0 ; test -f $updFile && updDate=`cat $updFile`
184     done
185     ls -l $updFile
186     echo " waited nCount=$nCount for updated code ($updDate) to copy"
187     if test -d $srcDIR/$srcCode ; then
188     echo -n "Make local copy of dir '$srcDIR/$srcCode' to: $gcmDIR ..."
189     cp -pra $srcDIR/$srcCode $gcmDIR
190     echo " done"
191     else echo " dir: $srcDIR/$srcCode missing => exit" ; exit ; fi
192     fi
193    
194     #- change dir to $gcmDIR/verification dir:
195     if test -e $gcmDIR/verification ; then
196     if [ $checkOut -lt 2 ] ; then
197     echo " dir $gcmDIR/verification exist" ; fi
198     cd $gcmDIR/verification
199     for exp2add in $addExp ; do
200     test -r $exp2add && /bin/rm -rf $exp2add
201     if test -d ../$git_other/$exp2add ; then
202     echo " add $exp2add link from $git_other"
203     ln -s ../$git_other/$exp2add .
204     if test $exp2add = 'global_oce_cs32' ; then
205     echo " link dir 'other_input/core2_cnyf' in here"
206     ( cd ../${git_other}/${exp2add}
207     test -L core2_cnyf && /bin/rm -f core2_cnyf
208     ln -s ../../../other_input/core2_cnyf . )
209     fi
210     if test $exp2add = 'global_oce_llc90' ; then
211     echo " link dir 'other_input/gael_oce_llc90_input' to 'input_fields'"
212     ( cd ../${git_other}/${exp2add}
213     test -L input_fields && /bin/rm -f input_fields
214     ln -s ../../../other_input/gael_oce_llc90_input input_fields
215     echo " link dirs: 'core2_cnyf' & 'global_oce_input_fields/*' in input_verifs"
216     test ! -e input_verifs && mkdir input_verifs
217     ( cd input_verifs ; /bin/rm -f *
218     ln -s ../../../../other_input/core2_cnyf .
219     ln -s ../../../../other_input/global_oce_input_fields/* . )
220     )
221     fi
222     else
223     echo " missing dir: $git_other/$exp2add"
224     continue
225     fi
226     done
227     else
228     echo "no dir: $gcmDIR/verification => exit"
229     exit
230     fi
231    
232     if [ $checkOut -ge 1 ] ; then
233     if [ $dAlt -eq 1 ] ; then
234     echo ''
235     #- 1) just compile ("-nr"), using "-j 2" to speed up
236     echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
237     -j 4 -nr -odir ${dNam}-$sfx
238     ./testreport $options -of $OPTFILE -skd "$skipExp" \
239     -j 4 -nr -odir ${dNam}-$sfx
240     nFc=`grep -c '^Y . N N ' tr_out.txt`
241     echo " <= fail to compile $nFc experiments"
242     else
243     echo ''
244     #- 0) just make all module header ( *__genmod.mod files) using modified Makefile
245     echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
246     -j 4 -repl_mk do_make_syntax.sh -obj -dd
247     ./testreport $options -of $OPTFILE -skd "$skipExp" \
248     -j 4 -repl_mk do_make_syntax.sh -obj -dd 2>&1
249     echo ''
250     #- 1) just compile ("-nr"), using "-j 2" to speed up
251     echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
252     -j 4 -nr -q -odir ${dNam}-$sfx
253     ./testreport $options -of $OPTFILE -skd "$skipExp" \
254     -j 4 -nr -q -odir ${dNam}-$sfx
255     nFc=`grep -c '^Y . N N ' tr_out.txt`
256     echo " <= fail to compile $nFc experiments"
257     fi
258     fi
259    
260     echo ''
261     #- 2) run and report results ; also finish to compile those who failed with "-j"
262     echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
263     -q -odir ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jm_c@mitgcm.org
264     ./testreport $options -of $OPTFILE -skd "$skipExp" \
265     -q -odir ${dNam}-$sfx -send "$SEND" -sd $SavD -a jm_c@mitgcm.org
266     # echo ''
267     # #- 3) test restart and report results
268     # echo ../tools/do_tst_2+2 -mpi \
269     # -o ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jm_c@mitgcm.org
270     # ../tools/do_tst_2+2 -mpi \
271     # -o ${dNam}-$sfx -send "$SEND" -sd $SavD -a jm_c@mitgcm.org
272    

  ViewVC Help
Powered by ViewVC 1.1.22