/[MITgcm]/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_ifort
ViewVC logotype

Annotation of /MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_ifort

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.10 - (hide annotations) (download)
Tue Sep 22 15:51:00 2020 UTC (4 years, 9 months ago) by mlosch
Branch: MAIN
CVS Tags: HEAD
Changes since 1.9: +7 -1 lines
add 2+2 test

1 mlosch 1.1 #!/bin/bash -e
2     # new script for running testreport on ollie.awi.de
3 mlosch 1.3 # - compile and run on compute nodes
4     # - use ssh to call mpack command from the head node ollie0
5 mlosch 1.10 #$Header: /u/gcmpack/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_ifort,v 1.9 2020/06/10 13:58:05 mlosch Exp $
6 mlosch 1.2 #$Name: $
7 mlosch 1.1
8     # needed for cron-job
9 mlosch 1.9 #source /usr/Modules/current/init/bash
10     # this seems to be enough to make the module cmd work
11     source /etc/profile.d/modules.sh
12 mlosch 1.1 #
13     module purge
14 mlosch 1.9 module load intel.compiler intel.mpi
15 mlosch 1.5 module load netcdf
16 mlosch 1.9 # set the netcdf root directory here, because the definitions always
17     # change with different "default" netcdf modules
18     export NETCDF_ROOT=`nc-config --prefix`
19 mlosch 1.1
20 mlosch 1.5 export MPI_ROOT=${I_MPI_ROOT}
21     #(dirname $(dirname `which mpiifort`))
22 mlosch 1.1 export MPI_INC_DIR=${MPI_ROOT}/include64
23    
24 mlosch 1.5 # there is no slurm module anymore and this is the current recommendation to
25     # have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
26     export PATH=${PATH}:/global/opt/slurm/default/bin
27    
28 mlosch 1.1 dNam='ollie'
29     TST_DIR="/work/ollie/mlosch/test_$dNam"
30     echo "start from TST_DIR='$TST_DIR' at: "`date`
31    
32     umask 0022
33    
34     sfx='ifort'
35     RUNIT="runit_"$sfx
36    
37 mlosch 1.2 OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
38 mlosch 1.1 options="-MPI 6"
39 mlosch 1.2 options="$options -odir ${dNam}-i"
40 mlosch 1.1 options="$options -j 6"
41 mlosch 1.8 #options="$options -t global_ocean.cs32x15"
42 mlosch 1.3
43 mlosch 1.6 EXE='srun -n TR_NPROC ./mitgcmuv'
44 mlosch 1.1
45     if [ -e $TST_DIR ]; then
46     echo "$TST_DIR exists"
47 mlosch 1.9 else
48 mlosch 1.1 mkdir $TST_DIR
49     fi
50     cd $TST_DIR
51     HERE=$TST_DIR/output
52     if [ -e $HERE ]; then
53     echo "$HERE"
54 mlosch 1.9 else
55 mlosch 1.1 mkdir $HERE
56     fi
57 mlosch 1.2 OUTFILE=$HERE/slurm_${sfx}.out
58 mlosch 1.1 MYOUTPUT=$HERE/out_$sfx
59     if [ -e $MYOUTPUT ]; then
60     rm -rf $MYOUTPUT
61     fi
62     if [ -e $OUTFILE ]; then
63     rm -r $OUTFILE
64     fi
65 mlosch 1.7 gcmDIR="MITgcm_${sfx}"
66 mlosch 1.6 git_repo='MITgcm'
67     git_code='MITgcm'
68    
69     # checkOut determines how much checking out is being done
70     # checkOut = 3: new clone from GitHub and make a new copy
71     # checkOut = 2: update (git pull) existing repo and make a new copy
72     # checkOut = 1: skip update
73     # checkOut = 0: use existing test code (if available otherwise switch to 1)
74    
75     checkOut=2
76     tdir=${TST_DIR}
77     today=`date +%Y%m%d`
78     TODAY=`date +%d`
79     #tmpFil="/tmp/"`basename $0`".$$"
80     tmpFil=$TST_DIR/error.out
81    
82     if [ $checkOut -le 1 ] ; then
83     if test -e $TST_DIR/${gcmDIR}/doc ; then
84     echo $TST_DIR/${gcmDIR}/doc 'exist'
85     else
86 mlosch 1.9 echo -n "$TST_DIR/${gcmDIR} missing ; "
87 mlosch 1.6 checkOut=2
88     echo "will make a new copy ( checkOut=$checkOut )"
89 mlosch 1.1 fi
90 mlosch 1.6 fi
91    
92     if [ $checkOut -ge 2 ] ; then
93     #---- cleaning:
94     cd $TST_DIR
95    
96     #---- Make a new clone or update existing one:
97     if test -e ${gcmDIR}/.git/config ; then
98     echo "${gcmDIR}/.git/config exist"
99     else
100 mlosch 1.9 echo -n "${gcmDIR}/.git/config missing ; "
101 mlosch 1.6 checkOut=3
102     echo "will get new clone ( checkOut=$checkOut )"
103     fi
104     if [ $checkOut -eq 3 ] ; then
105     echo -n "Removing old clone: $TST_DIR/${gcmDIR} ..."
106     test -e $TST_DIR/${gcmDIR} && rm -rf $TST_DIR/${gcmDIR}
107     echo " done"
108     echo -n "Make a new clone of $git_code from repo: $git_repo ..."
109     git clone https://github.com/$git_repo/${git_code}.git ${gcmDIR} 2> $tmpFil
110     retVal=$?
111     if test $retVal = 0 ; then
112     echo ' --> done!'
113     rm -f $tmpFil
114     else
115     echo " Error: 'git clone' returned: $retVal"
116     cat $tmpFil
117     rm -f $tmpFil
118     exit 2
119     fi
120     else
121 mlosch 1.9 echo "clean tst_2+2 + testreport output"
122     ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
123     ( cd $gcmDIR/verification ; ./testreport -clean )
124 mlosch 1.6 echo "Updating current clone ( $git_code ) ..."
125     ( cd ${gcmDIR}; git checkout master ; git pull )
126     echo ' --> done!'
127 mlosch 1.1 fi
128 mlosch 1.6 else
129     cd $TST_DIR
130 mlosch 1.1 fi
131    
132 mlosch 1.7 cd ${TST_DIR}/${gcmDIR}/verification
133 mlosch 1.1
134 mlosch 1.4 cwd=\`pwd\`
135 mlosch 1.3 SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
136 mlosch 1.9 runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
137 mlosch 1.8 emailaddress="-a jm_c@mitgcm.org"
138 mlosch 1.9 testrestart="../tools/do_tst_2+2 -mpi -exe \"${EXE}\" -o ${dNam}-i -send \"${SENDCMD}\" -sd ${cwd}"
139 mlosch 1.1
140     if [ ! -e $MYOUTPUT ]
141     then
142     touch $MYOUTPUT
143     fi
144    
145 mlosch 1.2 # echo "running testreport like this:"
146     # echo ${runtestreport} -norun
147     # eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"
148 mlosch 1.1
149     # create batch script
150     #
151 mlosch 1.2 JOBNAME=tst$sfx
152 mlosch 1.1 echo "creating batch script $HERE/$RUNIT"
153     cat << EOF >| $HERE/$RUNIT
154 mlosch 1.3 #!/bin/bash
155 mlosch 1.1 #SBATCH --job-name=${JOBNAME}
156     #SBATCH -o ${OUTFILE}
157 mlosch 1.3 #SBATCH --time=6:00:00
158 mlosch 1.1 #SBATCH --ntasks=6
159    
160 mlosch 1.9 # still need this to be able to run a few experiments:
161 mlosch 1.3 # cfc_example, fizhi, tutorial_deep_convection
162     ulimit -s unlimited
163 mlosch 1.1
164 mlosch 1.6 # binds OpenMP task to given cores
165     export OMP_PROC_BIND=TRUE
166    
167     # for debugging
168     export FLEXLM_DIAGNOSTICS=2
169     export FNP_IP_ENV=1
170     export LM_A_CONN_TIMEOUT=99
171    
172 mlosch 1.1 cd \${SLURM_SUBMIT_DIR}
173    
174 mlosch 1.3 cwd=`pwd`
175     echo "running testreport like this in \${cwd}:"
176     echo "${runtestreport} -devel -match 10"
177 mlosch 1.8 ${runtestreport} -devel -match 10 ${emailaddress} >> $MYOUTPUT 2>&1
178 mlosch 1.3
179 mlosch 1.10 echo "running restart test like this in \${cwd}:"
180     echo "${testrestart}"
181     ${testrestart} ${emailaddress} >> $MYOUTPUT 2>&1
182    
183     ../tools/do_tst_2+2 -clean
184    
185 mlosch 1.3 echo "running testreport like this in \${cwd}:"
186     echo "./testreport -clean"
187     ./testreport -clean
188    
189     echo "running testreport like this in \${cwd}:"
190     echo "${runtestreport} -fast -match 10"
191 mlosch 1.8 ${runtestreport} -fast -match 10 ${emailaddress} >> $MYOUTPUT 2>&1
192 mlosch 1.1
193 mlosch 1.9 echo "running restart test like this in \${cwd}:"
194     echo "${testrestart}"
195     ${testrestart} ${emailaddress} >> $MYOUTPUT 2>&1
196     ###${testrestart} -a Martin.Losch@awi.de >> $MYOUTPUT 2>&1
197    
198 mlosch 1.1 EOF
199    
200     chmod a+x $HERE/$RUNIT
201    
202     echo " " >> $MYOUTPUT
203     echo "***********************************************************" >> $MYOUTPUT
204     echo "Submitting this job script:" >> $MYOUTPUT
205     echo "***********************************************************" >> $MYOUTPUT
206     cat $HERE/$RUNIT >> $MYOUTPUT
207     echo "***********************************************************" >> $MYOUTPUT
208     echo "end of job script" >> $MYOUTPUT
209     echo "***********************************************************" >> $MYOUTPUT
210     echo " " >> $MYOUTPUT
211    
212 mlosch 1.6 echo "sbatch $HERE/$RUNIT"
213 mlosch 1.1 sbatch $HERE/$RUNIT
214    
215 mlosch 1.3 # # keep looking for the job in the job queues and wait until it has disappeared
216     # jobruns=somedummy
217     # while [ "${jobruns}"x != x ]
218     # do
219     # sleep 200
220     # jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`
221     # echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
222     # currentexp=`grep Experiment $MYOUTPUT | tail -1`
223     # echo "currently running $currentexp"
224     # done
225    
226     # # workaround for mailing the stuff
227     # echo "mail the stuff"
228    
229     # MPACKCMD=../tools/mpack-1.6/mpack
230     # fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`
231     # echo "fname ${fn}"
232     # tar cf - $fn | gzip > "${fn}.tar.gz"
233 mlosch 1.6 # $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jm_c@mitgcm.org
234 mlosch 1.3 # sleep 2
235     # rm -rf "${fn}.tar.gz"
236 mlosch 1.1
237     echo "end of mitgcmtestreport"

  ViewVC Help
Powered by ViewVC 1.1.22