Annotation of /MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray

#!/bin/bash -e
# new script for running testreport on ollie.awi.de
# - compile and run on compute nodes
# - use ssh to call mpack command from the head node ollie0
#$Header: /u/gcmpack/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray,v 1.9 2018/04/16 08:35:39 mlosch Exp $
#$Name:  $

# needed for cron-job
#source /usr/Modules/current/init/bash
source /etc/profile.d/cray_pe.sh
# this seems to be enough to make the module cmd work
source /etc/profile.d/modules.sh
#
module purge
module load PrgEnv-cray
module load netcdf
# set the netcdf root directory here, because the definitions always
# change with different "default" netcdf modules
export NETCDF_ROOT=`nc-config --prefix`
# not sure why I have to set these paths here again
export MPI_ROOT=$(dirname $(dirname `which mpicc`))
# $(dirname `echo $LD_LIBRARY_PATH  | awk -F: '{print $1}'`)
export MPI_INC_DIR=${MPI_ROOT}/include

# there is no slurm module anymore and this is the current recommendation to
# have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
export PATH=${PATH}:/global/opt/slurm/default/bin

dNam='ollie'
TST_DIR="/work/ollie/mlosch/test_$dNam"
echo "start from TST_DIR='$TST_DIR' at: "`date`

umask 0022

sfx='cray'
RUNIT="runit_"$sfx

OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
options="-MPI 6"
options="$options -odir ${dNam}-c"
options="$options -j 6"
#options="$options -t global_ocean.cs32x15"

#EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'
#EXE='srun -n TR_NPROC ./mitgcmuv'
EXE='srun -n TR_NPROC --cpu_bind=cores ./mitgcmuv'

if [ -e $TST_DIR ]; then
    echo "$TST_DIR exists"
else
    mkdir $TST_DIR
fi
cd $TST_DIR
HERE=$TST_DIR/output
if [ -e $HERE ]; then
    echo "$HERE"
else
    mkdir $HERE
fi
OUTFILE=$HERE/slurm_${sfx}.out
MYOUTPUT=$HERE/out_$sfx
if [ -e $MYOUTPUT ]; then
  rm -rf $MYOUTPUT
fi
if [ -e $OUTFILE ]; then
  rm -r $OUTFILE
fi
gcmDIR="MITgcm_${sfx}"
git_repo='MITgcm'
git_code='MITgcm'

# checkOut determines how much checking out is being done
# checkOut = 3: new clone from GitHub and make a new copy
# checkOut = 2: update (git pull) existing repo and make a new copy
# checkOut = 1: skip update
# checkOut = 0: use existing test code (if available otherwise switch to 1)

checkOut=2
tdir=${TST_DIR}
today=`date +%Y%m%d`
TODAY=`date +%d`
#tmpFil="/tmp/"`basename $0`".$$"
tmpFil=$TST_DIR/error.out

if [ $checkOut -le 1 ] ; then
  if test -e $TST_DIR/${gcmDIR}/doc ; then
    echo $TST_DIR/${gcmDIR}/doc 'exist'
  else
    echo -n "$TST_DIR/${gcmDIR} missing ; "
    checkOut=2
    echo "will make a new copy ( checkOut=$checkOut )"
  fi
fi

if [ $checkOut -ge 2 ] ; then
  #---- cleaning:
  cd $TST_DIR

  #---- Make a new clone or update existing one:
  if test -e ${gcmDIR}/.git/config ; then
    echo "${gcmDIR}/.git/config exist"
  else
    echo -n "${gcmDIR}/.git/config missing ; "
    checkOut=3
    echo "will get new clone ( checkOut=$checkOut )"
  fi
  if [ $checkOut -eq 3 ] ; then
    echo -n "Removing old clone: $TST_DIR/${gcmDIR} ..."
    test -e $TST_DIR/${gcmDIR}  &&  rm -rf $TST_DIR/${gcmDIR}
    echo "  done"
    echo -n "Make a new clone of $git_code from repo: $git_repo ..."
    git clone https://github.com/$git_repo/${git_code}.git ${gcmDIR} 2> $tmpFil
    retVal=$?
    if test $retVal = 0 ; then
       echo ' --> done!'
       rm -f $tmpFil
    else
       echo " Error: 'git clone' returned: $retVal"
       cat $tmpFil
       rm -f $tmpFil
       exit 2
    fi
  else
#    echo "clean tst_2+2 + testreport output"
    ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
    ( cd $gcmDIR/verification ; ./testreport -clean )
    echo "Updating current clone ( $git_code ) ..."
    ( cd ${gcmDIR}; git checkout master ; git pull ; git ls-files -d | xargs git checkout -- )
    echo ' --> done!'
  fi
else
  cd $TST_DIR
fi

cd ${TST_DIR}/${gcmDIR}/verification

cwd=\`pwd\`
SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
emailaddress="-a jm_c@mitgcm.org"
testrestart="../tools/do_tst_2+2 -mpi -exe \"${EXE}\" -o ${dNam}-c -send \"${SENDCMD}\" -sd ${cwd}"

if [ ! -e $MYOUTPUT ]
    then
    touch $MYOUTPUT
fi

# echo "running testreport like this:"
# echo ${runtestreport} -norun
# eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"

# create batch script
#
JOBNAME=tst$sfx
echo "creating batch script $HERE/$RUNIT"
cat << EOF >| $HERE/$RUNIT
#!/bin/bash
#SBATCH --job-name=${JOBNAME}
#SBATCH  -o ${OUTFILE}
#SBATCH --time=12:00:00
#SBATCH --ntasks=6

# still need this to be able to run a few experiments:
# cfc_example, fizhi, tutorial_deep_convection
ulimit -s unlimited

# binds OpenMP task to given cores
export OMP_PROC_BIND=TRUE

# for debugging
# export FLEXLM_DIAGNOSTICS=2
# export FNP_IP_ENV=1
# export LM_A_CONN_TIMEOUT=99

cd \${SLURM_SUBMIT_DIR}

cwd=`pwd`
echo "running testreport like this in \${cwd}:"
echo "${runtestreport} -devel -match 10"
${runtestreport} -devel -match 10 ${emailaddress} >> $MYOUTPUT 2>&1

echo "running testreport like this in \${cwd}:"
echo "./testreport -clean"
./testreport -clean

# Hack,hack,hack to avoid running dome:
rm -f dome/results/output.txt
echo "running testreport like this in \${cwd}:"
echo "${runtestreport} -fast -match 10"
${runtestreport} -fast -match 10 ${emailaddress} >> $MYOUTPUT 2>&1

echo "running restart test like this in \${cwd}:"
echo "${testrestart}"
${testrestart} ${emailaddress} >> $MYOUTPUT 2>&1

EOF

chmod a+x $HERE/$RUNIT

echo " " >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "Submitting this job script:" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
cat $HERE/$RUNIT >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "end of job script" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo " " >> $MYOUTPUT

echo "sbatch $HERE/$RUNIT"
sbatch $HERE/$RUNIT

# # keep looking for the job in the job queues and wait until it has disappeared
# jobruns=somedummy
# while [ "${jobruns}"x != x ]
# do
#   sleep 200
#   jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`
#   echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
#   currentexp=`grep Experiment $MYOUTPUT | tail -1`
#   echo "currently running $currentexp"
# done

# # workaround for mailing the stuff
# echo "mail the stuff"

# MPACKCMD=../tools/mpack-1.6/mpack
# fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`
# echo "fname ${fn}"
# tar cf - $fn | gzip > "${fn}.tar.gz"
# $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jm_c@mitgcm.org
# sleep 2
# rm -rf "${fn}.tar.gz"

echo "end of mitgcmtestreport"
1	mlosch	1.1	#!/bin/bash -e
2			# new script for running testreport on ollie.awi.de
3	mlosch	1.4	# - compile and run on compute nodes
4	mlosch	1.3	# - use ssh to call mpack command from the head node ollie0
5	mlosch	1.10	#$Header: /u/gcmpack/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray,v 1.9 2018/04/16 08:35:39 mlosch Exp $
6	mlosch	1.2	#$Name: $
7	mlosch	1.1
8			# needed for cron-job
9	mlosch	1.10	#source /usr/Modules/current/init/bash
10	mlosch	1.1	source /etc/profile.d/cray_pe.sh
11	mlosch	1.10	# this seems to be enough to make the module cmd work
12			source /etc/profile.d/modules.sh
13	mlosch	1.1	#
14			module purge
15	mlosch	1.10	module load PrgEnv-cray
16	mlosch	1.7	module load netcdf
17	mlosch	1.10	# set the netcdf root directory here, because the definitions always
18			# change with different "default" netcdf modules
19			export NETCDF_ROOT=`nc-config --prefix`
20	mlosch	1.8	# not sure why I have to set these paths here again
21	mlosch	1.7	export MPI_ROOT=$(dirname $(dirname `which mpicc`))
22			# $(dirname `echo $LD_LIBRARY_PATH \| awk -F: '{print $1}'`)
23	mlosch	1.6	export MPI_INC_DIR=${MPI_ROOT}/include
24
25	mlosch	1.7	# there is no slurm module anymore and this is the current recommendation to
26			# have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
27			export PATH=${PATH}:/global/opt/slurm/default/bin
28	mlosch	1.1
29			dNam='ollie'
30			TST_DIR="/work/ollie/mlosch/test_$dNam"
31			echo "start from TST_DIR='$TST_DIR' at: "`date`
32
33			umask 0022
34
35	mlosch	1.2	sfx='cray'
36	mlosch	1.1	RUNIT="runit_"$sfx
37
38	mlosch	1.2	OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
39	mlosch	1.1	options="-MPI 6"
40	mlosch	1.2	options="$options -odir ${dNam}-c"
41	mlosch	1.8	options="$options -j 6"
42	mlosch	1.3	#options="$options -t global_ocean.cs32x15"
43
44	mlosch	1.8	#EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'
45			#EXE='srun -n TR_NPROC ./mitgcmuv'
46			EXE='srun -n TR_NPROC --cpu_bind=cores ./mitgcmuv'
47	mlosch	1.1
48			if [ -e $TST_DIR ]; then
49			echo "$TST_DIR exists"
50	mlosch	1.10	else
51	mlosch	1.1	mkdir $TST_DIR
52			fi
53			cd $TST_DIR
54			HERE=$TST_DIR/output
55			if [ -e $HERE ]; then
56			echo "$HERE"
57	mlosch	1.10	else
58	mlosch	1.1	mkdir $HERE
59			fi
60	mlosch	1.2	OUTFILE=$HERE/slurm_${sfx}.out
61	mlosch	1.1	MYOUTPUT=$HERE/out_$sfx
62			if [ -e $MYOUTPUT ]; then
63			rm -rf $MYOUTPUT
64			fi
65			if [ -e $OUTFILE ]; then
66			rm -r $OUTFILE
67			fi
68	mlosch	1.8	gcmDIR="MITgcm_${sfx}"
69			git_repo='MITgcm'
70			git_code='MITgcm'
71
72			# checkOut determines how much checking out is being done
73			# checkOut = 3: new clone from GitHub and make a new copy
74			# checkOut = 2: update (git pull) existing repo and make a new copy
75			# checkOut = 1: skip update
76			# checkOut = 0: use existing test code (if available otherwise switch to 1)
77
78			checkOut=2
79			tdir=${TST_DIR}
80			today=`date +%Y%m%d`
81			TODAY=`date +%d`
82			#tmpFil="/tmp/"`basename $0`".$$"
83			tmpFil=$TST_DIR/error.out
84
85			if [ $checkOut -le 1 ] ; then
86			if test -e $TST_DIR/${gcmDIR}/doc ; then
87			echo $TST_DIR/${gcmDIR}/doc 'exist'
88			else
89	mlosch	1.10	echo -n "$TST_DIR/${gcmDIR} missing ; "
90	mlosch	1.8	checkOut=2
91			echo "will make a new copy ( checkOut=$checkOut )"
92	mlosch	1.1	fi
93	mlosch	1.8	fi
94
95			if [ $checkOut -ge 2 ] ; then
96			#---- cleaning:
97			cd $TST_DIR
98
99			#---- Make a new clone or update existing one:
100			if test -e ${gcmDIR}/.git/config ; then
101			echo "${gcmDIR}/.git/config exist"
102			else
103	mlosch	1.10	echo -n "${gcmDIR}/.git/config missing ; "
104	mlosch	1.8	checkOut=3
105			echo "will get new clone ( checkOut=$checkOut )"
106	mlosch	1.1	fi
107	mlosch	1.8	if [ $checkOut -eq 3 ] ; then
108			echo -n "Removing old clone: $TST_DIR/${gcmDIR} ..."
109			test -e $TST_DIR/${gcmDIR} && rm -rf $TST_DIR/${gcmDIR}
110			echo " done"
111			echo -n "Make a new clone of $git_code from repo: $git_repo ..."
112			git clone https://github.com/$git_repo/${git_code}.git ${gcmDIR} 2> $tmpFil
113			retVal=$?
114			if test $retVal = 0 ; then
115			echo ' --> done!'
116			rm -f $tmpFil
117			else
118			echo " Error: 'git clone' returned: $retVal"
119			cat $tmpFil
120			rm -f $tmpFil
121			exit 2
122			fi
123			else
124	mlosch	1.10	# echo "clean tst_2+2 + testreport output"
125			( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
126			( cd $gcmDIR/verification ; ./testreport -clean )
127	mlosch	1.8	echo "Updating current clone ( $git_code ) ..."
128	mlosch	1.10	( cd ${gcmDIR}; git checkout master ; git pull ; git ls-files -d \| xargs git checkout -- )
129	mlosch	1.8	echo ' --> done!'
130			fi
131			else
132			cd $TST_DIR
133	mlosch	1.1	fi
134
135	mlosch	1.8	cd ${TST_DIR}/${gcmDIR}/verification
136	mlosch	1.1
137	mlosch	1.6	cwd=\`pwd\`
138	mlosch	1.3	SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
139	mlosch	1.10	runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
140	mlosch	1.8	emailaddress="-a jm_c@mitgcm.org"
141	mlosch	1.10	testrestart="../tools/do_tst_2+2 -mpi -exe \"${EXE}\" -o ${dNam}-c -send \"${SENDCMD}\" -sd ${cwd}"
142	mlosch	1.1
143			if [ ! -e $MYOUTPUT ]
144			then
145			touch $MYOUTPUT
146			fi
147
148	mlosch	1.2	# echo "running testreport like this:"
149			# echo ${runtestreport} -norun
150			# eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"
151	mlosch	1.1
152			# create batch script
153			#
154	mlosch	1.2	JOBNAME=tst$sfx
155	mlosch	1.1	echo "creating batch script $HERE/$RUNIT"
156			cat << EOF >\| $HERE/$RUNIT
157	mlosch	1.3	#!/bin/bash
158	mlosch	1.1	#SBATCH --job-name=${JOBNAME}
159			#SBATCH -o ${OUTFILE}
160	mlosch	1.3	#SBATCH --time=12:00:00
161	mlosch	1.1	#SBATCH --ntasks=6
162
163	mlosch	1.10	# still need this to be able to run a few experiments:
164	mlosch	1.3	# cfc_example, fizhi, tutorial_deep_convection
165			ulimit -s unlimited
166	mlosch	1.1
167	mlosch	1.8	# binds OpenMP task to given cores
168			export OMP_PROC_BIND=TRUE
169
170			# for debugging
171	mlosch	1.7	# export FLEXLM_DIAGNOSTICS=2
172			# export FNP_IP_ENV=1
173			# export LM_A_CONN_TIMEOUT=99
174
175	mlosch	1.1	cd \${SLURM_SUBMIT_DIR}
176
177	mlosch	1.3	cwd=`pwd`
178			echo "running testreport like this in \${cwd}:"
179			echo "${runtestreport} -devel -match 10"
180	mlosch	1.8	${runtestreport} -devel -match 10 ${emailaddress} >> $MYOUTPUT 2>&1
181	mlosch	1.3
182			echo "running testreport like this in \${cwd}:"
183			echo "./testreport -clean"
184			./testreport -clean
185
186	mlosch	1.10	# Hack,hack,hack to avoid running dome:
187			rm -f dome/results/output.txt
188	mlosch	1.3	echo "running testreport like this in \${cwd}:"
189			echo "${runtestreport} -fast -match 10"
190	mlosch	1.8	${runtestreport} -fast -match 10 ${emailaddress} >> $MYOUTPUT 2>&1
191	mlosch	1.1
192	mlosch	1.10	echo "running restart test like this in \${cwd}:"
193			echo "${testrestart}"
194			${testrestart} ${emailaddress} >> $MYOUTPUT 2>&1
195
196	mlosch	1.1	EOF
197
198			chmod a+x $HERE/$RUNIT
199
200			echo " " >> $MYOUTPUT
201			echo "***********************************************************" >> $MYOUTPUT
202			echo "Submitting this job script:" >> $MYOUTPUT
203			echo "***********************************************************" >> $MYOUTPUT
204			cat $HERE/$RUNIT >> $MYOUTPUT
205			echo "***********************************************************" >> $MYOUTPUT
206			echo "end of job script" >> $MYOUTPUT
207			echo "***********************************************************" >> $MYOUTPUT
208			echo " " >> $MYOUTPUT
209
210	mlosch	1.8	echo "sbatch $HERE/$RUNIT"
211	mlosch	1.1	sbatch $HERE/$RUNIT
212
213	mlosch	1.3	# # keep looking for the job in the job queues and wait until it has disappeared
214			# jobruns=somedummy
215			# while [ "${jobruns}"x != x ]
216			# do
217			# sleep 200
218			# jobruns=`squeue --noheader -u mlosch \| grep "$JOBNAME" \| awk '{print $1}'`
219			# echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
220			# currentexp=`grep Experiment $MYOUTPUT \| tail -1`
221			# echo "currently running $currentexp"
222			# done
223
224			# # workaround for mailing the stuff
225			# echo "mail the stuff"
226
227			# MPACKCMD=../tools/mpack-1.6/mpack
228			# fn=`ls -dtr tr_$dNam* \| grep -v tar.gz \| tail -1`
229			# echo "fname ${fn}"
230			# tar cf - $fn \| gzip > "${fn}.tar.gz"
231	mlosch	1.8	# $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jm_c@mitgcm.org
232	mlosch	1.3	# sleep 2
233			# rm -rf "${fn}.tar.gz"
234	mlosch	1.1
235			echo "end of mitgcmtestreport"