Annotation of /MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray

#!/bin/bash -e
# new script for running testreport on ollie.awi.de
# - compile and run on compute nodes
# - use ssh to call mpack command from the head node ollie0
#$Header: /u/gcmpack/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray,v 1.6 2016/08/29 15:08:54 mlosch Exp $
#$Name:  $

# needed for cron-job
source /usr/Modules/3.2.10/init/bash
source /etc/profile.d/cray_pe.sh
source /etc/profile.d/AWI.sh
#

module purge
module load craype-broadwell
module load PrgEnv-cray
# to me this looks like a terrible hack, but that is what we are supposed 
# to use now
module unload craype-network-infiniband cudatoolkit mvapich2_cce
module load craype-network-opa
module load mvapich2_cce
module load netcdf
#export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}:
# not sure why I have set these paths here again
export MPI_ROOT=$(dirname $(dirname `which mpicc`))
# $(dirname `echo $LD_LIBRARY_PATH  | awk -F: '{print $1}'`)
export MPI_INC_DIR=${MPI_ROOT}/include

# there is no slurm module anymore and this is the current recommendation to
# have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
export PATH=${PATH}:/global/opt/slurm/default/bin

dNam='ollie'
TST_DIR="/work/ollie/mlosch/test_$dNam"
echo "start from TST_DIR='$TST_DIR' at: "`date`

umask 0022

sfx='cray'
RUNIT="runit_"$sfx

OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
options="-MPI 6"
options="$options -odir ${dNam}-c"
#options="$options -j 6"
#options="$options -t global_ocean.cs32x15"

#EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'
EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'

if [ -e $TST_DIR ]; then
    echo "$TST_DIR exists"
else 
    mkdir $TST_DIR
fi
cd $TST_DIR
HERE=$TST_DIR/output
if [ -e $HERE ]; then
    echo "$HERE"
else 
    mkdir $HERE
fi
OUTFILE=$HERE/slurm_${sfx}.out
MYOUTPUT=$HERE/out_$sfx
if [ -e $MYOUTPUT ]; then
  rm -rf $MYOUTPUT
fi
if [ -e $OUTFILE ]; then
  rm -r $OUTFILE
fi
gcmDIR="MITgcm_$sfx"
checkOut=1
if [ $checkOut -eq 1 ] ; then
 if test -d $gcmDIR/CVS ; then
  /bin/rm -rf $gcmDIR/verification/??_${dNam}_????????_?
  ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
  echo "cvs update of dir $gcmDIR :"
  ( cd $gcmDIR ; cvs -q -d :pserver:cvsanon@mitgcm.org:/u/gcmpack update -P -d ) 2>&1
  RETVAL=$?
  if test "x$RETVAL" != x0 ; then
   echo "cvs update on '"`hostname`"' failed (return val=$RETVAL) => exit"
   exit
  fi
 else
  echo "no dir: $gcmDIR/CVS => try a fresh check-out"
  checkOut=2
 fi
fi
if [ $checkOut -eq 2 ] ; then
  if test -e $gcmDIR ; then
   rm -rf $gcmDIR
  fi
# cvs co MITgcm
  cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co -P -d $gcmDIR MITgcm > /dev/null 2>&1
  /usr/bin/find $gcmDIR -type d | xargs chmod g+rxs
  /usr/bin/find $gcmDIR -type f | xargs chmod g+r
fi

cd ${gcmDIR}/verification

cwd=\`pwd\`
SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}" 

if [ ! -e $MYOUTPUT ]
    then
    touch $MYOUTPUT
fi

# echo "running testreport like this:"
# echo ${runtestreport} -norun
# eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"

# create batch script
#
JOBNAME=tst$sfx
echo "creating batch script $HERE/$RUNIT"
cat << EOF >| $HERE/$RUNIT
#!/bin/bash
#SBATCH --job-name=${JOBNAME}
#SBATCH  -o ${OUTFILE}
#SBATCH --time=12:00:00
#SBATCH --ntasks=6

# still need this to be able to run a few experiments: 
# cfc_example, fizhi, tutorial_deep_convection
ulimit -s unlimited

# # for debugging
# export FLEXLM_DIAGNOSTICS=2
# export FNP_IP_ENV=1
# export LM_A_CONN_TIMEOUT=99

cd \${SLURM_SUBMIT_DIR}

cwd=`pwd`
echo "running testreport like this in \${cwd}:"
echo "${runtestreport} -devel -match 10"
${runtestreport} -devel -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1

echo "running testreport like this in \${cwd}:"
echo "./testreport -clean"
./testreport -clean

echo "running testreport like this in \${cwd}:"
echo "${runtestreport} -fast -match 10"
${runtestreport} -fast -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1

EOF

chmod a+x $HERE/$RUNIT

echo " " >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "Submitting this job script:" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
cat $HERE/$RUNIT >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "end of job script" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo " " >> $MYOUTPUT

sbatch $HERE/$RUNIT

# # keep looking for the job in the job queues and wait until it has disappeared
# jobruns=somedummy
# while [ "${jobruns}"x != x ]
# do
#   sleep 200
#   jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`
#   echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
#   currentexp=`grep Experiment $MYOUTPUT | tail -1`
#   echo "currently running $currentexp"
# done

# # workaround for mailing the stuff
# echo "mail the stuff"

# MPACKCMD=../tools/mpack-1.6/mpack
# fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`
# echo "fname ${fn}"
# tar cf - $fn | gzip > "${fn}.tar.gz"
# $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org
# sleep 2
# rm -rf "${fn}.tar.gz"

echo "end of mitgcmtestreport"


1	mlosch	1.1	#!/bin/bash -e
2			# new script for running testreport on ollie.awi.de
3	mlosch	1.4	# - compile and run on compute nodes
4	mlosch	1.3	# - use ssh to call mpack command from the head node ollie0
5	mlosch	1.7	#$Header: /u/gcmpack/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray,v 1.6 2016/08/29 15:08:54 mlosch Exp $
6	mlosch	1.2	#$Name: $
7	mlosch	1.1
8			# needed for cron-job
9			source /usr/Modules/3.2.10/init/bash
10			source /etc/profile.d/cray_pe.sh
11			source /etc/profile.d/AWI.sh
12			#
13	mlosch	1.6
14	mlosch	1.1	module purge
15			module load craype-broadwell
16	mlosch	1.6	module load PrgEnv-cray
17	mlosch	1.7	# to me this looks like a terrible hack, but that is what we are supposed
18			# to use now
19			module unload craype-network-infiniband cudatoolkit mvapich2_cce
20			module load craype-network-opa
21			module load mvapich2_cce
22			module load netcdf
23	mlosch	1.6	#export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}:
24			# not sure why I have set these paths here again
25	mlosch	1.7	export MPI_ROOT=$(dirname $(dirname `which mpicc`))
26			# $(dirname `echo $LD_LIBRARY_PATH \| awk -F: '{print $1}'`)
27	mlosch	1.6	export MPI_INC_DIR=${MPI_ROOT}/include
28
29	mlosch	1.7	# there is no slurm module anymore and this is the current recommendation to
30			# have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
31			export PATH=${PATH}:/global/opt/slurm/default/bin
32	mlosch	1.1
33			dNam='ollie'
34			TST_DIR="/work/ollie/mlosch/test_$dNam"
35			echo "start from TST_DIR='$TST_DIR' at: "`date`
36
37			umask 0022
38
39	mlosch	1.2	sfx='cray'
40	mlosch	1.1	RUNIT="runit_"$sfx
41
42	mlosch	1.2	OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
43	mlosch	1.1	options="-MPI 6"
44	mlosch	1.2	options="$options -odir ${dNam}-c"
45	mlosch	1.5	#options="$options -j 6"
46	mlosch	1.3	#options="$options -t global_ocean.cs32x15"
47
48	mlosch	1.1	#EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'
49			EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'
50
51			if [ -e $TST_DIR ]; then
52			echo "$TST_DIR exists"
53			else
54			mkdir $TST_DIR
55			fi
56			cd $TST_DIR
57			HERE=$TST_DIR/output
58			if [ -e $HERE ]; then
59			echo "$HERE"
60			else
61			mkdir $HERE
62			fi
63	mlosch	1.2	OUTFILE=$HERE/slurm_${sfx}.out
64	mlosch	1.1	MYOUTPUT=$HERE/out_$sfx
65			if [ -e $MYOUTPUT ]; then
66			rm -rf $MYOUTPUT
67			fi
68			if [ -e $OUTFILE ]; then
69			rm -r $OUTFILE
70			fi
71			gcmDIR="MITgcm_$sfx"
72			checkOut=1
73			if [ $checkOut -eq 1 ] ; then
74			if test -d $gcmDIR/CVS ; then
75			/bin/rm -rf $gcmDIR/verification/??_${dNam}_????????_?
76			( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
77			echo "cvs update of dir $gcmDIR :"
78			( cd $gcmDIR ; cvs -q -d :pserver:cvsanon@mitgcm.org:/u/gcmpack update -P -d ) 2>&1
79			RETVAL=$?
80			if test "x$RETVAL" != x0 ; then
81			echo "cvs update on '"`hostname`"' failed (return val=$RETVAL) => exit"
82			exit
83			fi
84			else
85			echo "no dir: $gcmDIR/CVS => try a fresh check-out"
86			checkOut=2
87			fi
88			fi
89			if [ $checkOut -eq 2 ] ; then
90			if test -e $gcmDIR ; then
91			rm -rf $gcmDIR
92			fi
93			# cvs co MITgcm
94			cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co -P -d $gcmDIR MITgcm > /dev/null 2>&1
95			/usr/bin/find $gcmDIR -type d \| xargs chmod g+rxs
96			/usr/bin/find $gcmDIR -type f \| xargs chmod g+r
97			fi
98
99			cd ${gcmDIR}/verification
100
101	mlosch	1.6	cwd=\`pwd\`
102	mlosch	1.3	SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
103			runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
104	mlosch	1.1
105			if [ ! -e $MYOUTPUT ]
106			then
107			touch $MYOUTPUT
108			fi
109
110	mlosch	1.2	# echo "running testreport like this:"
111			# echo ${runtestreport} -norun
112			# eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"
113	mlosch	1.1
114			# create batch script
115			#
116	mlosch	1.2	JOBNAME=tst$sfx
117	mlosch	1.1	echo "creating batch script $HERE/$RUNIT"
118			cat << EOF >\| $HERE/$RUNIT
119	mlosch	1.3	#!/bin/bash
120	mlosch	1.1	#SBATCH --job-name=${JOBNAME}
121			#SBATCH -o ${OUTFILE}
122	mlosch	1.3	#SBATCH --time=12:00:00
123	mlosch	1.1	#SBATCH --ntasks=6
124
125	mlosch	1.3	# still need this to be able to run a few experiments:
126			# cfc_example, fizhi, tutorial_deep_convection
127			ulimit -s unlimited
128	mlosch	1.1
129	mlosch	1.7	# # for debugging
130			# export FLEXLM_DIAGNOSTICS=2
131			# export FNP_IP_ENV=1
132			# export LM_A_CONN_TIMEOUT=99
133
134	mlosch	1.1	cd \${SLURM_SUBMIT_DIR}
135
136	mlosch	1.3	cwd=`pwd`
137			echo "running testreport like this in \${cwd}:"
138			echo "${runtestreport} -devel -match 10"
139			${runtestreport} -devel -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1
140
141			echo "running testreport like this in \${cwd}:"
142			echo "./testreport -clean"
143			./testreport -clean
144
145			echo "running testreport like this in \${cwd}:"
146			echo "${runtestreport} -fast -match 10"
147			${runtestreport} -fast -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1
148	mlosch	1.1
149			EOF
150
151			chmod a+x $HERE/$RUNIT
152
153			echo " " >> $MYOUTPUT
154			echo "***********************************************************" >> $MYOUTPUT
155			echo "Submitting this job script:" >> $MYOUTPUT
156			echo "***********************************************************" >> $MYOUTPUT
157			cat $HERE/$RUNIT >> $MYOUTPUT
158			echo "***********************************************************" >> $MYOUTPUT
159			echo "end of job script" >> $MYOUTPUT
160			echo "***********************************************************" >> $MYOUTPUT
161			echo " " >> $MYOUTPUT
162
163			sbatch $HERE/$RUNIT
164
165	mlosch	1.3	# # keep looking for the job in the job queues and wait until it has disappeared
166			# jobruns=somedummy
167			# while [ "${jobruns}"x != x ]
168			# do
169			# sleep 200
170			# jobruns=`squeue --noheader -u mlosch \| grep "$JOBNAME" \| awk '{print $1}'`
171			# echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
172			# currentexp=`grep Experiment $MYOUTPUT \| tail -1`
173			# echo "currently running $currentexp"
174			# done
175
176			# # workaround for mailing the stuff
177			# echo "mail the stuff"
178
179			# MPACKCMD=../tools/mpack-1.6/mpack
180			# fn=`ls -dtr tr_$dNam* \| grep -v tar.gz \| tail -1`
181			# echo "fname ${fn}"
182			# tar cf - $fn \| gzip > "${fn}.tar.gz"
183			# $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org
184			# sleep 2
185			# rm -rf "${fn}.tar.gz"
186	mlosch	1.1
187			echo "end of mitgcmtestreport"
188
189
190
191
192