Annotation of /MITgcm_contrib/test_scripts/sxace/mitgcmtestreport

#!/bin/bash
# new script for running testreport on stan1.awi.de
# - split the testreport into 3 steps:
# 1/ compiling on head node (tx7.awi.de), with -norun option
# 2/ running on compute node (using PBS qsub), with -runonly option
# 3/ evaluating result on head node with -runonly option
# 
# Notes: 
# - step 2 leads to many error messages, because the OS on the compute
#   nodes does not have the appropriate shell tools, modifying the
#   runonly option to skip the evalution step would be nice but not
#   necessary; you'll just have to live with the error messages
# - step 3 assumes that all experiments have been run successfully, i.e.
#   that the output files are up-to-date.
#   if not, testreport will try to run the sx ace-code on the tx7 frontend
#   which will fail inevitably and produce more errors, maybe we can
#   have a flag that skips everything but the evaluation step to avoid this
# $Header: /u/gcmpack/MITgcm_contrib/test_scripts/sxace/mitgcmtestreport,v 1.2 2015/09/01 08:03:35 mlosch Exp $
# $Name:  $

# for some reason the module command is not available in a bash script on
# this computer so we have to create it here
#module () { eval `/usr/bin/modulecmd bash $*` ; }
# alternatively we can source this script that contains all relevant
# definitions
source /usr/share/Modules/init/bash
#module use --append /sx8/user2/awisoft/modulefiles
# load latest compilers:
#module load sxf90/460
#module load sxc++/094
module load sxf90
module load sxc++
module load sxmpi
module load sxnetcdf
#
# make sure that we have qsub and qstat
#export PATH=${PATH}:/usr/bin/nqsII
source /etc/profile.d/nec.sh
#
VENDOR=sxf90
RUNIT="runit_"$VENDOR
HERE=`pwd`
EXE='mpirun -np TR_NPROC ./mitgcmuv'
NPROCS=2
MPI="-MPI $NPROCS"
OUTFILE=$HOME/out_${VENDOR}
MYOUTPUT=$HOME/testreport_${VENDOR}
OUTFILE=out_${VENDOR}
JOBNAME=test_ace
JOBSCRIPT=job_${VENDOR}
selectexperiment='-t exp2'
selectexperiment=''
# download code into this directory
TDIR=/ace/user/mlosch/tmp_$VENDOR

OPTFILE=../tools/build_options/SUPER-UX_SX-ACE_sxf90_awi
#OPTFILE=/home/ace/mlosch/MITgcm/tools/build_options/SUPER-UX_SX-ACE_sxf90_awi

RUNTESTREPORT="./testreport $MPI -of=${OPTFILE} $selectexperiment -small_f"
#
# create batch script
#
cat << EOF > $HERE/$JOBSCRIPT
#PBS -q ace-r                            # job queue
#PBS -N $JOBNAME                         # give the job a name
#PBS -l cpunum_job=$NPROCS               # cpus per node
#PBS -l elapstim_req=2:00:00
#PBS -l cputim_job=2:00:00               # time limit
#PBS -l memsz_job=32gb                   # max accumulated memory, we need this much because of many netcdf files
#PBS -j o                                # join i/o
#PBS -S /bin/sh
#PBS -o $OUTFILE                         # o Where to write output
#

cd \${PBS_O_WORKDIR}
$RUNTESTREPORT -runonly -command "$EXE" >> $MYOUTPUT 2>&1

EOF

# clean up old testreport output
if [ -e $MYOUTPUT ]; then
  rm -rf $MYOUTPUT
fi
if [ -e $OUTFILE ]; then
  rm -r $OUTFILE
fi
if [ -e $TDIR/MITgcm ]; then
# if availabe just update
    cd $TDIR/MITgcm
    cvs -q update -d -P >> cvs_co.log 2>&1
    if [ "$?" != "0" ]; then
	cat cvs_co.log
    fi
else
# create directory and download code
    if [ -e $TDIR ]; then
	rm -rf $TDIR
    fi
    mkdir $TDIR
    cd $TDIR
    cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm > cvs_co.log 2>&1
    if [ "$?" != "0" ]; then
	cat cvs_co.log
    fi
fi

cd $TDIR/MITgcm/verification

# make sure that we do not use the cross compiler for testreport
unset CC
# make sure that do use the cross compiler for testreport
#export CC=sxcc

$RUNTESTREPORT  -j 8 -norun > $MYOUTPUT 2>&1

if [ "$?" != "0" ]
    then
    echo "something wrong with testreport"
    echo "keeping the working directory"
#else
#  echo "check restarts"
#  echo ../tools/do_tst_2+2 -mpi -exe \"$HERE/$RUNIT\" -a NONE
#  ../tools/do_tst_2+2 -mpi -exe $HERE/$RUNIT -a NONE
# everything OK: delete working directory
#  rm -rf $TDIR
fi

if [ ! -e $MYOUTPUT ]
    then
    touch $MYOUTPUT
fi

echo " " >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "Submitting this job script:" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
cat $HERE/$JOBSCRIPT >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "end of job script" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo " " >> $MYOUTPUT

# now submit the job that actually runs all the experiments in one go
qsub $HERE/$JOBSCRIPT
# keep looking for the job in the job queues and wait until has disappeared
jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`
while [ "${jobruns}"x != x ]
do 
  sleep 200
  jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`
  echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
  currentexp=`grep Experiment $MYOUTPUT | tail -1`
  echo "currently running $currentexp"
done

# after running the experiments on the compute node run testreport
# for a third time to evaluate results on the head node again
echo " " >> $MYOUTPUT
echo "now run testreport for a final time to evaluate results:" >> $MYOUTPUT
echo "$RUNTESTREPORT -match 10 -runonly" >> $MYOUTPUT
#$RUNTESTREPORT -match 10 -runonly >> $MYOUTPUT 2>&1
$RUNTESTREPORT -match 10 -runonly \
    -a "jm_c@mitgcm.org" >> $MYOUTPUT 2>&1
#   -a "jm_c@mitgcm.org, Martin.Losch@awi.de" >> $MYOUTPUT 2>&1

echo "end of mitgcmtestreport"
1	mlosch	1.1	#!/bin/bash
2			# new script for running testreport on stan1.awi.de
3			# - split the testreport into 3 steps:
4			# 1/ compiling on head node (tx7.awi.de), with -norun option
5			# 2/ running on compute node (using PBS qsub), with -runonly option
6			# 3/ evaluating result on head node with -runonly option
7			#
8			# Notes:
9			# - step 2 leads to many error messages, because the OS on the compute
10			# nodes does not have the appropriate shell tools, modifying the
11			# runonly option to skip the evalution step would be nice but not
12			# necessary; you'll just have to live with the error messages
13			# - step 3 assumes that all experiments have been run successfully, i.e.
14			# that the output files are up-to-date.
15			# if not, testreport will try to run the sx ace-code on the tx7 frontend
16			# which will fail inevitably and produce more errors, maybe we can
17			# have a flag that skips everything but the evaluation step to avoid this
18	mlosch	1.3	# $Header: /u/gcmpack/MITgcm_contrib/test_scripts/sxace/mitgcmtestreport,v 1.2 2015/09/01 08:03:35 mlosch Exp $
19	mlosch	1.1	# $Name: $
20
21			# for some reason the module command is not available in a bash script on
22			# this computer so we have to create it here
23			#module () { eval `/usr/bin/modulecmd bash $*` ; }
24			# alternatively we can source this script that contains all relevant
25			# definitions
26			source /usr/share/Modules/init/bash
27			#module use --append /sx8/user2/awisoft/modulefiles
28			# load latest compilers:
29			#module load sxf90/460
30			#module load sxc++/094
31			module load sxf90
32			module load sxc++
33			module load sxmpi
34			module load sxnetcdf
35			#
36			# make sure that we have qsub and qstat
37			#export PATH=${PATH}:/usr/bin/nqsII
38			source /etc/profile.d/nec.sh
39			#
40			VENDOR=sxf90
41			RUNIT="runit_"$VENDOR
42			HERE=`pwd`
43			EXE='mpirun -np TR_NPROC ./mitgcmuv'
44			NPROCS=2
45			MPI="-MPI $NPROCS"
46			OUTFILE=$HOME/out_${VENDOR}
47			MYOUTPUT=$HOME/testreport_${VENDOR}
48			OUTFILE=out_${VENDOR}
49			JOBNAME=test_ace
50			JOBSCRIPT=job_${VENDOR}
51			selectexperiment='-t exp2'
52			selectexperiment=''
53			# download code into this directory
54	mlosch	1.2	TDIR=/ace/user/mlosch/tmp_$VENDOR
55	mlosch	1.1
56			OPTFILE=../tools/build_options/SUPER-UX_SX-ACE_sxf90_awi
57	mlosch	1.2	#OPTFILE=/home/ace/mlosch/MITgcm/tools/build_options/SUPER-UX_SX-ACE_sxf90_awi
58	mlosch	1.1
59			RUNTESTREPORT="./testreport $MPI -of=${OPTFILE} $selectexperiment -small_f"
60			#
61			# create batch script
62			#
63			cat << EOF > $HERE/$JOBSCRIPT
64	mlosch	1.2	#PBS -q ace-r # job queue
65	mlosch	1.1	#PBS -N $JOBNAME # give the job a name
66			#PBS -l cpunum_job=$NPROCS # cpus per node
67	mlosch	1.2	#PBS -l elapstim_req=2:00:00
68	mlosch	1.1	#PBS -l cputim_job=2:00:00 # time limit
69			#PBS -l memsz_job=32gb # max accumulated memory, we need this much because of many netcdf files
70			#PBS -j o # join i/o
71			#PBS -S /bin/sh
72			#PBS -o $OUTFILE # o Where to write output
73			#
74
75			cd \${PBS_O_WORKDIR}
76			$RUNTESTREPORT -runonly -command "$EXE" >> $MYOUTPUT 2>&1
77
78			EOF
79
80			# clean up old testreport output
81			if [ -e $MYOUTPUT ]; then
82			rm -rf $MYOUTPUT
83			fi
84			if [ -e $OUTFILE ]; then
85			rm -r $OUTFILE
86			fi
87			if [ -e $TDIR/MITgcm ]; then
88			# if availabe just update
89			cd $TDIR/MITgcm
90			cvs -q update -d -P >> cvs_co.log 2>&1
91			if [ "$?" != "0" ]; then
92			cat cvs_co.log
93			fi
94			else
95			# create directory and download code
96			if [ -e $TDIR ]; then
97			rm -rf $TDIR
98			fi
99			mkdir $TDIR
100			cd $TDIR
101			cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm > cvs_co.log 2>&1
102			if [ "$?" != "0" ]; then
103			cat cvs_co.log
104			fi
105			fi
106
107			cd $TDIR/MITgcm/verification
108
109			# make sure that we do not use the cross compiler for testreport
110			unset CC
111			# make sure that do use the cross compiler for testreport
112			#export CC=sxcc
113
114			$RUNTESTREPORT -j 8 -norun > $MYOUTPUT 2>&1
115
116			if [ "$?" != "0" ]
117			then
118			echo "something wrong with testreport"
119			echo "keeping the working directory"
120			#else
121			# echo "check restarts"
122			# echo ../tools/do_tst_2+2 -mpi -exe \"$HERE/$RUNIT\" -a NONE
123			# ../tools/do_tst_2+2 -mpi -exe $HERE/$RUNIT -a NONE
124			# everything OK: delete working directory
125			# rm -rf $TDIR
126			fi
127
128			if [ ! -e $MYOUTPUT ]
129			then
130			touch $MYOUTPUT
131			fi
132
133			echo " " >> $MYOUTPUT
134			echo "***********************************************************" >> $MYOUTPUT
135			echo "Submitting this job script:" >> $MYOUTPUT
136			echo "***********************************************************" >> $MYOUTPUT
137			cat $HERE/$JOBSCRIPT >> $MYOUTPUT
138			echo "***********************************************************" >> $MYOUTPUT
139			echo "end of job script" >> $MYOUTPUT
140			echo "***********************************************************" >> $MYOUTPUT
141			echo " " >> $MYOUTPUT
142
143			# now submit the job that actually runs all the experiments in one go
144			qsub $HERE/$JOBSCRIPT
145			# keep looking for the job in the job queues and wait until has disappeared
146			jobruns=`qstat -n -u mlosch \| grep "$JOBNAME"`
147			while [ "${jobruns}"x != x ]
148			do
149			sleep 200
150			jobruns=`qstat -n -u mlosch \| grep "$JOBNAME"`
151			echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
152			currentexp=`grep Experiment $MYOUTPUT \| tail -1`
153			echo "currently running $currentexp"
154			done
155
156			# after running the experiments on the compute node run testreport
157			# for a third time to evaluate results on the head node again
158			echo " " >> $MYOUTPUT
159			echo "now run testreport for a final time to evaluate results:" >> $MYOUTPUT
160			echo "$RUNTESTREPORT -match 10 -runonly" >> $MYOUTPUT
161			#$RUNTESTREPORT -match 10 -runonly >> $MYOUTPUT 2>&1
162			$RUNTESTREPORT -match 10 -runonly \
163	mlosch	1.3	-a "jm_c@mitgcm.org" >> $MYOUTPUT 2>&1
164			# -a "jm_c@mitgcm.org, Martin.Losch@awi.de" >> $MYOUTPUT 2>&1
165	mlosch	1.1
166			echo "end of mitgcmtestreport"