Annotation of /MITgcm_contrib/test_scripts/sx8/mitgcmtestreport_split

#!/bin/bash
# new script for running testreport on sx8.awi.de
# - split the testreport into 3 steps:
# 1/ compiling on head node (tx7.awi.de), with -norun option
# 2/ running on compute node (using PBS qsub), with -runonly option
# 3/ evaluating result on head node with -runonly option
# 
# Notes: 
# - step 2 leads to many error messages, because the OS on the compute
#   nodes does not have the appropriate shell tools, modifying the
# - runonly option to skip the evalution step would be nice but not
#   necessary; you'll just have to live with the error messages
# - step 3 assumes that all experiments have been run successfully, i.e.
#   that the output files are up-to-date.
#   if not, testreport will try to run the sx8-code on the tx7 frontend
#   which will fail inevitably and produce more errors, maybe we can
#   have a flag that skips everything but the evaluation step to avoid this
# $Header: /u/gcmpack/MITgcm_contrib/test_scripts/sx8/mitgcmtestreport_split,v 1.3 2015/02/26 17:52:45 mlosch Exp $
# $Name:  $

# load latest compiler:
module load sxf90/460

VENDOR=sxf90
RUNIT="runit_"$VENDOR
HERE=`pwd`
EXE='mpirun -np TR_NPROC ./mitgcmuv'
NPROCS=2
MPI="-MPI $NPROCS"
OUTFILE=/home/sx8/mlosch/out_${VENDOR}
MYOUTPUT=/home/sx8/mlosch/testreport_${VENDOR}
OUTFILE=out_${VENDOR}
JOBNAME=testsx8
JOBSCRIPT=job_${VENDOR}
#selectexperiment='-t exp2'
selectexperiment=''
# download code into this directory
TDIR=/sx8/scr/mlosch/tmp_$VENDOR

OPTFILE=../tools/build_options/SUPER-UX_SX-8_sxf90_awi

RUNTESTREPORT="./testreport $MPI -of=${OPTFILE} $selectexperiment"
#
# create batch script
#
cat << EOF > $HERE/$JOBSCRIPT
#PBS -q sx8-r                            # job queue not neccesary  so far 
#PBS -N $JOBNAME                         # give the job a name
#PBS -l cpunum_job=$NPROCS               # cpus per node
#PBS -l cputim_job=2:00:00               # time limit
#PBS -l memsz_job=32gb                   # max accumulated memory, we need this much because of many netcdf files
#PBS -j o                                # join i/o
#PBS -S /bin/sh
#PBS -o $OUTFILE                         # o Where to write output
#

cd \${PBS_O_WORKDIR}
$RUNTESTREPORT -runonly -command "$EXE" >> $MYOUTPUT 2>&1

EOF

# clean up old testreport output
if [ -e $MYOUTPUT ]; then
  rm -rf $MYOUTPUT
fi
if [ -e $OUTFILE ]; then
  rm -r $OUTFILE
fi
if [ 0 = 0 ]; then
# create directory and download code
    if [ -e $TDIR ]; then
	rm -rf $TDIR
    fi
    mkdir $TDIR
    cd $TDIR
#   cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm_verif_basic > cvs_co.log 2>&1
    cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm > cvs_co.log 2>&1
    if [ $status > 0 ]; then
	cat cvs_co.log
    fi
else
    cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q update >> cvs_co.log 2>&1
    if [ $status > 0 ]; then
	cat cvs_co.log
    fi
fi

cd $TDIR/MITgcm/verification

# make sure that we do not use the cross compiler for testreport
unset CC
# make sure that do use the cross compiler for testreport
#export CC=sxcc

$RUNTESTREPORT  -j 8 -norun > $MYOUTPUT 2>&1

if [ $status > 0 ]
    then
    echo "something wrong with testreport"
    echo "keeping the working directory"
#else
#  echo "check restarts"
#  echo ../tools/do_tst_2+2 -mpi -exe \"$HERE/$RUNIT\" -a NONE
#  ../tools/do_tst_2+2 -mpi -exe $HERE/$RUNIT -a NONE
# everything OK: delete working directory
#  rm -rf $TDIR
fi

if [ ! -e $MYOUTPUT ]
    then
    touch $MYOUTPUT
fi

echo " " >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "Submitting this job script:" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
cat $HERE/$JOBSCRIPT >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo "end of job script" >> $MYOUTPUT
echo "***********************************************************" >> $MYOUTPUT
echo " " >> $MYOUTPUT

# now submit the job that actually runs all the experiments in one go
qsub $HERE/$JOBSCRIPT
# keep looking for the job in the job queues and wait until has disappeared
jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`
while [ "${jobruns}"x != x ]
do 
  sleep 20
  jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`
  echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
  currentexp=`grep Experiment $MYOUTPUT | tail -1`
  echo "currently running $currentexp"
done

# after running the experiments on the compute node run testreport
# for a third time to evaluate results on the head node again
echo " " >> $MYOUTPUT
echo "now run testreport for a final time to evaluate results:" >> $MYOUTPUT
echo "$RUNTESTREPORT -match 10 -runonly" >> $MYOUTPUT
#$RUNTESTREPORT -match 10 -runonly >> $MYOUTPUT 2>&1
$RUNTESTREPORT -match 10 -runonly \
    -a "jmc@mitgcm.org, Martin.Losch@awi.de" >> $MYOUTPUT 2>&1

echo "end of mitgcmtestreport_split"
1	mlosch	1.4	#!/bin/bash
2	mlosch	1.2	# new script for running testreport on sx8.awi.de
3			# - split the testreport into 3 steps:
4			# 1/ compiling on head node (tx7.awi.de), with -norun option
5			# 2/ running on compute node (using PBS qsub), with -runonly option
6			# 3/ evaluating result on head node with -runonly option
7			#
8			# Notes:
9			# - step 2 leads to many error messages, because the OS on the compute
10			# nodes does not have the appropriate shell tools, modifying the
11			# - runonly option to skip the evalution step would be nice but not
12			# necessary; you'll just have to live with the error messages
13			# - step 3 assumes that all experiments have been run successfully, i.e.
14			# that the output files are up-to-date.
15			# if not, testreport will try to run the sx8-code on the tx7 frontend
16			# which will fail inevitably and produce more errors, maybe we can
17			# have a flag that skips everything but the evaluation step to avoid this
18	mlosch	1.4	# $Header: /u/gcmpack/MITgcm_contrib/test_scripts/sx8/mitgcmtestreport_split,v 1.3 2015/02/26 17:52:45 mlosch Exp $
19			# $Name: $
20	mlosch	1.1
21			# load latest compiler:
22			module load sxf90/460
23
24			VENDOR=sxf90
25			RUNIT="runit_"$VENDOR
26			HERE=`pwd`
27			EXE='mpirun -np TR_NPROC ./mitgcmuv'
28			NPROCS=2
29			MPI="-MPI $NPROCS"
30			OUTFILE=/home/sx8/mlosch/out_${VENDOR}
31			MYOUTPUT=/home/sx8/mlosch/testreport_${VENDOR}
32			OUTFILE=out_${VENDOR}
33			JOBNAME=testsx8
34			JOBSCRIPT=job_${VENDOR}
35			#selectexperiment='-t exp2'
36			selectexperiment=''
37			# download code into this directory
38			TDIR=/sx8/scr/mlosch/tmp_$VENDOR
39
40			OPTFILE=../tools/build_options/SUPER-UX_SX-8_sxf90_awi
41
42			RUNTESTREPORT="./testreport $MPI -of=${OPTFILE} $selectexperiment"
43			#
44			# create batch script
45			#
46			cat << EOF > $HERE/$JOBSCRIPT
47			#PBS -q sx8-r # job queue not neccesary so far
48			#PBS -N $JOBNAME # give the job a name
49			#PBS -l cpunum_job=$NPROCS # cpus per node
50			#PBS -l cputim_job=2:00:00 # time limit
51			#PBS -l memsz_job=32gb # max accumulated memory, we need this much because of many netcdf files
52			#PBS -j o # join i/o
53			#PBS -S /bin/sh
54			#PBS -o $OUTFILE # o Where to write output
55			#
56
57			cd \${PBS_O_WORKDIR}
58			$RUNTESTREPORT -runonly -command "$EXE" >> $MYOUTPUT 2>&1
59
60			EOF
61
62			# clean up old testreport output
63			if [ -e $MYOUTPUT ]; then
64			rm -rf $MYOUTPUT
65			fi
66			if [ -e $OUTFILE ]; then
67			rm -r $OUTFILE
68			fi
69			if [ 0 = 0 ]; then
70			# create directory and download code
71			if [ -e $TDIR ]; then
72			rm -rf $TDIR
73			fi
74			mkdir $TDIR
75			cd $TDIR
76	mlosch	1.4	# cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm_verif_basic > cvs_co.log 2>&1
77			cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm > cvs_co.log 2>&1
78			if [ $status > 0 ]; then
79			cat cvs_co.log
80			fi
81			else
82			cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q update >> cvs_co.log 2>&1
83	mlosch	1.1	if [ $status > 0 ]; then
84			cat cvs_co.log
85			fi
86			fi
87
88			cd $TDIR/MITgcm/verification
89
90			# make sure that we do not use the cross compiler for testreport
91			unset CC
92			# make sure that do use the cross compiler for testreport
93			#export CC=sxcc
94
95	mlosch	1.4	$RUNTESTREPORT -j 8 -norun > $MYOUTPUT 2>&1
96	mlosch	1.1
97	mlosch	1.4	if [ $status > 0 ]
98			then
99			echo "something wrong with testreport"
100			echo "keeping the working directory"
101	mlosch	1.1	#else
102			# echo "check restarts"
103			# echo ../tools/do_tst_2+2 -mpi -exe \"$HERE/$RUNIT\" -a NONE
104			# ../tools/do_tst_2+2 -mpi -exe $HERE/$RUNIT -a NONE
105			# everything OK: delete working directory
106			# rm -rf $TDIR
107			fi
108
109	mlosch	1.4	if [ ! -e $MYOUTPUT ]
110			then
111			touch $MYOUTPUT
112			fi
113
114			echo " " >> $MYOUTPUT
115			echo "***********************************************************" >> $MYOUTPUT
116			echo "Submitting this job script:" >> $MYOUTPUT
117			echo "***********************************************************" >> $MYOUTPUT
118	mlosch	1.1	cat $HERE/$JOBSCRIPT >> $MYOUTPUT
119	mlosch	1.4	echo "***********************************************************" >> $MYOUTPUT
120	mlosch	1.1	echo "end of job script" >> $MYOUTPUT
121	mlosch	1.4	echo "***********************************************************" >> $MYOUTPUT
122	mlosch	1.1	echo " " >> $MYOUTPUT
123
124			# now submit the job that actually runs all the experiments in one go
125			qsub $HERE/$JOBSCRIPT
126			# keep looking for the job in the job queues and wait until has disappeared
127			jobruns=`qstat -n -u mlosch \| grep "$JOBNAME"`
128			while [ "${jobruns}"x != x ]
129			do
130			sleep 20
131			jobruns=`qstat -n -u mlosch \| grep "$JOBNAME"`
132	mlosch	1.4	echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
133			currentexp=`grep Experiment $MYOUTPUT \| tail -1`
134			echo "currently running $currentexp"
135	mlosch	1.1	done
136
137			# after running the experiments on the compute node run testreport
138			# for a third time to evaluate results on the head node again
139	mlosch	1.4	echo " " >> $MYOUTPUT
140			echo "now run testreport for a final time to evaluate results:" >> $MYOUTPUT
141			echo "$RUNTESTREPORT -match 10 -runonly" >> $MYOUTPUT
142			#$RUNTESTREPORT -match 10 -runonly >> $MYOUTPUT 2>&1
143	mlosch	1.1	$RUNTESTREPORT -match 10 -runonly \
144			-a "jmc@mitgcm.org, Martin.Losch@awi.de" >> $MYOUTPUT 2>&1
145
146			echo "end of mitgcmtestreport_split"