/[MITgcm]/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray
ViewVC logotype

Diff of /MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.1 by mlosch, Thu Jun 2 12:51:56 2016 UTC revision 1.3 by mlosch, Tue Jun 21 09:43:58 2016 UTC
# Line 1  Line 1 
1  #!/bin/bash -e  #!/bin/bash -e
2  # new script for running testreport on ollie.awi.de  # new script for running testreport on ollie.awi.de
3  # - split the testreport into 2 steps:  p# - compile and run on compute nodes
4  # 1/ compiling on head node, with -norun option  # - use ssh to call mpack command from the head node ollie0
 # 2/ running on compute node (using SBATCH), with -runonly option  
5  #$Header$  #$Header$
6  #$Name$  #$Name$
7    
# Line 26  echo "start from TST_DIR='$TST_DIR' at: Line 25  echo "start from TST_DIR='$TST_DIR' at:
25    
26  umask 0022  umask 0022
27    
28  sfx='ftn'  sfx='cray'
29  RUNIT="runit_"$sfx  RUNIT="runit_"$sfx
30    
31  OPTFILE=../tools/build_options/linux_ia64_cray_ollie  OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
32  options="-MPI 6"  options="-MPI 6"
33  options="$options -odir $dNam"  options="$options -odir ${dNam}-c"
34  options="$options -devel"  p#options="$options -j 6"
35  #options="$options -j 2"  #options="$options -t global_ocean.cs32x15"
36  #options="$options -j 6"  
 #options="$options -t global_ocean.90x40x15"  
37  #EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'  #EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'
38  EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'  EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'
39    
# Line 51  if [ -e $HERE ]; then Line 49  if [ -e $HERE ]; then
49  else  else
50      mkdir $HERE      mkdir $HERE
51  fi  fi
52  OUTFILE=$HERE/testreport.out  OUTFILE=$HERE/slurm_${sfx}.out
53  MYOUTPUT=$HERE/out_$sfx  MYOUTPUT=$HERE/out_$sfx
54  if [ -e $MYOUTPUT ]; then  if [ -e $MYOUTPUT ]; then
55    rm -rf $MYOUTPUT    rm -rf $MYOUTPUT
# Line 60  if [ -e $OUTFILE ]; then Line 58  if [ -e $OUTFILE ]; then
58    rm -r $OUTFILE    rm -r $OUTFILE
59  fi  fi
60  gcmDIR="MITgcm_$sfx"  gcmDIR="MITgcm_$sfx"
 #gcmDIR="MITgcm"  
61  checkOut=1  checkOut=1
62  if [ $checkOut -eq 1 ] ; then  if [ $checkOut -eq 1 ] ; then
63   if test -d $gcmDIR/CVS ; then   if test -d $gcmDIR/CVS ; then
# Line 90  fi Line 87  fi
87    
88  cd ${gcmDIR}/verification  cd ${gcmDIR}/verification
89    
90  runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\""  cwd=`pwd`
91    SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
92    runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
93    
94  if [ ! -e $MYOUTPUT ]  if [ ! -e $MYOUTPUT ]
95      then      then
96      touch $MYOUTPUT      touch $MYOUTPUT
97  fi  fi
98    
99  echo "running testreport like this:"  # echo "running testreport like this:"
100  echo ${runtestreport} -norun  # echo ${runtestreport} -norun
101  eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"  # eval "${runtestreport} -norun >> $MYOUTPUT 2>&1"
102    
103  # create batch script  # create batch script
104  #  #
105  JOBNAME=tstoll  JOBNAME=tst$sfx
106  echo "creating batch script $HERE/$RUNIT"  echo "creating batch script $HERE/$RUNIT"
107  cat << EOF >| $HERE/$RUNIT  cat << EOF >| $HERE/$RUNIT
108  #!/bin/bash -x  #!/bin/bash
109  #SBATCH --job-name=${JOBNAME}  #SBATCH --job-name=${JOBNAME}
110  #SBATCH  -o ${OUTFILE}  #SBATCH  -o ${OUTFILE}
111  #SBATCH --time=3:00:00  #SBATCH --time=12:00:00
112  #SBATCH --ntasks=6  #SBATCH --ntasks=6
 #SBATCH  -p smp  
 #SBATCH --qos=short  
113    
114  ulimit -s 1048576  # still need this to be able to run a few experiments:
115    # cfc_example, fizhi, tutorial_deep_convection
116    ulimit -s unlimited
117    
118  cd \${SLURM_SUBMIT_DIR}  cd \${SLURM_SUBMIT_DIR}
119    
120  echo "running testreport like this:"  cwd=`pwd`
121  echo "${runtestreport} -runonly"  echo "running testreport like this in \${cwd}:"
122  #${runtestreport} -runonly -match 10 -a 'Martin.Losch@awi.de' >> $MYOUTPUT  echo "${runtestreport} -devel -match 10"
123  ${runtestreport} -runonly -match 10 -a 'jmc@mitcm.org' >> $MYOUTPUT  ${runtestreport} -devel -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1
124    
125    echo "running testreport like this in \${cwd}:"
126    echo "./testreport -clean"
127    ./testreport -clean
128    
129    echo "running testreport like this in \${cwd}:"
130    echo "${runtestreport} -fast -match 10"
131    ${runtestreport} -fast -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1
132    
133  EOF  EOF
134    
# Line 139  echo " " >> $MYOUTPUT Line 146  echo " " >> $MYOUTPUT
146    
147  sbatch $HERE/$RUNIT  sbatch $HERE/$RUNIT
148    
149  # keep looking for the job in the job queues and wait until has disappeared  # # keep looking for the job in the job queues and wait until it has disappeared
150  jobruns=somedummy  # jobruns=somedummy
151  while [ "${jobruns}"x != x ]  # while [ "${jobruns}"x != x ]
152  do  # do
153    sleep 20  #   sleep 200
154    jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`  #   jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`
155    echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"  #   echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
156    currentexp=`grep Experiment $MYOUTPUT | tail -1`  #   currentexp=`grep Experiment $MYOUTPUT | tail -1`
157    echo "currently running $currentexp"  #   echo "currently running $currentexp"
158  done  # done
159    
160    # # workaround for mailing the stuff
161    # echo "mail the stuff"
162    
 # # after running the experiments send email to jmc (cannot be done from  
 # # the compute node, yet)  
163  # MPACKCMD=../tools/mpack-1.6/mpack  # MPACKCMD=../tools/mpack-1.6/mpack
164  # fn=`ls -dtr tr_ollie* | grep -v tar.gz | tail -1`  # fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`
165  # echo "fn $fn"  # echo "fname ${fn}"
166  # tar cf - $fn | gzip > "${fn}.tar.gz"  # tar cf - $fn | gzip > "${fn}.tar.gz"
167  # ../tools/mpack-1.6/mpack -s MITgcm-test -m 3555000 ${fn}.tar.gz mlosch@awi.de  # $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org
168  # sleep 2  # sleep 2
169  # \rm -f ${fn}.tar.gz  # rm -rf "${fn}.tar.gz"
 # ../tools/mpack-1.6/mpack -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org  
   
 # after running the experiments on the compute node run testreport  
 # for a third time to evaluate results on the head node again  
 # echo " " >> $MYOUTPUT  
 # echo "now run testreport for a final time to evaluate results:" >> $MYOUTPUT  
 # echo "$RUNTESTREPORT -match 10 -runonly" >> $MYOUTPUT  
 # #$RUNTESTREPORT -match 10 -runonly >> $MYOUTPUT 2>&1  
 # $RUNTESTREPORT -match 10 -runonly \  
 #     -a "jmc@mitgcm.org" >> $MYOUTPUT 2>&1  
 # #   -a "jmc@mitgcm.org, Martin.Losch@awi.de" >> $MYOUTPUT 2>&1  
   
 # workaround for mailing the stuff  
 echo "mail the stuff"  
 # 1. set name of remote host where to do the mpack command  
 # 2. pack directory into an archive an compress it  
 # 3. copy gzipped archive to remote host  
 # 4. on the remote host execute the mpack command, that send the email  
 # 5. wait a little, just to be sure everything is done  
 # 6. remove archives  
 MPACKCMD=\${HOME}/bin/mpack  
 rmhost=rayl4.awi.de  
 pwd  
 fn=`ls -dtr tr_ollie* | grep -v tar.gz | tail -1`  
 echo "fname ${fn}"  
 tar cf - $fn | gzip > "${fn}.tar.gz"  
 eval "scp ${fn}.tar.gz ${rmhost}:"  
 #ssh -Y $rmhost "$MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz mlosch@awi.de"  
 ssh -Y $rmhost "$MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org"  
 sleep 2  
 ssh -Y $rmhost "rm -f ${fn}.tar.gz"  
 rm -rf "${fn}.tar.gz"  
 # end workaround for mailing the stuff  
170    
171  echo "end of mitgcmtestreport"  echo "end of mitgcmtestreport"
172    
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.3
  ViewVC Help
Powered by ViewVC 1.1.22