/[MITgcm]/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray
ViewVC logotype

Diff of /MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.2 by mlosch, Fri Jun 10 07:55:29 2016 UTC revision 1.7 by mlosch, Mon Mar 13 09:31:51 2017 UTC
# Line 1  Line 1 
1  #!/bin/bash -e  #!/bin/bash -e
2  # new script for running testreport on ollie.awi.de  # new script for running testreport on ollie.awi.de
3  # - split the testreport into 2 steps:  # - compile and run on compute nodes
4  # 1/ compiling on head node, with -norun option  # - use ssh to call mpack command from the head node ollie0
 # 2/ running on compute node (using SBATCH), with -runonly option  
5  #$Header$  #$Header$
6  #$Name$  #$Name$
7    
# Line 11  source /usr/Modules/3.2.10/init/bash Line 10  source /usr/Modules/3.2.10/init/bash
10  source /etc/profile.d/cray_pe.sh  source /etc/profile.d/cray_pe.sh
11  source /etc/profile.d/AWI.sh  source /etc/profile.d/AWI.sh
12  #  #
13    
14  module purge  module purge
 module load slurm  
15  module load craype-broadwell  module load craype-broadwell
16  module load PrgEnv-cray/1.0.0  module load PrgEnv-cray
17  module unload cudatoolkit  # to me this looks like a terrible hack, but that is what we are supposed
18  module swap mvapich2_cce cray-impi  # to use now
19  module load intel/impi-5.1.3  module unload craype-network-infiniband cudatoolkit mvapich2_cce
20  export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}:  module load craype-network-opa
21    module load mvapich2_cce
22    module load netcdf
23    #export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}:
24    # not sure why I have set these paths here again
25    export MPI_ROOT=$(dirname $(dirname `which mpicc`))
26    # $(dirname `echo $LD_LIBRARY_PATH  | awk -F: '{print $1}'`)
27    export MPI_INC_DIR=${MPI_ROOT}/include
28    
29    # there is no slurm module anymore and this is the current recommendation to
30    # have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
31    export PATH=${PATH}:/global/opt/slurm/default/bin
32    
33  dNam='ollie'  dNam='ollie'
34  TST_DIR="/work/ollie/mlosch/test_$dNam"  TST_DIR="/work/ollie/mlosch/test_$dNam"
# Line 32  RUNIT="runit_"$sfx Line 42  RUNIT="runit_"$sfx
42  OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie  OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
43  options="-MPI 6"  options="-MPI 6"
44  options="$options -odir ${dNam}-c"  options="$options -odir ${dNam}-c"
 options="$options -devel"  
 #options="$options -j 2"  
45  #options="$options -j 6"  #options="$options -j 6"
46  #options="$options -t global_ocean.cs32x15 -q"  #options="$options -t global_ocean.cs32x15"
47    
48  #EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'  #EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'
49  EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'  EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'
50    
# Line 60  if [ -e $OUTFILE ]; then Line 69  if [ -e $OUTFILE ]; then
69    rm -r $OUTFILE    rm -r $OUTFILE
70  fi  fi
71  gcmDIR="MITgcm_$sfx"  gcmDIR="MITgcm_$sfx"
 #gcmDIR="MITgcm"  
72  checkOut=1  checkOut=1
73  if [ $checkOut -eq 1 ] ; then  if [ $checkOut -eq 1 ] ; then
74   if test -d $gcmDIR/CVS ; then   if test -d $gcmDIR/CVS ; then
# Line 90  fi Line 98  fi
98    
99  cd ${gcmDIR}/verification  cd ${gcmDIR}/verification
100    
101  runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\""  cwd=\`pwd\`
102    SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
103    runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
104    
105  if [ ! -e $MYOUTPUT ]  if [ ! -e $MYOUTPUT ]
106      then      then
# Line 106  fi Line 116  fi
116  JOBNAME=tst$sfx  JOBNAME=tst$sfx
117  echo "creating batch script $HERE/$RUNIT"  echo "creating batch script $HERE/$RUNIT"
118  cat << EOF >| $HERE/$RUNIT  cat << EOF >| $HERE/$RUNIT
119  #!/bin/bash -x  #!/bin/bash
120  #SBATCH --job-name=${JOBNAME}  #SBATCH --job-name=${JOBNAME}
121  #SBATCH  -o ${OUTFILE}  #SBATCH  -o ${OUTFILE}
122  #SBATCH --time=6:00:00  #SBATCH --time=12:00:00
123  #SBATCH --ntasks=6  #SBATCH --ntasks=6
 #SBATCH  -p smp  
124    
125  ulimit -s 1048576  # still need this to be able to run a few experiments:
126    # cfc_example, fizhi, tutorial_deep_convection
127    ulimit -s unlimited
128    
129    # # for debugging
130    # export FLEXLM_DIAGNOSTICS=2
131    # export FNP_IP_ENV=1
132    # export LM_A_CONN_TIMEOUT=99
133    
134  cd \${SLURM_SUBMIT_DIR}  cd \${SLURM_SUBMIT_DIR}
135    
136  echo "running testreport like this:"  cwd=`pwd`
137  echo "${runtestreport} -runonly"  echo "running testreport like this in \${cwd}:"
138  #${runtestreport} -runonly -match 10 -a 'Martin.Losch@awi.de' >> $MYOUTPUT  echo "${runtestreport} -devel -match 10"
139  #${runtestreport} -runonly -match 10 -a 'jmc@mitcm.org' >> $MYOUTPUT  ${runtestreport} -devel -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1
140  ${runtestreport} -match 10 >> $MYOUTPUT  
141    echo "running testreport like this in \${cwd}:"
142    echo "./testreport -clean"
143    ./testreport -clean
144    
145    echo "running testreport like this in \${cwd}:"
146    echo "${runtestreport} -fast -match 10"
147    ${runtestreport} -fast -match 10 -a jmc@mitgcm.org >> $MYOUTPUT 2>&1
148    
149  EOF  EOF
150    
# Line 139  echo " " >> $MYOUTPUT Line 162  echo " " >> $MYOUTPUT
162    
163  sbatch $HERE/$RUNIT  sbatch $HERE/$RUNIT
164    
165  # keep looking for the job in the job queues and wait until has disappeared  # # keep looking for the job in the job queues and wait until it has disappeared
166  jobruns=somedummy  # jobruns=somedummy
167  while [ "${jobruns}"x != x ]  # while [ "${jobruns}"x != x ]
168  do  # do
169    sleep 200  #   sleep 200
170    jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`  #   jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`
171    echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"  #   echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
172    currentexp=`grep Experiment $MYOUTPUT | tail -1`  #   currentexp=`grep Experiment $MYOUTPUT | tail -1`
173    echo "currently running $currentexp"  #   echo "currently running $currentexp"
174  done  # done
175    
176  # workaround for mailing the stuff  # # workaround for mailing the stuff
177  echo "mail the stuff"  # echo "mail the stuff"
178    
179  MPACKCMD=../tools/mpack-1.6/mpack  # MPACKCMD=../tools/mpack-1.6/mpack
180  fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`  # fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`
181  echo "fname ${fn}"  # echo "fname ${fn}"
182  tar cf - $fn | gzip > "${fn}.tar.gz"  # tar cf - $fn | gzip > "${fn}.tar.gz"
183  $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org  # $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org
184  sleep 2  # sleep 2
185  rm -rf "${fn}.tar.gz"  # rm -rf "${fn}.tar.gz"
186    
187  echo "end of mitgcmtestreport"  echo "end of mitgcmtestreport"
188    
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.7
  ViewVC Help
Powered by ViewVC 1.1.22