/[MITgcm]/MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray
ViewVC logotype

Diff of /MITgcm_contrib/test_scripts/ollie/mitgcmtestreport_cray

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.2 by mlosch, Fri Jun 10 07:55:29 2016 UTC revision 1.9 by mlosch, Mon Apr 16 08:35:39 2018 UTC
# Line 1  Line 1 
1  #!/bin/bash -e  #!/bin/bash -e
2  # new script for running testreport on ollie.awi.de  # new script for running testreport on ollie.awi.de
3  # - split the testreport into 2 steps:  # - compile and run on compute nodes
4  # 1/ compiling on head node, with -norun option  # - use ssh to call mpack command from the head node ollie0
 # 2/ running on compute node (using SBATCH), with -runonly option  
5  #$Header$  #$Header$
6  #$Name$  #$Name$
7    
# Line 12  source /etc/profile.d/cray_pe.sh Line 11  source /etc/profile.d/cray_pe.sh
11  source /etc/profile.d/AWI.sh  source /etc/profile.d/AWI.sh
12  #  #
13  module purge  module purge
14  module load slurm  module load PrgEnv-cray/1.0.2_ollie
15  module load craype-broadwell  module load netcdf
16  module load PrgEnv-cray/1.0.0  #export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}:
17  module unload cudatoolkit  # not sure why I have to set these paths here again
18  module swap mvapich2_cce cray-impi  export MPI_ROOT=$(dirname $(dirname `which mpicc`))
19  module load intel/impi-5.1.3  # $(dirname `echo $LD_LIBRARY_PATH  | awk -F: '{print $1}'`)
20  export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}:  export MPI_INC_DIR=${MPI_ROOT}/include
21    
22    # there is no slurm module anymore and this is the current recommendation to
23    # have sbatch in your path (rather than running /etc/profile.d./slurm.sh)
24    export PATH=${PATH}:/global/opt/slurm/default/bin
25    
26  dNam='ollie'  dNam='ollie'
27  TST_DIR="/work/ollie/mlosch/test_$dNam"  TST_DIR="/work/ollie/mlosch/test_$dNam"
# Line 32  RUNIT="runit_"$sfx Line 35  RUNIT="runit_"$sfx
35  OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie  OPTFILE=../tools/build_options/linux_ia64_${sfx}_ollie
36  options="-MPI 6"  options="-MPI 6"
37  options="$options -odir ${dNam}-c"  options="$options -odir ${dNam}-c"
38  options="$options -devel"  options="$options -j 6"
39  #options="$options -j 2"  #options="$options -t global_ocean.cs32x15"
40  #options="$options -j 6"  
41  #options="$options -t global_ocean.cs32x15 -q"  #EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'
42  #EXE='mpiexec.hydra -bootstrap slurm -n TR_NPROC ./mitgcmuv'  #EXE='srun -n TR_NPROC ./mitgcmuv'
43  EXE='srun --mpi=pmi2 -n TR_NPROC ./mitgcmuv'  EXE='srun -n TR_NPROC --cpu_bind=cores ./mitgcmuv'
44    
45  if [ -e $TST_DIR ]; then  if [ -e $TST_DIR ]; then
46      echo "$TST_DIR exists"      echo "$TST_DIR exists"
# Line 59  fi Line 62  fi
62  if [ -e $OUTFILE ]; then  if [ -e $OUTFILE ]; then
63    rm -r $OUTFILE    rm -r $OUTFILE
64  fi  fi
65  gcmDIR="MITgcm_$sfx"  gcmDIR="MITgcm_${sfx}"
66  #gcmDIR="MITgcm"  git_repo='MITgcm'
67  checkOut=1  git_code='MITgcm'
68  if [ $checkOut -eq 1 ] ; then  
69   if test -d $gcmDIR/CVS ; then  # checkOut determines how much checking out is being done
70    /bin/rm -rf $gcmDIR/verification/??_${dNam}_????????_?  # checkOut = 3: new clone from GitHub and make a new copy
71    ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )  # checkOut = 2: update (git pull) existing repo and make a new copy
72    echo "cvs update of dir $gcmDIR :"  # checkOut = 1: skip update
73    ( cd $gcmDIR ; cvs -q -d :pserver:cvsanon@mitgcm.org:/u/gcmpack update -P -d ) 2>&1  # checkOut = 0: use existing test code (if available otherwise switch to 1)
74    RETVAL=$?  
75    if test "x$RETVAL" != x0 ; then  checkOut=2
76     echo "cvs update on '"`hostname`"' failed (return val=$RETVAL) => exit"  tdir=${TST_DIR}
77     exit  today=`date +%Y%m%d`
78    TODAY=`date +%d`
79    #tmpFil="/tmp/"`basename $0`".$$"
80    tmpFil=$TST_DIR/error.out
81    
82    if [ $checkOut -le 1 ] ; then
83      if test -e $TST_DIR/${gcmDIR}/doc ; then
84        echo $TST_DIR/${gcmDIR}/doc 'exist'
85      else
86        echo -n $TST_DIR/${gcmDIR} 'missing ; '
87        checkOut=2
88        echo "will make a new copy ( checkOut=$checkOut )"
89      fi
90    fi
91    
92    if [ $checkOut -ge 2 ] ; then
93      #---- cleaning:
94      cd $TST_DIR
95    
96      #---- Make a new clone or update existing one:
97      if test -e ${gcmDIR}/.git/config ; then
98        echo "${gcmDIR}/.git/config exist"
99      else
100        echo -n "${gcmDIR}/.git/config 'missing "
101        checkOut=3
102        echo "will get new clone ( checkOut=$checkOut )"
103    fi    fi
104   else    if [ $checkOut -eq 3 ] ; then
105    echo "no dir: $gcmDIR/CVS => try a fresh check-out"      echo -n "Removing old clone: $TST_DIR/${gcmDIR} ..."
106    checkOut=2      test -e $TST_DIR/${gcmDIR}  &&  rm -rf $TST_DIR/${gcmDIR}
107   fi      echo "  done"
108  fi      echo -n "Make a new clone of $git_code from repo: $git_repo ..."
109  if [ $checkOut -eq 2 ] ; then      git clone https://github.com/$git_repo/${git_code}.git ${gcmDIR} 2> $tmpFil
110    if test -e $gcmDIR ; then      retVal=$?
111     rm -rf $gcmDIR      if test $retVal = 0 ; then
112           echo ' --> done!'
113           rm -f $tmpFil
114        else
115           echo " Error: 'git clone' returned: $retVal"
116           cat $tmpFil
117           rm -f $tmpFil
118           exit 2
119        fi
120      else
121        echo "Updating current clone ( $git_code ) ..."
122        ( cd ${gcmDIR}; git checkout master ; git pull )
123        echo ' --> done!'
124    fi    fi
125  # cvs co MITgcm  else
126    cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co -P -d $gcmDIR MITgcm > /dev/null 2>&1    cd $TST_DIR
   /usr/bin/find $gcmDIR -type d | xargs chmod g+rxs  
   /usr/bin/find $gcmDIR -type f | xargs chmod g+r  
127  fi  fi
128    
129  cd ${gcmDIR}/verification  cd ${TST_DIR}/${gcmDIR}/verification
130    
131  runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\""  cwd=\`pwd\`
132    SENDCMD="ssh ollie0 ${TST_DIR}/${gcmDIR}/tools/mpack-1.6/mpack"
133    runtestreport="./testreport $options -of $OPTFILE -command \"${EXE}\" -send \"${SENDCMD}\" -sd ${cwd}"
134    emailaddress="-a jm_c@mitgcm.org"
135    
136  if [ ! -e $MYOUTPUT ]  if [ ! -e $MYOUTPUT ]
137      then      then
# Line 106  fi Line 147  fi
147  JOBNAME=tst$sfx  JOBNAME=tst$sfx
148  echo "creating batch script $HERE/$RUNIT"  echo "creating batch script $HERE/$RUNIT"
149  cat << EOF >| $HERE/$RUNIT  cat << EOF >| $HERE/$RUNIT
150  #!/bin/bash -x  #!/bin/bash
151  #SBATCH --job-name=${JOBNAME}  #SBATCH --job-name=${JOBNAME}
152  #SBATCH  -o ${OUTFILE}  #SBATCH  -o ${OUTFILE}
153  #SBATCH --time=6:00:00  #SBATCH --time=12:00:00
154  #SBATCH --ntasks=6  #SBATCH --ntasks=6
 #SBATCH  -p smp  
155    
156  ulimit -s 1048576  # still need this to be able to run a few experiments:
157    # cfc_example, fizhi, tutorial_deep_convection
158    ulimit -s unlimited
159    
160    # binds OpenMP task to given cores
161    export OMP_PROC_BIND=TRUE
162    
163    # for debugging
164    # export FLEXLM_DIAGNOSTICS=2
165    # export FNP_IP_ENV=1
166    # export LM_A_CONN_TIMEOUT=99
167    
168  cd \${SLURM_SUBMIT_DIR}  cd \${SLURM_SUBMIT_DIR}
169    
170  echo "running testreport like this:"  cwd=`pwd`
171  echo "${runtestreport} -runonly"  echo "running testreport like this in \${cwd}:"
172  #${runtestreport} -runonly -match 10 -a 'Martin.Losch@awi.de' >> $MYOUTPUT  echo "${runtestreport} -devel -match 10"
173  #${runtestreport} -runonly -match 10 -a 'jmc@mitcm.org' >> $MYOUTPUT  ${runtestreport} -devel -match 10 ${emailaddress} >> $MYOUTPUT 2>&1
174  ${runtestreport} -match 10 >> $MYOUTPUT  
175    echo "running testreport like this in \${cwd}:"
176    echo "./testreport -clean"
177    ./testreport -clean
178    
179    echo "running testreport like this in \${cwd}:"
180    echo "${runtestreport} -fast -match 10"
181    ${runtestreport} -fast -match 10 ${emailaddress} >> $MYOUTPUT 2>&1
182    
183  EOF  EOF
184    
# Line 137  echo "end of job script" >> $MYOUTPUT Line 194  echo "end of job script" >> $MYOUTPUT
194  echo "***********************************************************" >> $MYOUTPUT  echo "***********************************************************" >> $MYOUTPUT
195  echo " " >> $MYOUTPUT  echo " " >> $MYOUTPUT
196    
197    echo "sbatch $HERE/$RUNIT"
198  sbatch $HERE/$RUNIT  sbatch $HERE/$RUNIT
199    
200  # keep looking for the job in the job queues and wait until has disappeared  # # keep looking for the job in the job queues and wait until it has disappeared
201  jobruns=somedummy  # jobruns=somedummy
202  while [ "${jobruns}"x != x ]  # while [ "${jobruns}"x != x ]
203  do  # do
204    sleep 200  #   sleep 200
205    jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`  #   jobruns=`squeue --noheader -u mlosch | grep "$JOBNAME" | awk '{print $1}'`
206    echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"  #   echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
207    currentexp=`grep Experiment $MYOUTPUT | tail -1`  #   currentexp=`grep Experiment $MYOUTPUT | tail -1`
208    echo "currently running $currentexp"  #   echo "currently running $currentexp"
209  done  # done
210    
211  # workaround for mailing the stuff  # # workaround for mailing the stuff
212  echo "mail the stuff"  # echo "mail the stuff"
213    
214  MPACKCMD=../tools/mpack-1.6/mpack  # MPACKCMD=../tools/mpack-1.6/mpack
215  fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`  # fn=`ls -dtr tr_$dNam* | grep -v tar.gz | tail -1`
216  echo "fname ${fn}"  # echo "fname ${fn}"
217  tar cf - $fn | gzip > "${fn}.tar.gz"  # tar cf - $fn | gzip > "${fn}.tar.gz"
218  $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jmc@mitgcm.org  # $MPACKCMD -s MITgcm-test -m 3555000 ${fn}.tar.gz jm_c@mitgcm.org
219  sleep 2  # sleep 2
220  rm -rf "${fn}.tar.gz"  # rm -rf "${fn}.tar.gz"
221    
222  echo "end of mitgcmtestreport"  echo "end of mitgcmtestreport"
223    
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.9
  ViewVC Help
Powered by ViewVC 1.1.22