/[MITgcm]/MITgcm_contrib/test_scripts/sxace/mitgcmtestreport
ViewVC logotype

Diff of /MITgcm_contrib/test_scripts/sxace/mitgcmtestreport

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.1 by mlosch, Wed Apr 8 14:28:53 2015 UTC revision 1.6 by mlosch, Tue Mar 24 07:40:41 2020 UTC
# Line 4  Line 4 
4  # 1/ compiling on head node (tx7.awi.de), with -norun option  # 1/ compiling on head node (tx7.awi.de), with -norun option
5  # 2/ running on compute node (using PBS qsub), with -runonly option  # 2/ running on compute node (using PBS qsub), with -runonly option
6  # 3/ evaluating result on head node with -runonly option  # 3/ evaluating result on head node with -runonly option
7  #  #
8  # Notes:  # Notes:
9  # - step 2 leads to many error messages, because the OS on the compute  # - step 2 leads to many error messages, because the OS on the compute
10  #   nodes does not have the appropriate shell tools, modifying the  #   nodes does not have the appropriate shell tools, modifying the
11  #   runonly option to skip the evalution step would be nice but not  #   runonly option to skip the evalution step would be nice but not
# Line 33  module load sxc++ Line 33  module load sxc++
33  module load sxmpi  module load sxmpi
34  module load sxnetcdf  module load sxnetcdf
35  #  #
36    module load git
37    #
38  # make sure that we have qsub and qstat  # make sure that we have qsub and qstat
39  #export PATH=${PATH}:/usr/bin/nqsII  #export PATH=${PATH}:/usr/bin/nqsII
40  source /etc/profile.d/nec.sh  source /etc/profile.d/nec.sh
# Line 51  JOBSCRIPT=job_${VENDOR} Line 53  JOBSCRIPT=job_${VENDOR}
53  selectexperiment='-t exp2'  selectexperiment='-t exp2'
54  selectexperiment=''  selectexperiment=''
55  # download code into this directory  # download code into this directory
56  TDIR=/ace/scr/mlosch/tmp_$VENDOR  TDIR=/ace/user/mlosch/tmp_$VENDOR
57    gcmDIR=MITgcm
58    git_repo='MITgcm'
59    git_code='MITgcm'
60    
61  OPTFILE=../tools/build_options/SUPER-UX_SX-ACE_sxf90_awi  OPTFILE=../tools/build_options/SUPER-UX_SX-ACE_sxf90_awi
62  OPTFILE=/home/ace/mlosch/MITgcm/tools/build_options/SUPER-UX_SX-ACE_sxf90_awi  #OPTFILE=/home/ace/mlosch/MITgcm/tools/build_options/SUPER-UX_SX-ACE_sxf90_awi
63    
64  RUNTESTREPORT="./testreport $MPI -of=${OPTFILE} $selectexperiment -small_f"  RUNTESTREPORT="./testreport $MPI -of=${OPTFILE} $selectexperiment -small_f"
65  #  #
66  # create batch script  # create batch script
67  #  #
68  cat << EOF > $HERE/$JOBSCRIPT  cat << EOF > $HERE/$JOBSCRIPT
69  ###PBS -q ace-test                         # job queue not neccesary  so far  #PBS -q ace-r                            # job queue
70  #PBS -N $JOBNAME                         # give the job a name  #PBS -N $JOBNAME                         # give the job a name
71  #PBS -l cpunum_job=$NPROCS               # cpus per node  #PBS -l cpunum_job=$NPROCS               # cpus per node
72    #PBS -l elapstim_req=2:00:00
73  #PBS -l cputim_job=2:00:00               # time limit  #PBS -l cputim_job=2:00:00               # time limit
74  #PBS -l memsz_job=32gb                   # max accumulated memory, we need this much because of many netcdf files  #PBS -l memsz_job=32gb                   # max accumulated memory, we need this much because of many netcdf files
75  #PBS -j o                                # join i/o  #PBS -j o                                # join i/o
# Line 72  cat << EOF > $HERE/$JOBSCRIPT Line 78  cat << EOF > $HERE/$JOBSCRIPT
78  #  #
79    
80  cd \${PBS_O_WORKDIR}  cd \${PBS_O_WORKDIR}
81  $RUNTESTREPORT -runonly -command "$EXE" >> $MYOUTPUT 2>&1  $RUNTESTREPORT -runonly -command "$EXE" >> ${MYOUTPUT} 2>&1
82    
83  EOF  EOF
84    
85  # clean up old testreport output  # clean up old testreport output
86  if [ -e $MYOUTPUT ]; then  if [ -e ${MYOUTPUT} ]; then
87    rm -rf $MYOUTPUT    rm -rf ${MYOUTPUT}
88  fi  fi
89  if [ -e $OUTFILE ]; then  if [ -e $OUTFILE ]; then
90    rm -r $OUTFILE    rm -r $OUTFILE
91  fi  fi
92  if [ -e $TDIR/MITgcm ]; then  
93  # if availabe just update  # checkOut determines how much checking out is being done
94      cd $TDIR/MITgcm  # checkOut = 3: new clone from GitHub and make a new copy
95      cvs -q update -d -P >> cvs_co.log 2>&1  # checkOut = 2: update (git pull) existing repo and make a new copy
96      if [ "$?" != "0" ]; then  # checkOut = 1: skip update
97          cat cvs_co.log  # checkOut = 0: use existing test code (if available otherwise switch to 1)
98      fi  
99  else  checkOut=2
100  # create directory and download code  
101      if [ -e $TDIR ]; then  #gitcmd=$HOME/git/git
102          rm -rf $TDIR  gitcmd=git
103    tmpFil=$TDIR/error.out
104    if [ $checkOut -le 1 ] ; then
105      if test -e $TDIR/${gcmDIR}/doc ; then
106        echo $TDIR/${gcmDIR}/doc 'exist'
107      else
108        echo -n $TDIR/${gcmDIR} 'missing ; '
109        checkOut=2
110        echo "will make a new copy ( checkOut=$checkOut )"
111      fi
112    fi
113    
114    if [ $checkOut -ge 2 ] ; then
115      #---- cleaning:
116      cd $TDIR
117    
118      #---- Make a new clone or update existing one:
119      if test -e ${gcmDIR}/.git/config ; then
120        echo "${gcmDIR}/.git/config exist"
121      else
122        echo -n "${gcmDIR}/.git/config 'missing, "
123        checkOut=3
124        echo "will get new clone ( checkOut=$checkOut )"
125      fi
126      if [ $checkOut -eq 3 ] ; then
127        echo -n "Removing old clone: $TDIR/${gcmDIR} ..."
128        test -e $TDIR/${gcmDIR}  &&  rm -rf $TDIR/${gcmDIR}
129        echo "  done"
130        echo -n "Make a new clone of $git_code from repo: $git_repo ..."
131        ${gitcmd} clone https://github.com/$git_repo/${git_code}.git ${gcmDIR} 2> $tmpFil
132        retVal=$?
133        if test $retVal = 0 ; then
134           echo ' --> done!'
135           rm -f $tmpFil
136        else
137           echo " Error: 'git clone' returned: $retVal"
138           cat $tmpFil
139           rm -f $tmpFil
140           exit 2
141      fi      fi
142      mkdir $TDIR    else
143      cd $TDIR      echo "Updating current clone ( $git_code ) ..."
144      cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack co MITgcm > cvs_co.log 2>&1      ( cd ${gcmDIR}; ${gitcmd} checkout master ; ${gitcmd} pull )
145      if [ "$?" != "0" ]; then      retVal=$?
146          cat cvs_co.log      if test $retVal = 0 ; then
147            echo ' --> done!'
148        else
149            echo " Error: 'git pull' returned: $retVal"
150            echo " Error: 'git pull' returned: $retVal" \
151                | mail -s "Git-error on Stan" Martin.Losch@awi.de
152            exit 2
153      fi      fi
154      fi
155    else
156      cd $TDIR
157  fi  fi
158    
159  cd $TDIR/MITgcm/verification  cd $TDIR/MITgcm/verification
# Line 110  unset CC Line 163  unset CC
163  # make sure that do use the cross compiler for testreport  # make sure that do use the cross compiler for testreport
164  #export CC=sxcc  #export CC=sxcc
165    
166  $RUNTESTREPORT  -j 8 -norun > $MYOUTPUT 2>&1  $RUNTESTREPORT  -j 8 -norun > ${MYOUTPUT} 2>&1
167    
168  if [ "$?" != "0" ]  if [ "$?" != "0" ]
169      then      then
# Line 124  if [ "$?" != "0" ] Line 177  if [ "$?" != "0" ]
177  #  rm -rf $TDIR  #  rm -rf $TDIR
178  fi  fi
179    
180  if [ ! -e $MYOUTPUT ]  if [ ! -e ${MYOUTPUT} ]
181      then      then
182      touch $MYOUTPUT      touch ${MYOUTPUT}
183  fi  fi
184    
185  echo " " >> $MYOUTPUT  echo " " >> ${MYOUTPUT}
186  echo "***********************************************************" >> $MYOUTPUT  echo "***********************************************************" >>${MYOUTPUT}
187  echo "Submitting this job script:" >> $MYOUTPUT  echo "Submitting this job script:" >> ${MYOUTPUT}
188  echo "***********************************************************" >> $MYOUTPUT  echo "***********************************************************" >>${MYOUTPUT}
189  cat $HERE/$JOBSCRIPT >> $MYOUTPUT  cat $HERE/$JOBSCRIPT >> ${MYOUTPUT}
190  echo "***********************************************************" >> $MYOUTPUT  echo "***********************************************************" >>${MYOUTPUT}
191  echo "end of job script" >> $MYOUTPUT  echo "end of job script" >> ${MYOUTPUT}
192  echo "***********************************************************" >> $MYOUTPUT  echo "***********************************************************" >>${MYOUTPUT}
193  echo " " >> $MYOUTPUT  echo " " >> ${MYOUTPUT}
194    
195  # now submit the job that actually runs all the experiments in one go  # now submit the job that actually runs all the experiments in one go
196    echo "qsub $HERE/$JOBSCRIPT"
197  qsub $HERE/$JOBSCRIPT  qsub $HERE/$JOBSCRIPT
198  # keep looking for the job in the job queues and wait until has disappeared  # keep looking for the job in the job queues and wait until has disappeared
199  jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`  jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`
200  while [ "${jobruns}"x != x ]  while [ "${jobruns}"x != x ]
201  do  do
202    sleep 200    sleep 200
203    jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`    jobruns=`qstat -n -u mlosch | grep "$JOBNAME"`
204    echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"    echo "waiting for job ${jobruns%% *} ($JOBNAME) to complete"
205    currentexp=`grep Experiment $MYOUTPUT | tail -1`    currentexp=`grep Experiment ${MYOUTPUT} | tail -1`
206    echo "currently running $currentexp"    echo "currently running $currentexp"
207  done  done
208    
209  # after running the experiments on the compute node run testreport  # after running the experiments on the compute node run testreport
210  # for a third time to evaluate results on the head node again  # for a third time to evaluate results on the head node again
211  echo " " >> $MYOUTPUT  echo " " >> ${MYOUTPUT}
212  echo "now run testreport for a final time to evaluate results:" >> $MYOUTPUT  echo "now run testreport for a final time to evaluate results:" >> ${MYOUTPUT}
213  echo "$RUNTESTREPORT -match 10 -runonly" >> $MYOUTPUT  echo "$RUNTESTREPORT -match 10 -runonly" >> ${MYOUTPUT}
214  #$RUNTESTREPORT -match 10 -runonly >> $MYOUTPUT 2>&1  #$RUNTESTREPORT -match 10 -runonly >> ${MYOUTPUT} 2>&1
215  $RUNTESTREPORT -match 10 -runonly \  $RUNTESTREPORT -match 10 -runonly \
216      -a "jmc@mitgcm.org, Martin.Losch@awi.de" >> $MYOUTPUT 2>&1      -a "jm_c@mitgcm.org" >> ${MYOUTPUT} 2>&1
217    #   -a "jm_c@mitgcm.org, Martin.Losch@awi.de" >> ${MYOUTPUT} 2>&1
218    
219  echo "end of mitgcmtestreport"  echo "end of mitgcmtestreport"
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.6
  ViewVC Help
Powered by ViewVC 1.1.22