/[MITgcm]/MITgcm/tools/example_scripts/ACESgrid/aces_test_all
ViewVC logotype

Diff of /MITgcm/tools/example_scripts/ACESgrid/aces_test_all

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.5 by jmc, Fri Dec 14 19:30:35 2007 UTC revision 1.17 by jmc, Mon May 10 14:03:18 2010 UTC
# Line 3  Line 3 
3  # $Header$  # $Header$
4  # $Name$  # $Name$
5    
6  echo -n '-- Starting: '  # action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both
7  date  case $1 in
8     '-subOnly') action=2 ; shift ;;
9     '-getOnly') action=1 ; shift ;;
10     '-sub+get') action=3 ; shift ;;
11              *) action=3 ;;
12    esac
13    #echo "action= $action , Arg= $# "
14    
15    today=`date +%Y%m%d`
16    
17  if test $# = 0  if test $# = 0
18  then  then
19    tst_list='gnu ifc pgi mth tuv'    tst_list='gnu ifc pgi adm mp2 mth tuv'
20  else  else
21    tst_list=$*    tst_list=$*
22  fi  fi
23    
 . /etc/profile.d/modules.sh  
 module list > ~/tmp/.module_list 2>&1  
   
 cd /home/jmc/test_ACES  
   
24  # QSUB="/usr/local/bin/qsub"  # QSUB="/usr/local/bin/qsub"
25  # QSTAT="/usr/local/bin/qstat"  # QSTAT="/usr/local/bin/qstat"
26  QSUB=qsub  QSUB=qsub
27  QSTAT=qstat  QSTAT=qstat
28    TST_DIR="/home/jmc/test_ACES"
29  MPACK="MITgcm_tools/mpack-1.6/mpack"  MPACK="MITgcm_tools/mpack-1.6/mpack"
30  TST_DIR="/home/jmc/test_ACES/gcm_tests"  SUB_DIR="MITgcm_tools/example_scripts/ACESgrid"
31    
32    TMP_FIL="$TST_DIR/output/TTT.$$"
33    LOG_FIL="$TST_DIR/output/tst_all."`date +%m%d`".log"
34    
35    rm -f $LOG_FIL ; touch $LOG_FIL
36    echo -n '-- Starting: ' | tee -a $LOG_FIL
37    date | tee -a $LOG_FIL
38    
39    #-- clean up old log files:
40    cd $TST_DIR/output
41    n=$(( `ls tst_all.*.log | wc -l` - 10 ))
42    if test $n -gt 0 ; then
43      echo ' remove old log files:' | tee -a $LOG_FIL
44        ls -lt tst_all.*.log | tail -"$n" | tee -a $LOG_FIL
45        ls -t  tst_all.*.log | tail -"$n" | xargs rm -f
46    fi
47    
48    . /etc/profile.d/modules.sh
49    module list >> $LOG_FIL 2>&1
50    
51    #-- now really do something:
52    
53    cd $TST_DIR
54    
55  nbtst=0  nbtst=0
56  for i in $tst_list  for i in $tst_list
57  do  do
58    sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/' | sed 's/tuv_mpi/op64_tuv/'`    sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/' | sed 's/tuv_mpi/op64_tuv/' | sed 's/mp2_mpi/mp2_mth/'`
59    if test -f MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx ; then    if test -f $SUB_DIR/aces_test_$sfx ; then
60      JOB="tst_"$i      JOB="tst_"$i
61      job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l`      job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l`
62      if test "x_$job_exist" = x_0 ; then      if [ $action -ge 2 ] ; then
63        $QSUB MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx      #-- to submit job
64        eval M_$i='submitted'        if test "x_$job_exist" = x_0 ; then
65        nbtst=`expr $nbtst + 1`          echo -n $JOB | tee -a $LOG_FIL
66      else          $QSUB $SUB_DIR/aces_test_$sfx | tee -a $LOG_FIL
67        echo $JOB          eval M_$i='submitted'
68        $QSTAT -a | grep $USER | grep $JOB          nbtst=`expr $nbtst + 1`
69        echo 'job already exist => skip this test'        else
70        eval M_$i='skipped'          echo $JOB | tee -a $LOG_FIL
71            $QSTAT -a | grep $USER | grep $JOB | tee -a $LOG_FIL
72            echo 'job already exist => skip this test' | tee -a $LOG_FIL
73            eval M_$i='skipped'
74          fi
75        else
76        #-- to get outp back:
77          if test "x_$job_exist" = x_0 ; then
78            echo -n 'did not find any job: ' | tee -a $LOG_FIL
79            echo $JOB | tee -a $LOG_FIL
80            eval M_$i='skipped'
81          else
82            echo -n 'found a job: ' | tee -a $LOG_FIL
83            echo $JOB | tee -a $LOG_FIL
84            $QSTAT -a | grep $USER | grep $JOB | tee -a $LOG_FIL
85            eval M_$i='submitted'
86            nbtst=`expr $nbtst + 1`
87          fi
88      fi      fi
89    else    else
90      echo 'no file:' aces_test_$sfx 'to submit'      echo 'no file:' aces_test_$sfx 'to submit' | tee -a $LOG_FIL
91      eval M_$i='skipped'      eval M_$i='skipped'
92    fi    fi
93  done  done
94    
95    if test $action = 2 ; then
96      count=0
97    else
98  #date_str=`date +%Y%m%d`"_0"  #date_str=`date +%Y%m%d`"_0"
99    
100  minutes=0    MPACKDIR=`dirname $MPACK`
101  count=$nbtst  #- build the mpack utility (from build_mpack in testreport):
102      ( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1
103      RETVAL=$?
104      if test "x$RETVAL" != x0 ; then
105        echo
106        echo "Error building the mpack tools at: $MPACK_DIR" | tee -a $LOG_FIL
107        if test -x $MPACK ; then
108           HAVE_MPACK=t
109           echo "  use (old ?) executable:" | tee -a $LOG_FIL
110           ls -l $MPACK | tee -a $LOG_FIL
111        else
112           HAVE_MPACK=f
113        fi
114      else
115        if test -x $MPACK ; then
116           rm -f TTT.build_mpack.$$
117           HAVE_MPACK=t
118           echo " building mpack: OK" | tee -a $LOG_FIL
119        else
120           echo " $MPACK not executable" | tee -a $LOG_FIL
121           HAVE_MPACK=f
122        fi
123      fi
124      echo
125    
126      count=$nbtst
127    fi
128    
129    #- when it's done, retrieve output and send e-mail
130    minutes=0 ; freq=10
131    fsec=`expr $freq \* 60`
132    echo "Check every $freq mn for $count test(s) to finish" | tee -a $LOG_FIL
133    echo "- start at :" `date` | tee -a $LOG_FIL
134  while test $count != 0 ; do  while test $count != 0 ; do
135    
136      sleep 600      sleep $fsec
137      minutes=$(( $minutes + 10 ))      minutes=$(( $minutes + $freq ))
138    
139      for i in $tst_list ; do      for i in $tst_list ; do
140    
141        eval comm=\$M_$i        eval comm=\$M_$i
142        if test $comm = 'submitted' ; then        if test $comm = 'submitted' ; then
143          JOB="tst_"$i          JOB="tst_"$i
144          ready_to_send=`$QSTAT -a | grep $USER | grep $JOB | wc -l`          $QSTAT -a > $TMP_FIL
145            RETVAL=$?
146            ready_to_send=`grep $USER $TMP_FIL | grep $JOB | wc -l`
147            rm -f $TMP_FIL
148            if test "x$RETVAL" != x0 ; then
149               echo " $QSTAT returned with error code: $RETVAL" | tee -a $LOG_FIL
150               continue
151            fi
152    
153          if test "x_$ready_to_send" = x_0 ; then          if test "x_$ready_to_send" = x_0 ; then
154              pat="/tmp/tr_aces-$i"'*'            run_dir=${TST_DIR}"/MITgcm_"$i"/verification"
155              rm -rf $pat  #-      results output:
156              run_dir=${TST_DIR}"/tmp_"$i"/MITgcm/verification"            tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^tr_aces' | head -1`
157              tdir=`ls -1 $run_dir | grep -v tr_out | grep '^tr_aces' | head -1`            if test "x$tdir" != x ; then
158              if test ! "x$tdir" = x ; then              #- check this is the right output
159                  ( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )              chk=`echo $tdir | grep -c $today`
160                if test $chk = '0' ; then
161                  curday=`date +%Y%m%d`
162                  chk=`echo $tdir | grep -c $curday`
163                fi
164                if test $chk = '0' ; then
165                  echo "tdir='$tdir'" | tee -a $LOG_FIL
166                  echo "Output do not match, no email sent for $i" | tee -a $LOG_FIL
167                else
168                  rm -f "/tmp/tr_aces-"$i".tar.gz"
169                  ( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )
170                  if test "x$HAVE_MPACK" = xt ; then
171                  $MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org                  $MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org
172                  echo "Email sent for $i"                  echo "Email sent for $i at:     " `date` | tee -a $LOG_FIL
173                  else
174                    echo " no email sent for $i (no mpack)" | tee -a $LOG_FIL
175                  fi
176                fi
177              else
178                  echo " no output found for $i" | tee -a $LOG_FIL
179              fi
180    #-      restart output:
181              tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^rs_aces' | head -1`
182              if test "x$tdir" != x ; then
183                #- check this is the right output
184                chk=`echo $tdir | grep -c $today`
185                if test $chk = '0' ; then
186                  curday=`date +%Y%m%d`
187                  chk=`echo $tdir | grep -c $curday`
188                fi
189                if test $chk = '0' ; then
190                  echo "tdir='$tdir'" | tee -a $LOG_FIL
191                  echo "Restart output do not match, no email sent for $i" | tee -a $LOG_FIL
192              else              else
193                  echo "Output not found so no email sent for $i"                rm -f "/tmp/rs_aces-"$i".tar.gz"
194                  ( cd $run_dir ; tar -czf "/tmp/rs_aces-"$i".tar.gz" ./$tdir )
195                  if test "x$HAVE_MPACK" = xt ; then
196                    $MPACK -s MITgcm-test -m 3555000 "/tmp/rs_aces-"$i".tar.gz" jmc@mitgcm.org
197                    echo "Email sent for $i restart:" `date` | tee -a $LOG_FIL
198                  else
199                    echo " no email sent for $i restart (no mpack)" | tee -a $LOG_FIL
200                  fi
201              fi              fi
202              eval M_$i=done            else
203              count=`expr $count - 1`                echo " no restart output for $i" | tee -a $LOG_FIL
204              chmod 644 output/tst_$i.std*            fi
205    #-      record successful sending
206              eval M_$i=done
207              count=`expr $count - 1`
208              chmod 644 output/tst_$i.std*
209          fi          fi
210        fi        fi
211      done      done
212        
213      #  "long" queue is 24hrs = 24*60min = 1440min      #  "long" queue is 24hrs = 24*60min = 1440min
214      if test $minutes -gt 2160 ; then      if test $minutes -gt 2160 ; then
215          hrs=$(( $minutes / 60 ));          hrs=$(( $minutes / 60 ));
216          echo "Time expired after $minutes minutes ($hrs hours)"          echo "Time expired after $minutes minutes ($hrs hours)" | tee -a $LOG_FIL
217          echo ' ' $count '/' $nbtst 'tests not yet finished'          echo ' ' $count '/' $nbtst 'tests not yet finished' | tee -a $LOG_FIL
218          exit 1          exit 1
219      fi      fi
220    
221  done  done
222    
223  echo "All $nbtst tests completed successfully."  echo "Retrieving $nbtst tests finish :" `date` | tee -a $LOG_FIL
224  echo -n '-- Finished: '  exit 0
 date  
225    
Legend:
Removed from v.1.5  
changed lines
  Added in v.1.17
  ViewVC Help
Powered by ViewVC 1.1.22