| 1 |
#! /usr/bin/env bash |
| 2 |
|
| 3 |
# $Header: /u/gcmpack/MITgcm/tools/example_scripts/ACESgrid/aces_test_all,v 1.8 2008/08/28 03:59:20 jmc Exp $ |
| 4 |
# $Name: $ |
| 5 |
|
| 6 |
# action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both |
| 7 |
case $1 in |
| 8 |
'-subOnly') action=2 ; shift ;; |
| 9 |
'-getOnly') action=1 ; shift ;; |
| 10 |
'-sub+get') action=3 ; shift ;; |
| 11 |
*) action=3 ;; |
| 12 |
esac |
| 13 |
#echo "action= $action , Arg= $# " |
| 14 |
|
| 15 |
echo -n '-- Starting: ' |
| 16 |
date |
| 17 |
start=`date +%Y%m%d` |
| 18 |
|
| 19 |
if test $# = 0 |
| 20 |
then |
| 21 |
tst_list='gnu ifc pgi mth tuv' |
| 22 |
else |
| 23 |
tst_list=$* |
| 24 |
fi |
| 25 |
|
| 26 |
. /etc/profile.d/modules.sh |
| 27 |
module list > ~/tmp/.module_list 2>&1 |
| 28 |
|
| 29 |
cd /home/jmc/test_ACES |
| 30 |
|
| 31 |
# QSUB="/usr/local/bin/qsub" |
| 32 |
# QSTAT="/usr/local/bin/qstat" |
| 33 |
QSUB=qsub |
| 34 |
QSTAT=qstat |
| 35 |
MPACK="MITgcm_tools/mpack-1.6/mpack" |
| 36 |
TST_DIR="/home/jmc/test_ACES/gcm_tests" |
| 37 |
|
| 38 |
nbtst=0 |
| 39 |
for i in $tst_list |
| 40 |
do |
| 41 |
sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/' | sed 's/tuv_mpi/op64_tuv/'` |
| 42 |
if test -f MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx ; then |
| 43 |
JOB="tst_"$i |
| 44 |
job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l` |
| 45 |
if [ $action -ge 2 ] ; then |
| 46 |
#-- to submit job |
| 47 |
if test "x_$job_exist" = x_0 ; then |
| 48 |
$QSUB MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx |
| 49 |
eval M_$i='submitted' |
| 50 |
nbtst=`expr $nbtst + 1` |
| 51 |
else |
| 52 |
echo $JOB |
| 53 |
$QSTAT -a | grep $USER | grep $JOB |
| 54 |
echo 'job already exist => skip this test' |
| 55 |
eval M_$i='skipped' |
| 56 |
fi |
| 57 |
else |
| 58 |
#-- to get outp back: |
| 59 |
if test "x_$job_exist" = x_0 ; then |
| 60 |
echo -n 'did not find any job: ' |
| 61 |
echo $JOB |
| 62 |
eval M_$i='skipped' |
| 63 |
else |
| 64 |
echo -n 'found a job: ' |
| 65 |
echo $JOB |
| 66 |
$QSTAT -a | grep $USER | grep $JOB |
| 67 |
eval M_$i='submitted' |
| 68 |
nbtst=`expr $nbtst + 1` |
| 69 |
fi |
| 70 |
fi |
| 71 |
else |
| 72 |
echo 'no file:' aces_test_$sfx 'to submit' |
| 73 |
eval M_$i='skipped' |
| 74 |
fi |
| 75 |
done |
| 76 |
|
| 77 |
if test $action = 2 ; then |
| 78 |
count=0 |
| 79 |
else |
| 80 |
#date_str=`date +%Y%m%d`"_0" |
| 81 |
|
| 82 |
MPACKDIR=`dirname $MPACK` |
| 83 |
#- build the mpack utility (from build_mpack in testreport): |
| 84 |
( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1 |
| 85 |
RETVAL=$? |
| 86 |
if test "x$RETVAL" != x0 ; then |
| 87 |
echo |
| 88 |
echo "Error building the mpack tools at: $MPACK_DIR" |
| 89 |
if test -x $MPACK ; then |
| 90 |
HAVE_MPACK=t |
| 91 |
echo " use (old ?) executable:" ; ls -l $MPACK |
| 92 |
else |
| 93 |
HAVE_MPACK=f |
| 94 |
fi |
| 95 |
else |
| 96 |
if test -x $MPACK ; then |
| 97 |
rm -f TTT.build_mpack.$$ |
| 98 |
HAVE_MPACK=t |
| 99 |
echo " building mpack: OK" |
| 100 |
else |
| 101 |
echo " $MPACK not executable" |
| 102 |
HAVE_MPACK=f |
| 103 |
fi |
| 104 |
fi |
| 105 |
echo |
| 106 |
|
| 107 |
count=$nbtst |
| 108 |
fi |
| 109 |
|
| 110 |
#- when it's done, retrieve output and send e-mail |
| 111 |
minutes=0 |
| 112 |
while test $count != 0 ; do |
| 113 |
|
| 114 |
sleep 600 |
| 115 |
minutes=$(( $minutes + 10 )) |
| 116 |
|
| 117 |
for i in $tst_list ; do |
| 118 |
|
| 119 |
eval comm=\$M_$i |
| 120 |
if test $comm = 'submitted' ; then |
| 121 |
JOB="tst_"$i |
| 122 |
ready_to_send=`$QSTAT -a | grep $USER | grep $JOB | wc -l` |
| 123 |
|
| 124 |
if test "x_$ready_to_send" = x_0 ; then |
| 125 |
run_dir=${TST_DIR}"/MITgcm_"$i"/verification" |
| 126 |
tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^tr_aces' | head -1` |
| 127 |
#- check this is the right output |
| 128 |
chk=`echo $tdir | grep -c $start` |
| 129 |
if test $chk = '0' ; then |
| 130 |
curday=`date +%Y%m%d` |
| 131 |
chk=`echo $tdir | grep -c $curday` |
| 132 |
fi |
| 133 |
if test "x$tdir" = x ; then |
| 134 |
echo "Output not found so no email sent for $i" |
| 135 |
elif test $chk = '0' ; then |
| 136 |
echo "tdir='$tdir'" |
| 137 |
echo "Output do not match so no email sent for $i" |
| 138 |
else |
| 139 |
rm -f "/tmp/tr_aces-"$i".tar.gz" |
| 140 |
( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir ) |
| 141 |
if test "x$HAVE_MPACK" = xt ; then |
| 142 |
$MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org |
| 143 |
echo "Email sent for $i" |
| 144 |
else |
| 145 |
echo "no email sent for $i (no mpack)" |
| 146 |
fi |
| 147 |
fi |
| 148 |
eval M_$i=done |
| 149 |
count=`expr $count - 1` |
| 150 |
chmod 644 output/tst_$i.std* |
| 151 |
fi |
| 152 |
fi |
| 153 |
done |
| 154 |
|
| 155 |
# "long" queue is 24hrs = 24*60min = 1440min |
| 156 |
if test $minutes -gt 2160 ; then |
| 157 |
hrs=$(( $minutes / 60 )); |
| 158 |
echo "Time expired after $minutes minutes ($hrs hours)" |
| 159 |
echo ' ' $count '/' $nbtst 'tests not yet finished' |
| 160 |
exit 1 |
| 161 |
fi |
| 162 |
|
| 163 |
done |
| 164 |
|
| 165 |
echo "All $nbtst tests completed successfully." |
| 166 |
echo -n '-- Finished: ' |
| 167 |
date |
Parent Directory
| 
Revision Log
| 
Revision Graph