3 |
# $Header$ |
# $Header$ |
4 |
# $Name$ |
# $Name$ |
5 |
|
|
6 |
|
echo -n '-- Starting: ' |
7 |
|
date |
8 |
|
start=`date +%Y%m%d` |
9 |
if test $# = 0 |
if test $# = 0 |
10 |
then |
then |
11 |
tst_list='gnu ifc pgi mth' |
tst_list='gnu ifc pgi mth tuv' |
12 |
else |
else |
13 |
tst_list=$* |
tst_list=$* |
14 |
fi |
fi |
16 |
. /etc/profile.d/modules.sh |
. /etc/profile.d/modules.sh |
17 |
module list > ~/tmp/.module_list 2>&1 |
module list > ~/tmp/.module_list 2>&1 |
18 |
|
|
19 |
cd /home/jmc/test_ACES/bin |
cd /home/jmc/test_ACES |
20 |
|
|
21 |
# QSUB="/usr/local/bin/qsub" |
# QSUB="/usr/local/bin/qsub" |
22 |
# QSTAT="/usr/local/bin/qstat" |
# QSTAT="/usr/local/bin/qstat" |
23 |
QSUB=qsub |
QSUB=qsub |
24 |
QSTAT=qstat |
QSTAT=qstat |
25 |
MPACK="./mpack" |
MPACK="MITgcm_tools/mpack-1.6/mpack" |
26 |
TST_DIR="/home/jmc/test_ACES/gcm_tests" |
TST_DIR="/home/jmc/test_ACES/gcm_tests" |
27 |
|
|
28 |
nbtst=0 |
nbtst=0 |
29 |
for i in $tst_list |
for i in $tst_list |
30 |
do |
do |
31 |
sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/g'` |
sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/' | sed 's/tuv_mpi/op64_tuv/'` |
32 |
if test -f ./aces_test_$sfx ; then |
if test -f MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx ; then |
33 |
$QSUB ./aces_test_$sfx |
JOB="tst_"$i |
34 |
eval M_$i='submitted' |
job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l` |
35 |
nbtst=`expr $nbtst + 1` |
if test "x_$job_exist" = x_0 ; then |
36 |
|
$QSUB MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx |
37 |
|
eval M_$i='submitted' |
38 |
|
nbtst=`expr $nbtst + 1` |
39 |
|
else |
40 |
|
echo $JOB |
41 |
|
$QSTAT -a | grep $USER | grep $JOB |
42 |
|
echo 'job already exist => skip this test' |
43 |
|
eval M_$i='skipped' |
44 |
|
fi |
45 |
else |
else |
46 |
echo 'no file:' aces_test_$sfx 'to submit' |
echo 'no file:' aces_test_$sfx 'to submit' |
47 |
|
eval M_$i='skipped' |
48 |
fi |
fi |
49 |
done |
done |
50 |
|
|
51 |
date_str=`date +%Y%m%d`"_0" |
#date_str=`date +%Y%m%d`"_0" |
52 |
|
|
53 |
|
MPACKDIR=`dirname $MPACK` |
54 |
|
#- build the mpack utility (from build_mpack in testreport): |
55 |
|
( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1 |
56 |
|
RETVAL=$? |
57 |
|
if test "x$RETVAL" != x0 ; then |
58 |
|
echo |
59 |
|
echo "Error building the mpack tools at: $MPACK_DIR" |
60 |
|
if test -x $MPACK ; then |
61 |
|
HAVE_MPACK=t |
62 |
|
echo " use (old ?) executable:" ; ls -l $MPACK |
63 |
|
else |
64 |
|
HAVE_MPACK=f |
65 |
|
fi |
66 |
|
else |
67 |
|
if test -x $MPACK ; then |
68 |
|
rm -f TTT.build_mpack.$$ |
69 |
|
HAVE_MPACK=t |
70 |
|
echo " building mpack: OK" |
71 |
|
else |
72 |
|
echo " $MPACK not executable" |
73 |
|
HAVE_MPACK=f |
74 |
|
fi |
75 |
|
fi |
76 |
|
echo |
77 |
|
|
78 |
minutes=0 |
minutes=0 |
79 |
count=$nbtst |
count=$nbtst |
80 |
while test $count != 0 ; do |
while test $count != 0 ; do |
81 |
|
|
82 |
sleep 300 |
sleep 600 |
83 |
minutes=$(( $minutes + 5 )) |
minutes=$(( $minutes + 10 )) |
84 |
|
|
85 |
for i in $tst_list ; do |
for i in $tst_list ; do |
86 |
|
|
90 |
ready_to_send=`$QSTAT -a | grep $USER | grep $JOB | wc -l` |
ready_to_send=`$QSTAT -a | grep $USER | grep $JOB | wc -l` |
91 |
|
|
92 |
if test "x_$ready_to_send" = x_0 ; then |
if test "x_$ready_to_send" = x_0 ; then |
93 |
pat="/tmp/tr_aces-$i"'*' |
run_dir=${TST_DIR}"/MITgcm_"$i"/verification" |
94 |
rm -rf $pat |
tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^tr_aces' | head -1` |
95 |
run_dir=${TST_DIR}"/tmp_"$i"/MITgcm/verification" |
#- check this is the right output |
96 |
tdir=`ls -1 $run_dir | grep -v tr_out | grep '^tr' | head -1` |
chk=`echo $tdir | grep -c $start` |
97 |
if test ! "x$tdir" = x ; then |
if test $chk = '0' ; then |
98 |
cp -ra $run_dir"/"$tdir "/tmp/tr_aces-"$i"_"$date_str |
curday=`date +%Y%m%d` |
99 |
( cd /tmp ; tar -czf "./tr_aces-"$i".tar.gz" "./tr_aces-"$i"_"$date_str ) |
chk=`echo $tdir | grep -c $curday` |
100 |
$MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" edhill@mitgcm.org |
fi |
101 |
echo "Email sent for $i" |
if test "x$tdir" = x ; then |
102 |
|
echo "Output not found so no email sent for $i" |
103 |
|
elif test $chk = '0' ; then |
104 |
|
echo "tdir='$tdir'" |
105 |
|
echo "Output do not match so no email sent for $i" |
106 |
else |
else |
107 |
echo "Output not found so no email sent for $i" |
rm -f "/tmp/tr_aces-"$i".tar.gz" |
108 |
|
( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir ) |
109 |
|
if test "x$HAVE_MPACK" = xt ; then |
110 |
|
$MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org |
111 |
|
echo "Email sent for $i" |
112 |
|
else |
113 |
|
echo "no email sent for $i (no mpack)" |
114 |
|
fi |
115 |
fi |
fi |
116 |
eval M_$i=done |
eval M_$i=done |
117 |
count=`expr $count - 1` |
count=`expr $count - 1` |
118 |
|
chmod 644 output/tst_$i.std* |
119 |
fi |
fi |
120 |
fi |
fi |
121 |
done |
done |
122 |
|
|
|
hrs=$(( $minutes / 60 )); |
|
123 |
# "long" queue is 24hrs = 24*60min = 1440min |
# "long" queue is 24hrs = 24*60min = 1440min |
124 |
if test $minutes -gt 1440 ; then |
if test $minutes -gt 2160 ; then |
125 |
|
hrs=$(( $minutes / 60 )); |
126 |
echo "Time expired after $minutes minutes ($hrs hours)" |
echo "Time expired after $minutes minutes ($hrs hours)" |
127 |
echo ' ' $count '/' $nbtst 'tests not yet finished' |
echo ' ' $count '/' $nbtst 'tests not yet finished' |
128 |
exit 1 |
exit 1 |
131 |
done |
done |
132 |
|
|
133 |
echo "All $nbtst tests completed successfully." |
echo "All $nbtst tests completed successfully." |
134 |
|
echo -n '-- Finished: ' |
135 |
|
date |
136 |
|
|