Annotation of /MITgcm/tools/example_scripts/ACESgrid/aces_test_all

#! /usr/bin/env bash

# $Header: /u/gcmpack/MITgcm/tools/example_scripts/ACESgrid/aces_test_all,v 1.14 2010/04/01 15:46:03 jmc Exp $
# $Name:  $

# action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both
case $1 in
 '-subOnly') action=2 ; shift ;;
 '-getOnly') action=1 ; shift ;;
 '-sub+get') action=3 ; shift ;;
          *) action=3 ;;
esac
#echo "action= $action , Arg= $# "

echo -n '-- Starting: '
date
start=`date +%Y%m%d`

if test $# = 0
then
  tst_list='gnu ifc pgi adm mp2 mth tuv'
else
  tst_list=$*
fi

. /etc/profile.d/modules.sh
module list > ~/tmp/.module_list 2>&1

# QSUB="/usr/local/bin/qsub"
# QSTAT="/usr/local/bin/qstat"
QSUB=qsub
QSTAT=qstat
TST_DIR="/home/jmc/test_ACES"
MPACK="MITgcm_tools/mpack-1.6/mpack"
SUB_DIR="MITgcm_tools/example_scripts/ACESgrid"

LOG_FIL="$TST_DIR/output/tst_all.log"
rm -f $LOG_FIL

cd $TST_DIR

nbtst=0
for i in $tst_list
do
  sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/' | sed 's/tuv_mpi/op64_tuv/' | sed 's/mp2_mpi/mp2_mth/'`
  if test -f $SUB_DIR/aces_test_$sfx ; then
    JOB="tst_"$i
    job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l`
    if [ $action -ge 2 ] ; then
    #-- to submit job
      if test "x_$job_exist" = x_0 ; then
        $QSUB $SUB_DIR/aces_test_$sfx
        eval M_$i='submitted'
        nbtst=`expr $nbtst + 1`
      else
        echo $JOB
        $QSTAT -a | grep $USER | grep $JOB
        echo 'job already exist => skip this test'
        eval M_$i='skipped'
      fi
    else
    #-- to get outp back:
      if test "x_$job_exist" = x_0 ; then
        echo -n 'did not find any job: '
        echo $JOB
        eval M_$i='skipped'
      else
        echo -n 'found a job: '
        echo $JOB
        $QSTAT -a | grep $USER | grep $JOB
        eval M_$i='submitted'
        nbtst=`expr $nbtst + 1`
      fi
    fi
  else
    echo 'no file:' aces_test_$sfx 'to submit'
    eval M_$i='skipped'
  fi
done

if test $action = 2 ; then
  count=0
else
#date_str=`date +%Y%m%d`"_0"

  MPACKDIR=`dirname $MPACK`
#- build the mpack utility (from build_mpack in testreport):
  ( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1
  RETVAL=$?
  if test "x$RETVAL" != x0 ; then
    echo
    echo "Error building the mpack tools at: $MPACK_DIR"
    if test -x $MPACK ; then
       HAVE_MPACK=t
       echo "  use (old ?) executable:" ; ls -l $MPACK
    else
       HAVE_MPACK=f
    fi
  else
    if test -x $MPACK ; then
       rm -f TTT.build_mpack.$$
       HAVE_MPACK=t
       echo " building mpack: OK"
    else
       echo " $MPACK not executable"
       HAVE_MPACK=f
    fi
  fi
  echo

  count=$nbtst
fi

#- when it's done, retrieve output and send e-mail
minutes=0 ; freq=10
fsec=`expr $freq \* 60`
echo "Check every $freq mn for $count test(s) to finish" | tee -a $LOG_FIL
echo "- start at :" `date` | tee -a $LOG_FIL
while test $count != 0 ; do

    sleep $fsec
    minutes=$(( $minutes + $freq ))

    for i in $tst_list ; do

      eval comm=\$M_$i
      if test $comm = 'submitted' ; then
        JOB="tst_"$i
        ready_to_send=`$QSTAT -a | grep $USER | grep $JOB | wc -l`

        if test "x_$ready_to_send" = x_0 ; then
          run_dir=${TST_DIR}"/MITgcm_"$i"/verification"
#-      results output:
          tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^tr_aces' | head -1`
          if test "x$tdir" != x ; then
            #- check this is the right output
            chk=`echo $tdir | grep -c $start`
            if test $chk = '0' ; then
              curday=`date +%Y%m%d`
              chk=`echo $tdir | grep -c $curday`
            fi
            if test $chk = '0' ; then
              echo "tdir='$tdir'" | tee -a $LOG_FIL
              echo "Output do not match, no email sent for $i" | tee -a $LOG_FIL
            else
              rm -f "/tmp/tr_aces-"$i".tar.gz"
              ( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )
              if test "x$HAVE_MPACK" = xt ; then
                $MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org
                echo "Email sent for $i at:     " `date` | tee -a $LOG_FIL
              else
                echo " no email sent for $i (no mpack)" | tee -a $LOG_FIL
              fi
            fi
          else
              echo " no output found for $i" | tee -a $LOG_FIL
          fi
#-      restart output:
          tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^rs_aces' | head -1`
          if test "x$tdir" != x ; then
            #- check this is the right output
            chk=`echo $tdir | grep -c $start`
            if test $chk = '0' ; then
              curday=`date +%Y%m%d`
              chk=`echo $tdir | grep -c $curday`
            fi
            if test $chk = '0' ; then
              echo "tdir='$tdir'" | tee -a $LOG_FIL
              echo "Restart output do not match, no email sent for $i" | tee -a $LOG_FIL
            else
              rm -f "/tmp/rs_aces-"$i".tar.gz"
              ( cd $run_dir ; tar -czf "/tmp/rs_aces-"$i".tar.gz" ./$tdir )
              if test "x$HAVE_MPACK" = xt ; then
                $MPACK -s MITgcm-test -m 3555000 "/tmp/rs_aces-"$i".tar.gz" jmc@mitgcm.org
                echo "Email sent for $i restart:" `date` | tee -a $LOG_FIL
              else
                echo " no email sent for $i restart (no mpack)" | tee -a $LOG_FIL
              fi
            fi
          else
              echo " no restart output for $i" | tee -a $LOG_FIL
          fi
#-      record successful sending
          eval M_$i=done
          count=`expr $count - 1`
          chmod 644 output/tst_$i.std*
        fi
      fi
    done

    #  "long" queue is 24hrs = 24*60min = 1440min
    if test $minutes -gt 2160 ; then
        hrs=$(( $minutes / 60 ));
        echo "Time expired after $minutes minutes ($hrs hours)"
        echo ' ' $count '/' $nbtst 'tests not yet finished'
        exit 1
    fi

done

echo "Retrieving $nbtst tests finish :" `date` | tee -a $LOG_FIL
exit 0

1	jmc	1.1	#! /usr/bin/env bash
2
3	jmc	1.15	# $Header: /u/gcmpack/MITgcm/tools/example_scripts/ACESgrid/aces_test_all,v 1.14 2010/04/01 15:46:03 jmc Exp $
4	jmc	1.1	# $Name: $
5
6	jmc	1.9	# action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both
7			case $1 in
8			'-subOnly') action=2 ; shift ;;
9			'-getOnly') action=1 ; shift ;;
10			'-sub+get') action=3 ; shift ;;
11			*) action=3 ;;
12			esac
13			#echo "action= $action , Arg= $# "
14
15	jmc	1.4	echo -n '-- Starting: '
16			date
17	jmc	1.8	start=`date +%Y%m%d`
18	jmc	1.9
19	jmc	1.1	if test $# = 0
20			then
21	jmc	1.13	tst_list='gnu ifc pgi adm mp2 mth tuv'
22	jmc	1.1	else
23			tst_list=$*
24			fi
25
26			. /etc/profile.d/modules.sh
27			module list > ~/tmp/.module_list 2>&1
28
29			# QSUB="/usr/local/bin/qsub"
30			# QSTAT="/usr/local/bin/qstat"
31			QSUB=qsub
32			QSTAT=qstat
33	jmc	1.14	TST_DIR="/home/jmc/test_ACES"
34	jmc	1.4	MPACK="MITgcm_tools/mpack-1.6/mpack"
35	jmc	1.14	SUB_DIR="MITgcm_tools/example_scripts/ACESgrid"
36
37	jmc	1.15	LOG_FIL="$TST_DIR/output/tst_all.log"
38			rm -f $LOG_FIL
39
40	jmc	1.14	cd $TST_DIR
41	jmc	1.1
42			nbtst=0
43			for i in $tst_list
44			do
45	jmc	1.13	sfx=`echo ${i}"_mpi" \| sed 's/mth_mpi/ifc_mth/' \| sed 's/tuv_mpi/op64_tuv/' \| sed 's/mp2_mpi/mp2_mth/'`
46	jmc	1.14	if test -f $SUB_DIR/aces_test_$sfx ; then
47	jmc	1.4	JOB="tst_"$i
48			job_exist=`$QSTAT -a \| grep $USER \| grep $JOB \| wc -l`
49	jmc	1.15	if [ $action -ge 2 ] ; then
50	jmc	1.9	#-- to submit job
51			if test "x_$job_exist" = x_0 ; then
52	jmc	1.14	$QSUB $SUB_DIR/aces_test_$sfx
53	jmc	1.9	eval M_$i='submitted'
54			nbtst=`expr $nbtst + 1`
55	jmc	1.15	else
56	jmc	1.9	echo $JOB
57			$QSTAT -a \| grep $USER \| grep $JOB
58			echo 'job already exist => skip this test'
59			eval M_$i='skipped'
60			fi
61			else
62			#-- to get outp back:
63			if test "x_$job_exist" = x_0 ; then
64			echo -n 'did not find any job: '
65			echo $JOB
66			eval M_$i='skipped'
67	jmc	1.15	else
68	jmc	1.9	echo -n 'found a job: '
69			echo $JOB
70			$QSTAT -a \| grep $USER \| grep $JOB
71			eval M_$i='submitted'
72			nbtst=`expr $nbtst + 1`
73			fi
74	jmc	1.4	fi
75	jmc	1.15	else
76	jmc	1.1	echo 'no file:' aces_test_$sfx 'to submit'
77	jmc	1.4	eval M_$i='skipped'
78	jmc	1.1	fi
79			done
80
81	jmc	1.15	if test $action = 2 ; then
82	jmc	1.9	count=0
83			else
84	jmc	1.4	#date_str=`date +%Y%m%d`"_0"
85	jmc	1.1
86	jmc	1.9	MPACKDIR=`dirname $MPACK`
87	jmc	1.7	#- build the mpack utility (from build_mpack in testreport):
88	jmc	1.9	( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1
89			RETVAL=$?
90			if test "x$RETVAL" != x0 ; then
91	jmc	1.7	echo
92			echo "Error building the mpack tools at: $MPACK_DIR"
93			if test -x $MPACK ; then
94			HAVE_MPACK=t
95			echo " use (old ?) executable:" ; ls -l $MPACK
96			else
97			HAVE_MPACK=f
98			fi
99	jmc	1.9	else
100	jmc	1.7	if test -x $MPACK ; then
101			rm -f TTT.build_mpack.$$
102			HAVE_MPACK=t
103			echo " building mpack: OK"
104			else
105			echo " $MPACK not executable"
106			HAVE_MPACK=f
107			fi
108	jmc	1.9	fi
109			echo
110
111			count=$nbtst
112	jmc	1.7	fi
113
114	jmc	1.9	#- when it's done, retrieve output and send e-mail
115	jmc	1.15	minutes=0 ; freq=10
116			fsec=`expr $freq \* 60`
117			echo "Check every $freq mn for $count test(s) to finish" \| tee -a $LOG_FIL
118			echo "- start at :" `date` \| tee -a $LOG_FIL
119	jmc	1.1	while test $count != 0 ; do
120
121	jmc	1.15	sleep $fsec
122			minutes=$(( $minutes + $freq ))
123	jmc	1.1
124			for i in $tst_list ; do
125
126			eval comm=\$M_$i
127			if test $comm = 'submitted' ; then
128			JOB="tst_"$i
129			ready_to_send=`$QSTAT -a \| grep $USER \| grep $JOB \| wc -l`
130
131			if test "x_$ready_to_send" = x_0 ; then
132	jmc	1.10	run_dir=${TST_DIR}"/MITgcm_"$i"/verification"
133			#- results output:
134			tdir=`ls -1 -t $run_dir \| grep -v tr_out \| grep '^tr_aces' \| head -1`
135			if test "x$tdir" != x ; then
136	jmc	1.8	#- check this is the right output
137			chk=`echo $tdir \| grep -c $start`
138			if test $chk = '0' ; then
139			curday=`date +%Y%m%d`
140			chk=`echo $tdir \| grep -c $curday`
141			fi
142	jmc	1.10	if test $chk = '0' ; then
143	jmc	1.15	echo "tdir='$tdir'" \| tee -a $LOG_FIL
144			echo "Output do not match, no email sent for $i" \| tee -a $LOG_FIL
145	jmc	1.8	else
146			rm -f "/tmp/tr_aces-"$i".tar.gz"
147			( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )
148	jmc	1.7	if test "x$HAVE_MPACK" = xt ; then
149	jmc	1.3	$MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org
150	jmc	1.15	echo "Email sent for $i at: " `date` \| tee -a $LOG_FIL
151	jmc	1.7	else
152	jmc	1.15	echo " no email sent for $i (no mpack)" \| tee -a $LOG_FIL
153	jmc	1.7	fi
154	jmc	1.1	fi
155	jmc	1.10	else
156	jmc	1.15	echo " no output found for $i" \| tee -a $LOG_FIL
157	jmc	1.10	fi
158			#- restart output:
159			tdir=`ls -1 -t $run_dir \| grep -v tr_out \| grep '^rs_aces' \| head -1`
160			if test "x$tdir" != x ; then
161			#- check this is the right output
162			chk=`echo $tdir \| grep -c $start`
163			if test $chk = '0' ; then
164			curday=`date +%Y%m%d`
165			chk=`echo $tdir \| grep -c $curday`
166			fi
167			if test $chk = '0' ; then
168	jmc	1.15	echo "tdir='$tdir'" \| tee -a $LOG_FIL
169			echo "Restart output do not match, no email sent for $i" \| tee -a $LOG_FIL
170	jmc	1.10	else
171			rm -f "/tmp/rs_aces-"$i".tar.gz"
172			( cd $run_dir ; tar -czf "/tmp/rs_aces-"$i".tar.gz" ./$tdir )
173			if test "x$HAVE_MPACK" = xt ; then
174			$MPACK -s MITgcm-test -m 3555000 "/tmp/rs_aces-"$i".tar.gz" jmc@mitgcm.org
175	jmc	1.15	echo "Email sent for $i restart:" `date` \| tee -a $LOG_FIL
176	jmc	1.10	else
177	jmc	1.15	echo " no email sent for $i restart (no mpack)" \| tee -a $LOG_FIL
178	jmc	1.10	fi
179			fi
180			else
181	jmc	1.15	echo " no restart output for $i" \| tee -a $LOG_FIL
182	jmc	1.10	fi
183			#- record successful sending
184			eval M_$i=done
185			count=`expr $count - 1`
186			chmod 644 output/tst_$i.std*
187	jmc	1.1	fi
188			fi
189			done
190	jmc	1.15
191	jmc	1.1	# "long" queue is 24hrs = 24*60min = 1440min
192	jmc	1.4	if test $minutes -gt 2160 ; then
193			hrs=$(( $minutes / 60 ));
194	jmc	1.1	echo "Time expired after $minutes minutes ($hrs hours)"
195			echo ' ' $count '/' $nbtst 'tests not yet finished'
196			exit 1
197			fi
198
199			done
200
201	jmc	1.15	echo "Retrieving $nbtst tests finish :" `date` \| tee -a $LOG_FIL
202			exit 0
203	jmc	1.1