Annotation of /MITgcm/tools/example_scripts/ACESgrid/aces_test_all

#! /usr/bin/env bash

# $Header: /u/gcmpack/MITgcm/tools/example_scripts/ACESgrid/aces_test_all,v 1.19 2010/05/11 18:33:32 jmc Exp $
# $Name:  $

# action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both
case $1 in
 '-subOnly') action=2 ; shift ;;
 '-getOnly') action=1 ; shift ;;
 '-sub+get') action=3 ; shift ;;
          *) action=3 ;;
esac
#echo "action= $action , Arg= $# "

today=`date +%Y%m%d`

if test $# = 0
then
  tst_list='g77 gnu ifc pgi adm mp2 mth tuv'
else
  tst_list=$*
fi

# QSUB="/usr/local/bin/qsub"
# QSTAT="/usr/local/bin/qstat"
QSUB=qsub
QSTAT=qstat
TST_DIR="/home/jmc/test_ACES"
MPACK="MITgcm_tools/mpack-1.6/mpack"
SUB_DIR="MITgcm_tools/example_scripts/ACESgrid"

TMP_FIL="$TST_DIR/output/TTT.$$"
LOG_FIL="$TST_DIR/output/tst_all."`date +%m%d`".log"

#-- clean up old log files and start a new one:
cd $TST_DIR/output

rm -f tst_all.*.log_bak
if test -f $LOG_FIL ; then mv -f $LOG_FIL ${LOG_FIL}_bak ; fi
echo -n '-- Starting: ' | tee -a $LOG_FIL
date | tee -a $LOG_FIL

n=$(( `ls tst_all.*.log | wc -l` - 10 ))
if test $n -gt 0 ; then
  echo ' remove old log files:' | tee -a $LOG_FIL
    ls -lt tst_all.*.log | tail -"$n" | tee -a $LOG_FIL
    ls -t  tst_all.*.log | tail -"$n" | xargs rm -f
fi

. /etc/profile.d/modules.sh
module list >> $LOG_FIL 2>&1

#-- now really do something:
cd $TST_DIR

nbtst=0
for i in $tst_list
do
  case $i in
    'mth') sfx='ifc_'${i}  ;;
    'tuv') sfx='op64_'${i} ;;
    'mp2') sfx=${i}'_mth'  ;;
    'g77') sfx=${i}'_adm'  ;;
        *) sfx=${i}'_mpi'  ;;
  esac
  if test -f $SUB_DIR/aces_test_$sfx ; then
    JOB="tst_"$i
    job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l`
    if [ $action -ge 2 ] ; then
    #-- to submit job
      if test "x_$job_exist" = x_0 ; then
        echo -n " $JOB : " | tee -a $LOG_FIL
        $QSUB $SUB_DIR/aces_test_$sfx | tee -a $LOG_FIL
        eval M_$i='submitted'
        nbtst=`expr $nbtst + 1`
      else
        echo $JOB | tee -a $LOG_FIL
        $QSTAT -a | grep $USER | grep $JOB | tee -a $LOG_FIL
        echo 'job already exist => skip this test' | tee -a $LOG_FIL
        eval M_$i='skipped'
      fi
    else
    #-- to get outp back:
      if test "x_$job_exist" = x_0 ; then
        echo -n "did not find any job: $JOB" | tee -a $LOG_FIL
        eval M_$i='skipped'
      else
        echo -n "found a job: $JOB" | tee -a $LOG_FIL
        $QSTAT -a | grep $USER | grep $JOB | tee -a $LOG_FIL
        eval M_$i='submitted'
        nbtst=`expr $nbtst + 1`
      fi
    fi
  else
    echo 'no file:' aces_test_$sfx 'to submit' | tee -a $LOG_FIL
    eval M_$i='skipped'
  fi
done

if test $action = 2 ; then
  count=0
else
#date_str=`date +%Y%m%d`"_0"

  MPACKDIR=`dirname $MPACK`
#- build the mpack utility (from build_mpack in testreport):
  ( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1
  RETVAL=$?
  if test "x$RETVAL" != x0 ; then
    echo
    echo "Error building the mpack tools at: $MPACK_DIR" | tee -a $LOG_FIL
    if test -x $MPACK ; then
       HAVE_MPACK=t
       echo "  use (old ?) executable:" | tee -a $LOG_FIL
       ls -l $MPACK | tee -a $LOG_FIL
    else
       HAVE_MPACK=f
    fi
  else
    if test -x $MPACK ; then
       rm -f TTT.build_mpack.$$
       HAVE_MPACK=t
       echo " building mpack: OK" | tee -a $LOG_FIL
    else
       echo " $MPACK not executable" | tee -a $LOG_FIL
       HAVE_MPACK=f
    fi
  fi
  echo

  count=$nbtst
fi

#- when it's done, retrieve output and send e-mail
minutes=0 ; freq=10
fsec=`expr $freq \* 60`
echo "Check every $freq mn for $count test(s) to finish" | tee -a $LOG_FIL
echo "- start at :" `date` | tee -a $LOG_FIL
while test $count != 0 ; do

    sleep $fsec
    minutes=$(( $minutes + $freq ))

    for i in $tst_list ; do

      eval comm=\$M_$i
      if test $comm = 'submitted' ; then
        JOB="tst_"$i
        $QSTAT -a > $TMP_FIL
        RETVAL=$?
        ready_to_send=`grep $USER $TMP_FIL | grep $JOB | wc -l`
        rm -f $TMP_FIL
        if test "x$RETVAL" != x0 ; then
           echo " $QSTAT returned with error code: $RETVAL" | tee -a $LOG_FIL
           continue
        fi

        if test "x_$ready_to_send" = x_0 ; then
          run_dir=${TST_DIR}"/MITgcm_"$i"/verification"
#-      results output:
          tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^tr_aces' | head -1`
          if test "x$tdir" != x ; then
            #- check this is the right output
            chk=`echo $tdir | grep -c $today`
            if test $chk = '0' ; then
              curday=`date +%Y%m%d`
              chk=`echo $tdir | grep -c $curday`
            fi
            if test $chk = '0' ; then
              echo "tdir='$tdir'" | tee -a $LOG_FIL
              echo "Output do not match, no email sent for $i" | tee -a $LOG_FIL
            else
              rm -f "/tmp/tr_aces-"$i".tar.gz"
              ( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )
              if test "x$HAVE_MPACK" = xt ; then
                $MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org
                echo "Email sent for $i at:     " `date` | tee -a $LOG_FIL
              else
                echo " no email sent for $i (no mpack)" | tee -a $LOG_FIL
              fi
            fi
          else
              echo " no output found for $i" | tee -a $LOG_FIL
          fi
#-      restart output:
          tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^rs_aces' | head -1`
          if test "x$tdir" != x ; then
            #- check this is the right output
            chk=`echo $tdir | grep -c $today`
            if test $chk = '0' ; then
              curday=`date +%Y%m%d`
              chk=`echo $tdir | grep -c $curday`
            fi
            if test $chk = '0' ; then
              echo "tdir='$tdir'" | tee -a $LOG_FIL
              echo "Restart output do not match, no email sent for $i" | tee -a $LOG_FIL
            else
              rm -f "/tmp/rs_aces-"$i".tar.gz"
              ( cd $run_dir ; tar -czf "/tmp/rs_aces-"$i".tar.gz" ./$tdir )
              if test "x$HAVE_MPACK" = xt ; then
                $MPACK -s MITgcm-test -m 3555000 "/tmp/rs_aces-"$i".tar.gz" jmc@mitgcm.org
                echo "Email sent for $i restart:" `date` | tee -a $LOG_FIL
              else
                echo " no email sent for $i restart (no mpack)" | tee -a $LOG_FIL
              fi
            fi
          else
              echo " no restart output for $i" | tee -a $LOG_FIL
          fi
#-      record successful sending
          eval M_$i=done
          count=`expr $count - 1`
          chmod 644 output/tst_$i.std*
        fi
      fi
    done

    #  "long" queue is 24hrs = 24*60min = 1440min
    if test $minutes -gt 2160 ; then
        hrs=$(( $minutes / 60 ));
        echo "Time expired after $minutes minutes ($hrs hours)" | tee -a $LOG_FIL
        echo ' ' $count '/' $nbtst 'tests not yet finished' | tee -a $LOG_FIL
        exit 1
    fi

done

echo "Retrieving $nbtst tests finish :" `date` | tee -a $LOG_FIL
exit 0

1	jmc	1.1	#! /usr/bin/env bash
2
3	jmc	1.20	# $Header: /u/gcmpack/MITgcm/tools/example_scripts/ACESgrid/aces_test_all,v 1.19 2010/05/11 18:33:32 jmc Exp $
4	jmc	1.1	# $Name: $
5
6	jmc	1.9	# action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both
7			case $1 in
8			'-subOnly') action=2 ; shift ;;
9			'-getOnly') action=1 ; shift ;;
10			'-sub+get') action=3 ; shift ;;
11			*) action=3 ;;
12			esac
13			#echo "action= $action , Arg= $# "
14
15	jmc	1.16	today=`date +%Y%m%d`
16	jmc	1.9
17	jmc	1.1	if test $# = 0
18			then
19	jmc	1.20	tst_list='g77 gnu ifc pgi adm mp2 mth tuv'
20	jmc	1.1	else
21			tst_list=$*
22			fi
23
24			# QSUB="/usr/local/bin/qsub"
25			# QSTAT="/usr/local/bin/qstat"
26			QSUB=qsub
27			QSTAT=qstat
28	jmc	1.14	TST_DIR="/home/jmc/test_ACES"
29	jmc	1.4	MPACK="MITgcm_tools/mpack-1.6/mpack"
30	jmc	1.14	SUB_DIR="MITgcm_tools/example_scripts/ACESgrid"
31
32	jmc	1.16	TMP_FIL="$TST_DIR/output/TTT.$$"
33			LOG_FIL="$TST_DIR/output/tst_all."`date +%m%d`".log"
34	jmc	1.17
35	jmc	1.18	#-- clean up old log files and start a new one:
36			cd $TST_DIR/output
37
38			rm -f tst_all.*.log_bak
39			if test -f $LOG_FIL ; then mv -f $LOG_FIL ${LOG_FIL}_bak ; fi
40	jmc	1.17	echo -n '-- Starting: ' \| tee -a $LOG_FIL
41			date \| tee -a $LOG_FIL
42	jmc	1.16
43			n=$(( `ls tst_all.*.log \| wc -l` - 10 ))
44			if test $n -gt 0 ; then
45			echo ' remove old log files:' \| tee -a $LOG_FIL
46			ls -lt tst_all.*.log \| tail -"$n" \| tee -a $LOG_FIL
47			ls -t tst_all.*.log \| tail -"$n" \| xargs rm -f
48			fi
49
50			. /etc/profile.d/modules.sh
51	jmc	1.17	module list >> $LOG_FIL 2>&1
52	jmc	1.16
53			#-- now really do something:
54	jmc	1.14	cd $TST_DIR
55	jmc	1.1
56			nbtst=0
57			for i in $tst_list
58			do
59	jmc	1.20	case $i in
60			'mth') sfx='ifc_'${i} ;;
61			'tuv') sfx='op64_'${i} ;;
62			'mp2') sfx=${i}'_mth' ;;
63			'g77') sfx=${i}'_adm' ;;
64			*) sfx=${i}'_mpi' ;;
65			esac
66	jmc	1.14	if test -f $SUB_DIR/aces_test_$sfx ; then
67	jmc	1.4	JOB="tst_"$i
68			job_exist=`$QSTAT -a \| grep $USER \| grep $JOB \| wc -l`
69	jmc	1.15	if [ $action -ge 2 ] ; then
70	jmc	1.9	#-- to submit job
71			if test "x_$job_exist" = x_0 ; then
72	jmc	1.19	echo -n " $JOB : " \| tee -a $LOG_FIL
73	jmc	1.17	$QSUB $SUB_DIR/aces_test_$sfx \| tee -a $LOG_FIL
74	jmc	1.9	eval M_$i='submitted'
75			nbtst=`expr $nbtst + 1`
76	jmc	1.15	else
77	jmc	1.17	echo $JOB \| tee -a $LOG_FIL
78			$QSTAT -a \| grep $USER \| grep $JOB \| tee -a $LOG_FIL
79			echo 'job already exist => skip this test' \| tee -a $LOG_FIL
80	jmc	1.9	eval M_$i='skipped'
81			fi
82			else
83			#-- to get outp back:
84			if test "x_$job_exist" = x_0 ; then
85	jmc	1.19	echo -n "did not find any job: $JOB" \| tee -a $LOG_FIL
86	jmc	1.9	eval M_$i='skipped'
87	jmc	1.15	else
88	jmc	1.19	echo -n "found a job: $JOB" \| tee -a $LOG_FIL
89	jmc	1.17	$QSTAT -a \| grep $USER \| grep $JOB \| tee -a $LOG_FIL
90	jmc	1.9	eval M_$i='submitted'
91			nbtst=`expr $nbtst + 1`
92			fi
93	jmc	1.4	fi
94	jmc	1.15	else
95	jmc	1.17	echo 'no file:' aces_test_$sfx 'to submit' \| tee -a $LOG_FIL
96	jmc	1.4	eval M_$i='skipped'
97	jmc	1.1	fi
98			done
99
100	jmc	1.15	if test $action = 2 ; then
101	jmc	1.9	count=0
102			else
103	jmc	1.4	#date_str=`date +%Y%m%d`"_0"
104	jmc	1.1
105	jmc	1.9	MPACKDIR=`dirname $MPACK`
106	jmc	1.7	#- build the mpack utility (from build_mpack in testreport):
107	jmc	1.9	( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1
108			RETVAL=$?
109			if test "x$RETVAL" != x0 ; then
110	jmc	1.7	echo
111	jmc	1.17	echo "Error building the mpack tools at: $MPACK_DIR" \| tee -a $LOG_FIL
112	jmc	1.7	if test -x $MPACK ; then
113			HAVE_MPACK=t
114	jmc	1.20	echo " use (old ?) executable:" \| tee -a $LOG_FIL
115	jmc	1.17	ls -l $MPACK \| tee -a $LOG_FIL
116	jmc	1.7	else
117			HAVE_MPACK=f
118			fi
119	jmc	1.9	else
120	jmc	1.7	if test -x $MPACK ; then
121			rm -f TTT.build_mpack.$$
122			HAVE_MPACK=t
123	jmc	1.17	echo " building mpack: OK" \| tee -a $LOG_FIL
124	jmc	1.7	else
125	jmc	1.17	echo " $MPACK not executable" \| tee -a $LOG_FIL
126	jmc	1.7	HAVE_MPACK=f
127			fi
128	jmc	1.9	fi
129			echo
130
131			count=$nbtst
132	jmc	1.7	fi
133
134	jmc	1.9	#- when it's done, retrieve output and send e-mail
135	jmc	1.15	minutes=0 ; freq=10
136			fsec=`expr $freq \* 60`
137			echo "Check every $freq mn for $count test(s) to finish" \| tee -a $LOG_FIL
138			echo "- start at :" `date` \| tee -a $LOG_FIL
139	jmc	1.1	while test $count != 0 ; do
140
141	jmc	1.15	sleep $fsec
142			minutes=$(( $minutes + $freq ))
143	jmc	1.1
144			for i in $tst_list ; do
145
146			eval comm=\$M_$i
147			if test $comm = 'submitted' ; then
148			JOB="tst_"$i
149	jmc	1.16	$QSTAT -a > $TMP_FIL
150			RETVAL=$?
151			ready_to_send=`grep $USER $TMP_FIL \| grep $JOB \| wc -l`
152			rm -f $TMP_FIL
153			if test "x$RETVAL" != x0 ; then
154			echo " $QSTAT returned with error code: $RETVAL" \| tee -a $LOG_FIL
155			continue
156			fi
157	jmc	1.1
158			if test "x_$ready_to_send" = x_0 ; then
159	jmc	1.10	run_dir=${TST_DIR}"/MITgcm_"$i"/verification"
160			#- results output:
161			tdir=`ls -1 -t $run_dir \| grep -v tr_out \| grep '^tr_aces' \| head -1`
162			if test "x$tdir" != x ; then
163	jmc	1.8	#- check this is the right output
164	jmc	1.16	chk=`echo $tdir \| grep -c $today`
165	jmc	1.8	if test $chk = '0' ; then
166			curday=`date +%Y%m%d`
167			chk=`echo $tdir \| grep -c $curday`
168			fi
169	jmc	1.10	if test $chk = '0' ; then
170	jmc	1.15	echo "tdir='$tdir'" \| tee -a $LOG_FIL
171			echo "Output do not match, no email sent for $i" \| tee -a $LOG_FIL
172	jmc	1.8	else
173			rm -f "/tmp/tr_aces-"$i".tar.gz"
174			( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )
175	jmc	1.7	if test "x$HAVE_MPACK" = xt ; then
176	jmc	1.3	$MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org
177	jmc	1.15	echo "Email sent for $i at: " `date` \| tee -a $LOG_FIL
178	jmc	1.7	else
179	jmc	1.15	echo " no email sent for $i (no mpack)" \| tee -a $LOG_FIL
180	jmc	1.7	fi
181	jmc	1.1	fi
182	jmc	1.10	else
183	jmc	1.15	echo " no output found for $i" \| tee -a $LOG_FIL
184	jmc	1.10	fi
185			#- restart output:
186			tdir=`ls -1 -t $run_dir \| grep -v tr_out \| grep '^rs_aces' \| head -1`
187			if test "x$tdir" != x ; then
188			#- check this is the right output
189	jmc	1.16	chk=`echo $tdir \| grep -c $today`
190	jmc	1.10	if test $chk = '0' ; then
191			curday=`date +%Y%m%d`
192			chk=`echo $tdir \| grep -c $curday`
193			fi
194			if test $chk = '0' ; then
195	jmc	1.15	echo "tdir='$tdir'" \| tee -a $LOG_FIL
196			echo "Restart output do not match, no email sent for $i" \| tee -a $LOG_FIL
197	jmc	1.10	else
198			rm -f "/tmp/rs_aces-"$i".tar.gz"
199			( cd $run_dir ; tar -czf "/tmp/rs_aces-"$i".tar.gz" ./$tdir )
200			if test "x$HAVE_MPACK" = xt ; then
201			$MPACK -s MITgcm-test -m 3555000 "/tmp/rs_aces-"$i".tar.gz" jmc@mitgcm.org
202	jmc	1.15	echo "Email sent for $i restart:" `date` \| tee -a $LOG_FIL
203	jmc	1.10	else
204	jmc	1.15	echo " no email sent for $i restart (no mpack)" \| tee -a $LOG_FIL
205	jmc	1.10	fi
206			fi
207			else
208	jmc	1.15	echo " no restart output for $i" \| tee -a $LOG_FIL
209	jmc	1.10	fi
210			#- record successful sending
211			eval M_$i=done
212			count=`expr $count - 1`
213			chmod 644 output/tst_$i.std*
214	jmc	1.1	fi
215			fi
216			done
217	jmc	1.15
218	jmc	1.1	# "long" queue is 24hrs = 24*60min = 1440min
219	jmc	1.4	if test $minutes -gt 2160 ; then
220			hrs=$(( $minutes / 60 ));
221	jmc	1.16	echo "Time expired after $minutes minutes ($hrs hours)" \| tee -a $LOG_FIL
222			echo ' ' $count '/' $nbtst 'tests not yet finished' \| tee -a $LOG_FIL
223	jmc	1.1	exit 1
224			fi
225
226			done
227
228	jmc	1.15	echo "Retrieving $nbtst tests finish :" `date` \| tee -a $LOG_FIL
229			exit 0
230	jmc	1.1