#! /usr/bin/env bash

# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/example_scripts/ACESgrid/Attic/aces_test_all,v 1.9 2008/12/30 16:01:11 jmc Exp $
# $Name: checkpoint61h $

# action =2 : submit test jobs ; =1 : get jobs output ; =3 : do both
case $1 in
 '-subOnly') action=2 ; shift ;;
 '-getOnly') action=1 ; shift ;;
 '-sub+get') action=3 ; shift ;;
          *) action=3 ;;
esac
#echo "action= $action , Arg= $# "

echo -n '-- Starting: '
date
start=`date +%Y%m%d`

if test $# = 0
then
  tst_list='gnu ifc pgi mth tuv'
else
  tst_list=$*
fi

. /etc/profile.d/modules.sh
module list > ~/tmp/.module_list 2>&1

cd /home/jmc/test_ACES

# QSUB="/usr/local/bin/qsub"
# QSTAT="/usr/local/bin/qstat"
QSUB=qsub
QSTAT=qstat
MPACK="MITgcm_tools/mpack-1.6/mpack"
TST_DIR="/home/jmc/test_ACES/gcm_tests"

nbtst=0
for i in $tst_list
do
  sfx=`echo ${i}"_mpi" | sed 's/mth_mpi/ifc_mth/' | sed 's/tuv_mpi/op64_tuv/'`
  if test -f MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx ; then
    JOB="tst_"$i
    job_exist=`$QSTAT -a | grep $USER | grep $JOB | wc -l`
    if [ $action -ge 2 ] ; then 
    #-- to submit job
      if test "x_$job_exist" = x_0 ; then
        $QSUB MITgcm_tools/example_scripts/ACESgrid/aces_test_$sfx
        eval M_$i='submitted'
        nbtst=`expr $nbtst + 1`
      else 
        echo $JOB
        $QSTAT -a | grep $USER | grep $JOB
        echo 'job already exist => skip this test'
        eval M_$i='skipped'
      fi
    else
    #-- to get outp back:
      if test "x_$job_exist" = x_0 ; then
        echo -n 'did not find any job: '
        echo $JOB
        eval M_$i='skipped'
      else 
        echo -n 'found a job: '
        echo $JOB
        $QSTAT -a | grep $USER | grep $JOB
        eval M_$i='submitted'
        nbtst=`expr $nbtst + 1`
      fi
    fi
  else 
    echo 'no file:' aces_test_$sfx 'to submit'
    eval M_$i='skipped'
  fi
done

if test $action = 2 ; then 
  count=0
else
#date_str=`date +%Y%m%d`"_0"

  MPACKDIR=`dirname $MPACK`
#- build the mpack utility (from build_mpack in testreport):
  ( cd $MPACKDIR && ./configure && make ) > TTT.build_mpack.$$ 2>&1
  RETVAL=$?
  if test "x$RETVAL" != x0 ; then
    echo
    echo "Error building the mpack tools at: $MPACK_DIR"
    if test -x $MPACK ; then
       HAVE_MPACK=t
       echo "  use (old ?) executable:" ; ls -l $MPACK
    else
       HAVE_MPACK=f
    fi
  else
    if test -x $MPACK ; then
       rm -f TTT.build_mpack.$$
       HAVE_MPACK=t
       echo " building mpack: OK"
    else
       echo " $MPACK not executable"
       HAVE_MPACK=f
    fi
  fi
  echo

  count=$nbtst
fi

#- when it's done, retrieve output and send e-mail
minutes=0
while test $count != 0 ; do

    sleep 600
    minutes=$(( $minutes + 10 ))

    for i in $tst_list ; do

      eval comm=\$M_$i
      if test $comm = 'submitted' ; then
        JOB="tst_"$i
        ready_to_send=`$QSTAT -a | grep $USER | grep $JOB | wc -l`

        if test "x_$ready_to_send" = x_0 ; then
            run_dir=${TST_DIR}"/MITgcm_"$i"/verification"
            tdir=`ls -1 -t $run_dir | grep -v tr_out | grep '^tr_aces' | head -1`
            #- check this is the right output
            chk=`echo $tdir | grep -c $start`
            if test $chk = '0' ; then
              curday=`date +%Y%m%d`
              chk=`echo $tdir | grep -c $curday`
            fi
            if test "x$tdir" = x ; then
              echo "Output not found so no email sent for $i"
            elif test $chk = '0' ; then
              echo "tdir='$tdir'"
              echo "Output do not match so no email sent for $i"
            else
              rm -f "/tmp/tr_aces-"$i".tar.gz"
              ( cd $run_dir ; tar -czf "/tmp/tr_aces-"$i".tar.gz" ./$tdir )
              if test "x$HAVE_MPACK" = xt ; then
                $MPACK -s MITgcm-test -m 3555000 "/tmp/tr_aces-"$i".tar.gz" jmc@mitgcm.org
                echo "Email sent for $i"
              else
                echo "no email sent for $i (no mpack)"
              fi
            fi
            eval M_$i=done
            count=`expr $count - 1`
            chmod 644 output/tst_$i.std*
        fi
      fi
    done
    
    #  "long" queue is 24hrs = 24*60min = 1440min
    if test $minutes -gt 2160 ; then
        hrs=$(( $minutes / 60 ));
        echo "Time expired after $minutes minutes ($hrs hours)"
        echo ' ' $count '/' $nbtst 'tests not yet finished'
        exit 1
    fi

done

echo "All $nbtst tests completed successfully."
echo -n '-- Finished: '
date