#!/bin/bash
#
#PBS -q four-twelve
#PBS -N tst_mth
#PBS -l nodes=1:ppn=2
#PBS -e /home/jmc/test_aces/output/tst_mth.stderr
#PBS -o /home/jmc/test_aces/output/tst_mth.stdout

# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/example_scripts/ACESgrid/Attic/aces_test_ifc_mth,v 1.31 2017/05/17 17:58:55 jmc dead $
# $Name:  $

# ACES-grid PBS-batch script for Multi-Threaded test using ifort (v9)

if test -f /etc/profile.d/modules.sh ; then
    . /etc/profile.d/modules.sh
fi
# Note: added "ulimit -s unlimited" in file "~/.bashrc"
# to pass big test (the 2 fizhi-cs-* test & adjoint tests) with MPI

umask 0022
#- to get case insensitive "ls" (and order of tested experiments)
export LC_ALL="en_US.UTF-8"
echo " running on: "`hostname`

dNam='aces'
HERE="$HOME/test_$dNam"
OUTP="$HERE/output"; SavD="$HERE/send"
SEND="ssh geo $SavD/mpack"
TST_DIR=$HERE

if test -d $TST_DIR ; then
  echo "start from TST_DIR='$TST_DIR' at: "`date`
else
# if test ! -d /scratch/jm_c ; then mkdir /scratch/jm_c ; fi
  if test ! -d $TST_DIR ; then mkdir $TST_DIR ; fi
  if test ! -d $TST_DIR ; then
    echo "ERROR: Can't create directory \"$TST_DIR\""
    exit 1
  fi
  echo "start in new dir TST_DIR='$TST_DIR' at: "`date`
fi

sfx='mth'
addExp=''
 module add ifc/9.0.021 icc/9.0.021 intel/9.0
 module add netcdf/3.6.1/icc
 OPTFILE="../tools/build_options/linux_ia32_ifort"
 options='-j 2 -mth -devel'
 export OMP_NUM_THREADS=2
 export KMP_STACKSIZE=400m

checkOut=1 ; #options="$options -do"
#options="$options -nc" ; checkOut=0
#options="$options -q"  ; checkOut=0

#- keep a copy of MPI_MFILE:
cp -p $PBS_NODEFILE $OUTP"/mf_"$sfx

cvsArg="-d :pserver:cvsanon@mitgcm.org:/u/gcmpack"
gcmDIR="MITgcm_$sfx"
cd $TST_DIR
#- check for disk space: relative space (99%) or absolute (1.G):
#dsp=`df -P . | tail -1 | awk '{print $5}' | sed 's/%$//'`
#if [ $dsp -gt 99 ] ; then
dsp=`df -P . | tail -1 | awk '{print $4}'`
if [ $dsp -le 1000000 ] ; then
  echo 'Not enough space on this disk => do not run testreport.'
  df .
  exit
fi
if [ $checkOut -eq 1 ] ; then
  if test -d $gcmDIR/CVS ; then
#- remove previous output tar files and tar & remove previous output-dir
    /bin/rm -f $gcmDIR/verification/??_${dNam}-${sfx}_????????_?.tar.gz
    ( cd $gcmDIR/verification
      listD=`ls -1 -d ??_${dNam}-${sfx}_????????_? 2> /dev/null`
      for dd in $listD
      do
        if test -d $dd ; then
          tar -cf ${dd}".tar" $dd > /dev/null 2>&1 && gzip ${dd}".tar" && /bin/rm -rf $dd
          RETVAL=$?
          if test "x$RETVAL" != x0 ; then
             echo "ERROR in tar+gzip prev outp-dir: $dd"
             echo " on '"`hostname`"' (return val=$RETVAL) but continue"
          fi
        fi
      done )
    ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
#- clean-up previous TAF output temp dir (generated on old system like here)
#   ( cd $gcmDIR/verification ; /bin/rm -rf  */build/PaxHeaders.[0-9]* )
    echo "cvs update of dir $gcmDIR :"
    ( cd $gcmDIR ; cvs -q $cvsArg update -P -d ) 2>&1
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
       echo "cvs update on '"`hostname`"' fail (return val=$RETVAL) => exit"
       exit
    fi
  else
    echo "no dir: $gcmDIR/CVS => try a fresh check-out"
    checkOut=2
  fi
fi
if [ $checkOut -eq 2 ] ; then
  if test -e $gcmDIR ; then
     rm -rf $gcmDIR
  fi
# cvs co MITgcm
  cvs $cvsArg co -P -d $gcmDIR MITgcm > /dev/null 2>&1
  for exp2add in $addExp ; do
   echo " add dir: $exp2add (from Contrib:verification_other)"
   ( cd $gcmDIR/verification ; cvs $cvsArg co -P -d $exp2add \
                  MITgcm_contrib/verification_other/$exp2add > /dev/null 2>&1 )
  done
  /usr/bin/find $gcmDIR -type d | xargs chmod g+rxs
  /usr/bin/find $gcmDIR -type f | xargs chmod g+r
fi

#- change dir to $gcmDIR/verification dir:
if test -e $gcmDIR/verification ; then
  if [ $checkOut -lt 2 ] ; then
    echo " dir $gcmDIR/verification exist" ; fi
  cd $gcmDIR/verification
else
  echo "no dir: $gcmDIR/verification => exit"
  exit
fi

echo ./testreport $options -of $OPTFILE -odir ${dNam}-$sfx \
  -send \"$SEND\" -sd $SavD -a jmc@mitgcm.org
./testreport $options -of $OPTFILE -odir ${dNam}-$sfx \
  -send "$SEND" -sd $SavD -a jmc@mitgcm.org

echo ''
#echo ../tools/do_tst_2+2 -o ${dNam}-$sfx -a NONE
#../tools/do_tst_2+2 -o ${dNam}-$sfx -a NONE
echo ../tools/do_tst_2+2 -o ${dNam}-$sfx -send \"$SEND\" -sd $SavD -a jmc@mitgcm.org
../tools/do_tst_2+2 -o ${dNam}-$sfx -send "$SEND" -sd $SavD -a jmc@mitgcm.org