#!/bin/bash
#SBATCH -J tst_gfoMp2
#SBATCH -p sched_mit_hill
#SBATCH --mem-per-cpu 4000
#SBATCH -N 2
#SBATCH --tasks-per-node 4
#SBATCH -e /home/jm_c/test_engaging/output/tst_gfoMp2.stderr
#SBATCH -o /home/jm_c/test_engaging/output/tst_gfoMp2.stdout
#SBATCH --no-requeue --constraint=centos6

# $Header: /home/ubuntu/mnt/e9_copy/MITgcm_contrib/test_scripts/engaging/Attic/test_engag_gfo_mp2,v 1.6 2018/02/05 17:00:11 jmc dead $
# $Name:  $

if test -f /etc/profile.d/modules.sh ; then
    . /etc/profile.d/modules.sh
fi
# Note: added "ulimit -s unlimited" in file "~/.bashrc"
# to pass big test (the 2 fizhi-cs-* test & adjoint tests) with MPI

umask 0022
#- to get case insensitive "ls" (and order of tested experiments)
export LC_ALL="en_US.UTF-8"
echo " running on: "`hostname`

dNam='engaging'
HERE="$HOME/test_$dNam"
OUTP="$HERE/output"; SavD="$HERE/send"
SEND="ssh eofe4 $SavD/mpack"
TST_DIR="/pool001/jm_c/test_$dNam"

if test -d $TST_DIR ; then
  echo "start from TST_DIR='$TST_DIR' at: "`date`
else
 #if test ! -d /scratch/jm_c ; then mkdir /scratch/jm_c ; fi
 #if test ! -d $TST_DIR ; then mkdir $TST_DIR ; fi
 #if test ! -d $TST_DIR ; then
 #  echo "ERROR: Can't create directory \"$TST_DIR\""
 #  exit 1
 #fi
 #echo "start in new dir TST_DIR='$TST_DIR' at: "`date`
  echo "ERROR: missing directory \"$TST_DIR\""
  exit 1
fi

typ='' ; addExp='' ; skipExp=''
sfx='gfoMp2'
 module add slurm
 module add gcc
 module add mvapich2/gcc/64/2.0b
#module add openmpi/gcc
 export MPI_INC_DIR="$MPI_HOME/include"
#- for NetCDF:
 module add harvard/centos6/hdf5-1.8.11_gcc-4.4.7
 module add harvard/centos6/netcdf-4.3.0_gcc-4.4.7
 OPTFILE="../tools/build_options/linux_amd64_gfortran"
 options="$typ -MPI 4 -mth"
 ulimit -s unlimited
#- need this to get "staf":
export PATH="$PATH:$HOME/bin"
#- note: with mpich1, need to set these 2 vars in .tcshrc
 export OMP_NUM_THREADS=2
 export GOMP_STACKSIZE=400m

dAlt=`date +%d` ; dAlt=`expr $dAlt % 3`
#if [ $dAlt -eq 1 ] ; then options="$options -fast"
#else options="$options -devel" ; fi
options="$options -devel"

checkOut=1 ; #options="$options -do"
#options="$options -nc" ; checkOut=1
#options="$options -q"  ; checkOut=0

NSLOTS=$SLURM_NTASKS
THEDATE=`date`
echo '********************************************************************************'
echo 'Start job '$THEDATE
echo 'NSLOTS = '$NSLOTS
echo '======= NODELIST ==============================================================='
echo $SLURM_NODELIST
cat /etc/redhat-release
echo '======= env ===================================================================='
env | grep SLURM
echo '======= modules ================================================================'
module list 2>&1
echo '================================================================================'

#-
 MPI_MFile="${OUTP}/mf_${sfx}"
 mpiCMD="mpirun -hostfile TR_MFILE ./mitgcmuv"
#- make the testreport MPI_MFILE:
 listNODES=`echo $SLURM_NODELIST | sed -e 's/\[/ /' -e 's/\]//' -e 's/,/ /' -e 's/-/ /'`
# duplicate the 2 listed nodes into a 4 list file:
 /bin/rm -f $MPI_MFile ; touch $MPI_MFile ; pfx=''
for nd in $listNODES
do
  if test "x$pfx" = x ; then pfx=$nd ; else
    echo "${pfx}${nd}" >> $MPI_MFile
    echo "${pfx}${nd}" >> $MPI_MFile
  fi
done

cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q'
gcmDIR="MITgcm_$sfx"
cd $TST_DIR
#- check for disk space: relative space (99%) or absolute (10.G):
dsp=`df -P . | tail -1 | awk '{print $5}' | sed 's/%$//'`
if [ $dsp -gt 99 ] ; then
#dsp=`df -P . | tail -1 | awk '{print $4}'`
#if [ $dsp -le 100000000 ] ; then
  echo 'Not enough space on this disk => do not run testreport.'
  df .
  exit
fi
if [ $checkOut -eq 1 ] ; then
  if test -d $gcmDIR/CVS ; then
    echo "cleaning output from $gcmDIR/verification :"
#- remove previous output tar files and tar & remove previous output-dir
    /bin/rm -f $gcmDIR/verification/??_${dNam}-${sfx}_????????_?.tar.gz
    ( cd $gcmDIR/verification
      listD=`ls -1 -d ??_${dNam}-${sfx}_????????_? 2> /dev/null`
      for dd in $listD
      do
        if test -d $dd ; then
          tar -cf ${dd}".tar" $dd > /dev/null 2>&1 && gzip ${dd}".tar" && /bin/rm -rf $dd
          RETVAL=$?
          if test "x$RETVAL" != x0 ; then
             echo "ERROR in tar+gzip prev outp-dir: $dd"
             echo " on '"`hostname`"' (return val=$RETVAL) but continue"
          fi
        fi
      done )
    ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
    ( cd $gcmDIR/verification ; ./testreport $typ -clean )
    echo "cvs update of dir $gcmDIR :"
    ( cd $gcmDIR ; $cmdCVS update -P -d ) 2>&1
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
       echo "cvs update on '"`hostname`"' fail (return val=$RETVAL) => exit"
       exit
    fi
  else
    echo "no dir: $gcmDIR/CVS => try a fresh check-out"
    checkOut=2
  fi
fi
if [ $checkOut -eq 2 ] ; then
  if test -e $gcmDIR ; then
    echo -n "Removing working copy: $gcmDIR ..."
    rm -rf $gcmDIR
    echo "  done"
  fi
# cvs co MITgcm
  echo "Downloading using '$cmdCVS co -P -d $gcmDIR MITgcm' ..."
  $cmdCVS co -P -d $gcmDIR MITgcm > /dev/null
  RETVAL=$?
  if test "x$RETVAL" != x0 ; then
      echo "cvs co on '"`hostname`"' fail (return val=$RETVAL) => exit"
      exit
  else echo "  done" ; fi
  if test -d $gcmDIR/verification ; then
    for exp2add in $addExp ; do
      echo " add dir: $exp2add (from Contrib:verification_other)"
      ( cd $gcmDIR/verification ; $cmdCVS co -P -d $exp2add \
                     MITgcm_contrib/verification_other/$exp2add > /dev/null )
    done
    /usr/bin/find $gcmDIR -type d | xargs chmod g+rxs
    /usr/bin/find $gcmDIR -type f | xargs chmod g+r
  fi
fi

#- change dir to $gcmDIR/verification dir:
if test -e $gcmDIR/verification ; then
  if [ $checkOut -lt 2 ] ; then
    echo " dir $gcmDIR/verification exist" ; fi
  cd $gcmDIR/verification
else
  echo "no dir: $gcmDIR/verification => exit"
  exit
fi

echo ''
echo ./testreport $options -of $OPTFILE -command \'$mpiCMD\' -mf $MPI_MFile \
  -odir ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jmc@mitgcm.org
./testreport $options -of $OPTFILE -command "$mpiCMD" -mf $MPI_MFile \
  -odir ${dNam}-$sfx -send "$SEND" -sd $SavD -a jmc@mitgcm.org

echo ''
echo ../tools/do_tst_2+2 -mpi -exe \'$mpiCMD\' -mf $MPI_MFile \
  -o ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jmc@mitgcm.org
../tools/do_tst_2+2 -mpi -exe "$mpiCMD" -mf $MPI_MFile \
  -o ${dNam}-$sfx -send "$SEND" -sd $SavD -a jmc@mitgcm.org