#! /usr/bin/env bash

# $Header: /home/ubuntu/mnt/e9_copy/MITgcm_contrib/test_scripts/other/test_local,v 1.13 2023/05/31 13:51:39 jmc Exp $
# $Name:  $

#-------------------------------------------------------------------------------
#-- process argument list:
if test $# = 0 ; then
  echo 'need 1 argument'
  exit
else
  if test $1 = ifort ; then
#   tst_list='iadm imp2 iur4'
    tst_list='iad4 imp2 iur4'
    dd1=`date +%d -d "1 day"`
#   if test $dd1 != '01' ; then echo 'not last day of month'; exit ; fi
  elif test $1 = gfort ; then
    tst_list='gadm gads gad4 gmp4 gmp2 gfo gfo4'
#   tst_list="$tst_list g77 g7a"
  elif test $1 = tap ; then
    tst_list='tapAD tapTL'
  elif test $1 = gfor8 ; then
#   tst_list='gadm gads gmp2 gmpi gfo g77'
    tst_list='gadm gads gmp2 gfo'
  elif test $1 = gfor4 ; then
    tst_list='gad4 gmp4 gfo4 g77'
  else
    tst_list=$*
  fi
fi

#-------------------------------------------------------------------------------
#-- general setting + MITgcm ref clone update:
#
#- to get case insensitive "ls" (and order of tested experiments)
#export LC_ALL="en_US.UTF-8"
#- Turn off stack limit for FIZHI & AD-tests
ulimit -s unlimited
#- method to acces CVS:
  cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q'
#- and which GitHub repository to use:
  git_repo='MITgcm';  git_code='MITgcm'
 #git_repo='altMITgcm'; #git_code='MITgcm66h'
#- other settings:
tmpFil="/tmp/"`basename $0`".$$"
sendCmd='-a jm_c@mitgcm.org'
sendCmd='-send scp -a jm_c@mitgcm.org:testing/MITgcm-test'

gcmDIR="MITgcm"
logFile='check_update.log'
echo "-- run: \""`basename $0` $*"\" on:" `date` >> $logFile
if test -e $gcmDIR/.git/config -a -f git_Hash ; then
  mv git_Hash prevHash
  ( cd $gcmDIR ; git pull > /dev/null )
  ( cd $gcmDIR ; git rev-parse HEAD > ../git_Hash )
else
  echo " No current clone or missing file "git_Hash" ==> get a fresh clone" >> $logFile
  rm -f prevHash
  if test -d $gcmDIR ; then /bin/rm -rf $gcmDIR ; fi
  git clone https://github.com/$git_repo/${git_code}.git $gcmDIR
  ( cd $gcmDIR ; git rev-parse HEAD > ../git_Hash )
fi

#-------------------------------------------------------------------------------
#-- check particular set of tests:
if test $1 = tap ; then
#- Only run this pair of test if MITgcm code got updated:
  if test -f prevHash ; then
   #ls -l prevHash  >> $logFile
    diff -q git_Hash prevHash > /dev/null 2>&1
    retv=$?
    if test $retv = 0 ; then
      echo " No update in repos since previous test ==> skip '$1' set of tests"  >> $logFile
      exit 0
    else
      echo " Will run '$1' set of tests since repos has been updated" >> $logFile
      echo -n ' prevHash: ' >> $logFile
      cat prevHash >> $logFile
      echo -n ' git_Hash: ' >> $logFile
      cat git_Hash >> $logFile
    fi
  else
    echo " No file 'prevHash' --> will run '$1' set of tests" >> $logFile
  fi
fi

echo "run: \""`basename $0` $*"\" on:" `date`
echo " tst_list='$tst_list'"

#- to get ~/bin in the patch (for staf) when run on cron:
if [ -d ~/bin ]; then
  echo 'add ~/bin to $PATH'
  export PATH=$PATH:~/bin
#- and for Tapenade:
  echo $tst_list | grep '\<tap' > /dev/null 2>&1 ; retv=$?
  if test $retv = 0 ; then
    source ~jmc/bin/set_tapenade.sh
  fi
fi

#- for now, cannot mix ifort/gfortran tests:
gfort=1
for tt in $tst_list
do
  echo $tt | grep '^tap' > /dev/null 2>&1 ; retv=$?
  if [ $retv -ne 0 ] ; then 
    echo $tt | grep '^g' > /dev/null 2>&1 ; retv=$?
  fi
  if   [ $retv -eq 0 -a $gfort -ge 1 ] ; then gfort=2
  elif [ $retv -ne 0 -a $gfort -le 1 ] ; then gfort=0
  else echo 'cannot mix ifort/gfortran' ; exit
  fi
done
if [ $gfort -eq 0 ] ; then
  #echo 'source ~jmc/bin/intel_v14.sh'
  #source ~jmc/bin/intel_v14.sh
   echo 'source ~jmc/bin/intel_v15.sh'
   source ~jmc/bin/intel_v15.sh
fi
if [ $gfort -eq 2 ] ; then
   echo 'source ~jmc/bin/openmpi.sh'
   source ~jmc/bin/openmpi.sh
fi

#-------------------------------------------------------------------------------
#-- Update & Run individual test:
for tt in $tst_list
do

#- Clean and Update testing clone: ---------------------------------------------
  gcmDIR="MITgcm_$tt"
  echo "=========================================================================="
# set -x
  rm -f tr_clean_$tt.log
  echo $tt | grep '^.ad' > /dev/null 2>&1 ; fwd=$?
  echo $tt | grep '^tap' > /dev/null 2>&1 ; tap=$?
  if test $tt = g7a -o $tap = 0 ; then fwd=0 ; clOpt='-adm' ; fi
  echo " testing tt= $tt , fwd= $fwd"
 if test -e $gcmDIR/.git/config -a -d $gcmDIR/verification ; then
 #- cleaning previous testreport run and updating the code:
  if [ $fwd -eq 0 ] ; then
   if test $tt = 'tapAD' ; then clOpt='-tap -adm' ; fi
   if test $tt = 'tapTL' ; then clOpt='-tap -tlm' ; fi
   set -x
  #- cleanup previous test:
   ( cd $gcmDIR/verification ; ./testreport $clOpt -clean > ../../tr_clean_$tt.log 2>&1 )
   set +x
  else
   set -x
  #- cleanup previous restart:
   ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean > ../../tr_clean_$tt.log 2>&1 )
  #- cleanup previous test:
   ( cd $gcmDIR/verification ; ./testreport -clean >> ../../tr_clean_$tt.log 2>&1 )
   set +x
  fi
  if [ $gfort -eq 0 ] ; then
    echo ' remove all Makefile_syntax' >> tr_clean_$tt.log
    ( cd $gcmDIR/verification ; rm -f */build/Makefile_syntax )
  fi
  echo "" >> tr_clean_$tt.log

  echo "==========================================================================" \
       >> tr_clean_$tt.log
  echo " Update MITgcm code in dir: $gcmDIR using 'git pull':" | tee -a tr_clean_$tt.log
  set -x
  #- update the code:
  ( cd $gcmDIR ; git pull )				>> tr_clean_$tt.log 2>&1 ; retv=$?
  set +x
  if test $retv != 0 ; then
    echo "'git pull' in $gcmDIR fail (return val=$retv) => skip" | tee -a tr_clean_$tt.log
    continue
  fi
  echo "  and checkout master:"				| tee -a tr_clean_$tt.log
  set -x
  ( cd $gcmDIR ; git checkout master -- . )		>> tr_clean_$tt.log 2>&1
  set +x
 else
  echo "Missing '$gcmDIR/.git/config' or dir '$gcmDIR/verification'" >> tr_clean_$tt.log
  if test -e $gcmDIR ; then
    echo -n " removing working copy: $gcmDIR ..."	>> tr_clean_$tt.log
    rm -rf $gcmDIR
    echo "  done"					>> tr_clean_$tt.log
  fi
 #- download new code:
  echo "==========================================================================" \
       >> tr_clean_$tt.log
  echo "Make a clone of $git_code from repo: $git_repo into: $gcmDIR ..." | tee -a  tr_clean_$tt.log
  set -x
  git clone https://github.com/$git_repo/${git_code}.git $gcmDIR 2> $tmpFil
  retv=$?
  set +x
  if test $retv = 0 ; then
    echo '  done' >> tr_clean_$tt.log ; rm -f $tmpFil
  else
    echo "'git clone' into $gcmDIR failed (return: $retv) => skip" | tee -a tr_clean_$tt.log
    cat $tmpFil >> tr_clean_$tt.log ; rm -f $tmpFil
    continue
  fi
 fi
  echo ""
  echo "=========================================================================="

 if test -d $gcmDIR/verification ; then
  if test -e tr_run_$tt.log ; then mv -f tr_run_$tt.log tr_run_$tt.log_bak ; fi
  cd $gcmDIR/verification

  date
  # verbose mode:
  set -x
  pwd
  test -e tr_out.txt && mv -f tr_out.txt tr_out.sav

#- Run test in corresponding clone: --------------------------------------------
case $tt in
'iadm' )

  ./testreport -MPI 3 -adm -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -nc -ncad -repl_mk do_make_syntax.sh -obj -dd > ../../tr_run_$tt.log 2>&1
  echo '' >> ../../tr_run_$tt.log 2>&1

  ./testreport -MPI 3 -adm -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -q $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'iad4' )

  ./testreport -MPI 3 -ur4 -adm -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -nc -ncad -repl_mk do_make_syntax.sh -obj -dd > ../../tr_run_$tt.log 2>&1
  echo '' >> ../../tr_run_$tt.log 2>&1

  ./testreport -MPI 3 -ur4 -adm -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -q -match 5 $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'imp2')

  export OMP_NUM_THREADS=2
  export KMP_STACKSIZE=400m

  ./testreport -MPI 2 -mth -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -nc -repl_mk do_make_syntax.sh -obj -dd > ../../tr_run_$tt.log 2>&1
  echo '' >> ../../tr_run_$tt.log 2>&1

  ./testreport -MPI 2 -mth -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -q $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 -mpi $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'iur4')

  ./testreport -MPI 3 -ur4 -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -nc -repl_mk do_make_syntax.sh -obj -dd > ../../tr_run_$tt.log 2>&1
  echo '' >> ../../tr_run_$tt.log 2>&1

  ./testreport -MPI 3 -ur4 -of ../tools/build_options/linux_amd64_ifort11 \
   -devel -q -match 5 $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 -mpi $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'gadm')

  ./testreport -MPI 3 -adm -of ../tools/build_options/linux_amd64_gfortran \
   -devel -ncad -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'gads')

  ./testreport -adm -of ../tools/build_options/linux_amd64_gfortran \
   -devel -ncad -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'gad4')

  ./testreport -adm -ur4 -of ../tools/build_options/linux_amd64_gfortran \
   -devel -ncad -nc -match 5 $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'tapAD')

  ./testreport -tap -adm -of ../tools/build_options/linux_amd64_gfortran \
   $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'tapTL')

  ./testreport -tap -tlm -of ../tools/build_options/linux_amd64_gfortran \
   $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'g7a')

  ./testreport -adm -of ../tools/build_options/linux_amd64_g77 \
   -skd 'halfpipe_streamice' \
   -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log
;;

'gmp2')

  export OMP_NUM_THREADS=2
  export GOMP_STACKSIZE=400m

  ./testreport -MPI 2 -mth -of ../tools/build_options/linux_amd64_gfortran \
   -devel -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 -mpi $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'gmpi')

  ./testreport -MPI 3 -of ../tools/build_options/linux_amd64_gfortran \
   -devel -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 -mpi $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'gmp4')

  ./testreport -MPI 3 -ur4 -of ../tools/build_options/linux_amd64_gfortran \
   -devel -nc -match 5 $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 -mpi $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'gfo')

  ./testreport -of ../tools/build_options/linux_amd64_gfortran \
   -devel -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'gfo4')

  ./testreport -ur4 -of ../tools/build_options/linux_amd64_gfortran \
   -devel -nc -match 5 $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

'g77')

  ./testreport -of ../tools/build_options/linux_amd64_g77 \
   -skd 'fizhi-cs-32x32x40 fizhi-cs-aqualev20 halfpipe_streamice internal_wave tutorial_advection_in_gyre' \
   -nc $sendCmd >> ../../tr_run_$tt.log 2>&1

  sed -n "/ email /,/^======== End of testreport / p" ../../tr_run_$tt.log

  #- test restart:
  echo '' >> ../../tr_run_$tt.log 2>&1
  ../tools/do_tst_2+2 $sendCmd >> ../../tr_run_$tt.log 2>&1
  sed -n '/^=====  Summary /,$ p' ../../tr_run_$tt.log
;;

*) echo "unrecognized test suffix '$tt' <== skipped" ;;
esac

  set +x
  cd ../..

 else
  echo "error: missing dir $gcmDIR/verification"
 fi

done