Annotation of /MITgcm/verification/cpl_aim+ocn/run_cpl_test

#! /usr/bin/env bash

# $Header: /u/gcmpack/MITgcm_contrib/verification_other/cpl_gray+ocn/run_cpl_test,v 1.4 2014/07/19 22:59:09 jmc Exp $
# $Name:  $

#- default:
 Npr=3 ; NpOc=1 ;
#Npr=25; NpOc=12;
MTH=
MTHo=
MTHa=
GMKopt='-devel'
#GMKopt='-ieee'
#GMKopt=
rnkO=1 ; rnkA=`expr $rnkO + $NpOc`

#- parse options:
if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
  MTH='-omp' ; shift
  if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
  if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
fi ; fi
#- allows more argument for building step (step=1)
chkArg=$# ; if [ $# -gt 1 ]; then chkArg=$1 ; fi
if [ $chkArg -ne 1 ]; then
  echo 'Usage:'`basename $0`' [opt] step [bld-arg]'
  echo ' => test coupled set-up on linux box (1.cpu)'
  echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
  echo ' step = 0 : clean all directories'
  echo ' step = 1 : compile the 3 executables (cpl,ocn,atm); optional arg:'
  echo '       -of Optfile_Name : using option-file "Optfile_Name"'
  echo ' step = 2 : copy input files and dir(s)'
  echo " step = 3 : run with $Npr mpi processes"
  echo ' step = 4 : check the results'
  echo ' step = 5 : remove output files in rank_0,1,2 dir.'
  exit
fi
kpr=$1
dir=`pwd`

#============================================================================

if test $kpr = 0
then
 rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
 rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
 /bin/rm -r -f rank_? rank_1? rank_2?
 if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
 if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
 if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
fi
if test $kpr = 5
then
 echo 'remove output files in rank_0,1,2 dir.'
 rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
 if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
 if test -d rank_$rnkO
 then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
 if test -d rank_$rnkA
 then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
fi

if test $kpr = 1
then

#- choice of the optfile:
#  default: take a local one in dir verification with sufix '+mpi'
 nbOpF=`ls ../linux_* | grep '+mpi' 2> /dev/null | wc -l`
#  or take the one given as argument:
 if [ $# -ge 3 ]; then
   if test $2 = '-of' -a -f $3 ; then nbOpF=-1 ; OPTFILE=$3 ; fi
 fi
 if test $nbOpF = 1 ; then
   OPTFILE=`ls ../linux_* | grep '+mpi'`
 elif [ $nbOpF -ge 2 ] ; then
   echo "pick the 1rst of these ( $nbOpF ) optfiles:"
   ls ../linux_* | grep '+mpi'
   OPTFILE=`ls ../linux_* | grep '+mpi' | head -1`
 elif [ $nbOpF -ne -1 ] ; then
   echo "Pb in finding optfile: found $nbOpF :"
   ls ../linux_* | grep '+mpi' ; exit
 fi
 zz=`grep '^ *FC=' $OPTFILE | tail -1`
 echo " Using optfile: $OPTFILE  (compiler: $zz) $MTH"
 zz=`echo $OPTFILE | grep -c '^\/'`
 if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
#---
 echo '==== compile coupler:'
 cd build_cpl
 echo ' --- genmake2 (cpl):'
 ../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt >  TTT.genmake.$$ 2>&1
 RetVal=$? ; tail -5 TTT.genmake.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in genmake2 (cpl)" ; exit 11
 fi
 echo ' --- make depend (cpl):'
 make depend > TTT.mkdepend.$$ 2>&1
 RetVal=$? ; tail -5 TTT.mkdepend.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in mkdepend (cpl)" ; exit 12
 fi
 echo ' --- make (cpl):' ; touch TTT.make.$$
#do_make_syntax.sh obj > TTT.make.$$ 2>&1
 make >> TTT.make.$$ 2>&1
 RetVal=$? ; tail -10 TTT.make.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in make     (cpl)" ; exit 13
 fi
 echo ' ' ; cd $dir

 echo '==== compile OGCM:'
 cd build_ocn
 echo ' --- genmake2 (ocn):'
 ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt >  TTT.genmake.$$ 2>&1
 RetVal=$? ; tail -5 TTT.genmake.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in genmake2 (ocn)" ; exit 21
 fi
 echo ' --- make depend (ocn):'
 make depend > TTT.mkdepend.$$ 2>&1
 RetVal=$? ; tail -10 TTT.mkdepend.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in mkdepend (ocn)" ; exit 22
 fi
 echo ' --- make (ocn):' ; touch TTT.make.$$
#do_make_syntax.sh obj > TTT.make.$$ 2>&1
 make >> TTT.make.$$ 2>&1
 RetVal=$? ; tail -10 TTT.make.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in make     (ocn)" ; exit 23
 fi
 echo ' ' ; cd $dir

 echo '==== compile AGCM:'
 cd build_atm
 echo ' --- genmake2 (atm):'
 ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt >  TTT.genmake.$$ 2>&1
 RetVal=$? ; tail -5 TTT.genmake.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in genmake2 (atm)" ; exit 31
 fi
 echo ' --- make depend (atm):'
 make depend > TTT.mkdepend.$$ 2>&1
 RetVal=$? ; tail -10 TTT.mkdepend.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in mkdepend (atm)" ; exit 32
 fi
 echo ' --- make (atm):' ; touch TTT.make.$$
#do_make_syntax.sh obj > TTT.make.$$ 2>&1
 make >> TTT.make.$$ 2>&1
 RetVal=$? ; tail -10 TTT.make.$$
 if test "x$RetVal" != x0 ; then
   echo "Error in make     (atm)" ; exit 33
 fi
 echo ' ' ; cd $dir

 ls -l build_???/mitgcmuv

fi

if test $kpr = 2
then
  echo 'rm dir:' rank_? rank_1? rank_2?
  /bin/rm -r -f rank_? rank_1? rank_2?
  echo 'Link files from dir:' input_cpl '->' rank_0
  mkdir rank_0
  ( cd rank_0 ; ln -s ../input_cpl/* . )

  n=$rnkO
  echo 'Link files from dir:' input_ocn '->' rank_$n
  mkdir rank_$n
  cd rank_$n
  ln -s ../input_ocn/* .
  if test -x prepare_run ; then ./prepare_run ; fi
  if test "x$MTHo" != x ; then
    echo " MTH run: mv -f eedata.mth eedata"
    if test -h eedata ; then rm -f eedata ; fi
    mv -f eedata.mth eedata
  fi
  cd $dir
  n=`expr $n + 1`
  while [ $n -le $NpOc ] ; do
    ln -s rank_$rnkO rank_$n
    n=`expr $n + 1`
  done

  n=$rnkA
  echo 'Link files from dir:' input_atm '->' rank_$n
  mkdir rank_$n
  cd rank_$n
  ln -s ../input_atm/* .
  if test -x prepare_run ; then ./prepare_run ; fi
  if test "x$MTHa" != x ; then
    echo " MTH run: mv -f eedata.mth eedata"
    if test -h eedata ; then rm -f eedata ; fi
    mv -f eedata.mth eedata
  fi
  cd $dir
  n=`expr $n + 1`
  while [ $n -lt $Npr ] ; do
    ln -s rank_$rnkA rank_$n
    n=`expr $n + 1`
  done

fi

if test $kpr = 3
then
  ROOTDIR=$dir
# rm -f rank_?/pickup*.ckptA.00?.00?.??ta
  echo $ROOTDIR
  tmpfil=TTT.$$

#--- running on the same node:
  list='' ; nc=0; xx=`hostname`
  while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
#-- On darwin cluster node (from qrsh session):
# JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
# NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
# echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
#-- On ACES cluster (in PBS batch job):
# NODEFILE=$PBS_NODEFILE
#--- running on different nodes:
# ls -l $NODEFILE
# nprc=`cat $NODEFILE | uniq | wc -l`
# if [ $nprc -ge $Npr ] ; then
#   list=`cat $NODEFILE | uniq | head -$Npr`
# else
#   list=`cat $NODEFILE | head -$Npr`
# fi

  nc=0; nn=0; dd1=cpl ;
  rm -f pr_group ; touch pr_group
  for xx in $list
  do
    echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
    nc=`expr $nc + 1`
    if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
    nn=1
  done
  NpAt=`expr $Npr - 1 - $NpOc`
  RunOpt="-np 1 ./build_cpl/mitgcmuv"
  RunOpt="$RunOpt : -np $NpOc ./build_ocn/mitgcmuv"
  RunOpt="$RunOpt : -np $NpAt ./build_atm/mitgcmuv"

  cd $ROOTDIR
  if test "x$MTH" != x ; then
    export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
    if test "x$MTHo" != x ; then
      echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
    fi
    if test "x$MTHa" != x ; then
      echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
    fi
    echo ""
  fi
  mpich=`which mpirun`
  echo $mpich | grep 'mpich-mx' > /dev/null 2>&1
  mpichmx=$?
  echo $mpich | grep 'mpich-1' > /dev/null 2>&1
  mpich1=$?
  echo $mpich | grep 'mpich2' > /dev/null 2>&1
  mpich2=$?
  echo $mpich | grep 'openmpi' > /dev/null 2>&1
  opnmpi=$?
  if test $mpich1 == 0 ; then
  # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1
  #- with mpich-1 (on danton, old aces: ao, geo, itrda):
    echo "execute 'mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv' :"
    mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
  elif test $mpichmx == 0 ; then
  #- with mpich-mx (on beagle):
    echo "execute 'mpirun -pg pr_group -v ./build_cpl/mitgcmuv' :"
    mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
  elif test $mpich2 == 0 -o $opnmpi == 0 ; then
  #- with Hydra mpich2 (on baudelaire) or with openmpi:
    echo "execute 'mpirun $RunOpt' :"
    mpirun $RunOpt  > std_outp 2>&1
  else
  #- new mpich (mpich2) installation often just put in "mpich" dir
    echo "execute 'mpirun $RunOpt' :"
    mpirun $RunOpt  > std_outp 2>&1
  fi
  tail -20 std_outp
  ls -l rank_$rnkO/pickup*.ckptA.001.001.data
  ls -l rank_$rnkA/pickup*.ckptA.001.001.data

fi

if test $kpr = 4
then
 CompRes="$HOME/bin/comp_res"

 if test -f rank_$rnkO/STDOUT.0000
 then echo '==> check Ocean output:'
    $CompRes rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
    mv -f comp_res.log comp_res.ocn
    echo ' '
 else echo "No Ocean output file in rank_$rnkO" ; fi

 if test -f rank_$rnkA/STDOUT.0000
 then
    echo '==> check Atmos output:'
    $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
    mv -f comp_res.log comp_res.atm
    echo '==> check Land output:'
    $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
    mv -f comp_res.log comp_res.land
    echo '==> check thSIce output:'
    $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
    mv -f comp_res.log comp_res.sice
    echo ' '
 else echo "No Atmos output file in rank_$rnkA" ; fi

fi

exit 0
1	jmc	1.22	#! /usr/bin/env bash
2	jmc	1.1
3	jmc	1.23	# $Header: /u/gcmpack/MITgcm_contrib/verification_other/cpl_gray+ocn/run_cpl_test,v 1.4 2014/07/19 22:59:09 jmc Exp $
4	jmc	1.1	# $Name: $
5
6	jmc	1.16	#- default:
7	jmc	1.17	Npr=3 ; NpOc=1 ;
8			#Npr=25; NpOc=12;
9	jmc	1.13	MTH=
10			MTHo=
11			MTHa=
12	jmc	1.20	GMKopt='-devel'
13			#GMKopt='-ieee'
14	jmc	1.17	#GMKopt=
15			rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
16	jmc	1.16
17			#- parse options:
18	jmc	1.15	if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
19	jmc	1.14	MTH='-omp' ; shift
20			if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
21			if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
22	jmc	1.15	fi ; fi
23	jmc	1.20	#- allows more argument for building step (step=1)
24			chkArg=$# ; if [ $# -gt 1 ]; then chkArg=$1 ; fi
25			if [ $chkArg -ne 1 ]; then
26			echo 'Usage:'`basename $0`' [opt] step [bld-arg]'
27	jmc	1.10	echo ' => test coupled set-up on linux box (1.cpu)'
28	jmc	1.14	echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
29			echo ' step = 0 : clean all directories'
30	jmc	1.20	echo ' step = 1 : compile the 3 executables (cpl,ocn,atm); optional arg:'
31			echo ' -of Optfile_Name : using option-file "Optfile_Name"'
32	jmc	1.14	echo ' step = 2 : copy input files and dir(s)'
33	jmc	1.17	echo " step = 3 : run with $Npr mpi processes"
34	jmc	1.14	echo ' step = 4 : check the results'
35			echo ' step = 5 : remove output files in rank_0,1,2 dir.'
36	jmc	1.1	exit
37			fi
38			kpr=$1
39			dir=`pwd`
40
41			#============================================================================
42
43			if test $kpr = 0
44			then
45	jmc	1.11	rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
46	jmc	1.1	rm -f build_???/TTT.make. build_???/TTT.mkdepend.*
47	jmc	1.17	/bin/rm -r -f rank_? rank_1? rank_2?
48			if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
49			if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
50			if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
51	jmc	1.1	fi
52	jmc	1.8	if test $kpr = 5
53			then
54			echo 'remove output files in rank_0,1,2 dir.'
55	jmc	1.11	rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
56	jmc	1.8	if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
57	jmc	1.17	if test -d rank_$rnkO
58			then cd rank_$rnkO ; rm -f .data .meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
59			if test -d rank_$rnkA
60			then cd rank_$rnkA ; rm -f .data .meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
61	jmc	1.8	fi
62	jmc	1.1
63			if test $kpr = 1
64			then
65
66	jmc	1.20	#- choice of the optfile:
67			# default: take a local one in dir verification with sufix '+mpi'
68	jmc	1.16	nbOpF=`ls ../linux_* \| grep '+mpi' 2> /dev/null \| wc -l`
69	jmc	1.20	# or take the one given as argument:
70			if [ $# -ge 3 ]; then
71			if test $2 = '-of' -a -f $3 ; then nbOpF=-1 ; OPTFILE=$3 ; fi
72			fi
73	jmc	1.17	if test $nbOpF = 1 ; then
74	jmc	1.16	OPTFILE=`ls ../linux_* \| grep '+mpi'`
75	jmc	1.17	elif [ $nbOpF -ge 2 ] ; then
76			echo "pick the 1rst of these ( $nbOpF ) optfiles:"
77			ls ../linux_* \| grep '+mpi'
78			OPTFILE=`ls ../linux_* \| grep '+mpi' \| head -1`
79	jmc	1.20	elif [ $nbOpF -ne -1 ] ; then
80	jmc	1.16	echo "Pb in finding optfile: found $nbOpF :"
81	jmc	1.17	ls ../linux_* \| grep '+mpi' ; exit
82	jmc	1.10	fi
83	jmc	1.20	zz=`grep '^ *FC=' $OPTFILE \| tail -1`
84			echo " Using optfile: $OPTFILE (compiler: $zz) $MTH"
85	jmc	1.11	zz=`echo $OPTFILE \| grep -c '^\/'`
86			if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
87	jmc	1.10	#---
88	jmc	1.1	echo '==== compile coupler:'
89			cd build_cpl
90			echo ' --- genmake2 (cpl):'
91	jmc	1.22	../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt > TTT.genmake.$$ 2>&1
92	jmc	1.23	RetVal=$? ; tail -5 TTT.genmake.$$
93			if test "x$RetVal" != x0 ; then
94			echo "Error in genmake2 (cpl)" ; exit 11
95			fi
96	jmc	1.1	echo ' --- make depend (cpl):'
97	jmc	1.22	make depend > TTT.mkdepend.$$ 2>&1
98	jmc	1.23	RetVal=$? ; tail -5 TTT.mkdepend.$$
99			if test "x$RetVal" != x0 ; then
100			echo "Error in mkdepend (cpl)" ; exit 12
101			fi
102	jmc	1.21	echo ' --- make (cpl):' ; touch TTT.make.$$
103			#do_make_syntax.sh obj > TTT.make.$$ 2>&1
104			make >> TTT.make.$$ 2>&1
105	jmc	1.23	RetVal=$? ; tail -10 TTT.make.$$
106			if test "x$RetVal" != x0 ; then
107			echo "Error in make (cpl)" ; exit 13
108			fi
109	jmc	1.1	echo ' ' ; cd $dir
110
111			echo '==== compile OGCM:'
112			cd build_ocn
113			echo ' --- genmake2 (ocn):'
114	jmc	1.22	../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt > TTT.genmake.$$ 2>&1
115	jmc	1.23	RetVal=$? ; tail -5 TTT.genmake.$$
116			if test "x$RetVal" != x0 ; then
117			echo "Error in genmake2 (ocn)" ; exit 21
118			fi
119	jmc	1.1	echo ' --- make depend (ocn):'
120	jmc	1.22	make depend > TTT.mkdepend.$$ 2>&1
121	jmc	1.23	RetVal=$? ; tail -10 TTT.mkdepend.$$
122			if test "x$RetVal" != x0 ; then
123			echo "Error in mkdepend (ocn)" ; exit 22
124			fi
125	jmc	1.21	echo ' --- make (ocn):' ; touch TTT.make.$$
126			#do_make_syntax.sh obj > TTT.make.$$ 2>&1
127			make >> TTT.make.$$ 2>&1
128	jmc	1.23	RetVal=$? ; tail -10 TTT.make.$$
129			if test "x$RetVal" != x0 ; then
130			echo "Error in make (ocn)" ; exit 23
131			fi
132	jmc	1.1	echo ' ' ; cd $dir
133
134			echo '==== compile AGCM:'
135			cd build_atm
136			echo ' --- genmake2 (atm):'
137	jmc	1.22	../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt > TTT.genmake.$$ 2>&1
138	jmc	1.23	RetVal=$? ; tail -5 TTT.genmake.$$
139			if test "x$RetVal" != x0 ; then
140			echo "Error in genmake2 (atm)" ; exit 31
141			fi
142	jmc	1.1	echo ' --- make depend (atm):'
143	jmc	1.22	make depend > TTT.mkdepend.$$ 2>&1
144	jmc	1.23	RetVal=$? ; tail -10 TTT.mkdepend.$$
145			if test "x$RetVal" != x0 ; then
146			echo "Error in mkdepend (atm)" ; exit 32
147			fi
148	jmc	1.21	echo ' --- make (atm):' ; touch TTT.make.$$
149			#do_make_syntax.sh obj > TTT.make.$$ 2>&1
150			make >> TTT.make.$$ 2>&1
151	jmc	1.23	RetVal=$? ; tail -10 TTT.make.$$
152			if test "x$RetVal" != x0 ; then
153			echo "Error in make (atm)" ; exit 33
154			fi
155	jmc	1.1	echo ' ' ; cd $dir
156
157			ls -l build_???/mitgcmuv
158
159			fi
160
161			if test $kpr = 2
162			then
163	jmc	1.17	echo 'rm dir:' rank_? rank_1? rank_2?
164			/bin/rm -r -f rank_? rank_1? rank_2?
165			echo 'Link files from dir:' input_cpl '->' rank_0
166			mkdir rank_0
167			( cd rank_0 ; ln -s ../input_cpl/* . )
168
169			n=$rnkO
170			echo 'Link files from dir:' input_ocn '->' rank_$n
171			mkdir rank_$n
172			cd rank_$n
173			ln -s ../input_ocn/* .
174			if test -x prepare_run ; then ./prepare_run ; fi
175	jmc	1.13	if test "x$MTHo" != x ; then
176			echo " MTH run: mv -f eedata.mth eedata"
177	jmc	1.17	if test -h eedata ; then rm -f eedata ; fi
178	jmc	1.13	mv -f eedata.mth eedata
179			fi
180	jmc	1.1	cd $dir
181	jmc	1.17	n=`expr $n + 1`
182			while [ $n -le $NpOc ] ; do
183			ln -s rank_$rnkO rank_$n
184			n=`expr $n + 1`
185			done
186
187			n=$rnkA
188			echo 'Link files from dir:' input_atm '->' rank_$n
189			mkdir rank_$n
190			cd rank_$n
191			ln -s ../input_atm/* .
192			if test -x prepare_run ; then ./prepare_run ; fi
193	jmc	1.13	if test "x$MTHa" != x ; then
194			echo " MTH run: mv -f eedata.mth eedata"
195	jmc	1.17	if test -h eedata ; then rm -f eedata ; fi
196	jmc	1.13	mv -f eedata.mth eedata
197			fi
198	jmc	1.1	cd $dir
199	jmc	1.17	n=`expr $n + 1`
200			while [ $n -lt $Npr ] ; do
201			ln -s rank_$rnkA rank_$n
202			n=`expr $n + 1`
203			done
204	jmc	1.1
205			fi
206
207			if test $kpr = 3
208			then
209	jmc	1.17	ROOTDIR=$dir
210			# rm -f rank_?/pickup*.ckptA.00?.00?.??ta
211			echo $ROOTDIR
212			tmpfil=TTT.$$
213
214			#--- running on the same node:
215			list='' ; nc=0; xx=`hostname`
216			while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
217			#-- On darwin cluster node (from qrsh session):
218			# JOB_ID=`qstat \| sed -n '3,$ p' \| grep " $USER " \| awk '{print $1}'`
219			# NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
220			# echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
221			#-- On ACES cluster (in PBS batch job):
222			# NODEFILE=$PBS_NODEFILE
223			#--- running on different nodes:
224			# ls -l $NODEFILE
225			# nprc=`cat $NODEFILE \| uniq \| wc -l`
226			# if [ $nprc -ge $Npr ] ; then
227			# list=`cat $NODEFILE \| uniq \| head -$Npr`
228			# else
229			# list=`cat $NODEFILE \| head -$Npr`
230			# fi
231
232			nc=0; nn=0; dd1=cpl ;
233			rm -f pr_group ; touch pr_group
234			for xx in $list
235			do
236			echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
237			nc=`expr $nc + 1`
238			if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
239			nn=1
240			done
241	jmc	1.18	NpAt=`expr $Npr - 1 - $NpOc`
242			RunOpt="-np 1 ./build_cpl/mitgcmuv"
243			RunOpt="$RunOpt : -np $NpOc ./build_ocn/mitgcmuv"
244			RunOpt="$RunOpt : -np $NpAt ./build_atm/mitgcmuv"
245	jmc	1.17
246			cd $ROOTDIR
247			if test "x$MTH" != x ; then
248			export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
249			if test "x$MTHo" != x ; then
250			echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
251			fi
252			if test "x$MTHa" != x ; then
253			echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
254			fi
255			echo ""
256			fi
257	jmc	1.18	mpich=`which mpirun`
258			echo $mpich \| grep 'mpich-mx' > /dev/null 2>&1
259			mpichmx=$?
260	jmc	1.21	echo $mpich \| grep 'mpich-1' > /dev/null 2>&1
261			mpich1=$?
262	jmc	1.18	echo $mpich \| grep 'mpich2' > /dev/null 2>&1
263			mpich2=$?
264	jmc	1.19	echo $mpich \| grep 'openmpi' > /dev/null 2>&1
265			opnmpi=$?
266	jmc	1.21	if test $mpich1 == 0 ; then
267			# /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v ./build_cpl/mitgcmuv > std_outp 2>&1
268			#- with mpich-1 (on danton, old aces: ao, geo, itrda):
269			echo "execute 'mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv' :"
270			mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
271	jmc	1.18	elif test $mpichmx == 0 ; then
272			#- with mpich-mx (on beagle):
273			echo "execute 'mpirun -pg pr_group -v ./build_cpl/mitgcmuv' :"
274			mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
275	jmc	1.21	elif test $mpich2 == 0 -o $opnmpi == 0 ; then
276			#- with Hydra mpich2 (on baudelaire) or with openmpi:
277			echo "execute 'mpirun $RunOpt' :"
278			mpirun $RunOpt > std_outp 2>&1
279	jmc	1.18	else
280	jmc	1.21	#- new mpich (mpich2) installation often just put in "mpich" dir
281			echo "execute 'mpirun $RunOpt' :"
282			mpirun $RunOpt > std_outp 2>&1
283	jmc	1.18	fi
284	jmc	1.17	tail -20 std_outp
285			ls -l rank_$rnkO/pickup*.ckptA.001.001.data
286			ls -l rank_$rnkA/pickup*.ckptA.001.001.data
287	jmc	1.1
288			fi
289
290			if test $kpr = 4
291			then
292	jmc	1.18	CompRes="$HOME/bin/comp_res"
293	jmc	1.1
294	jmc	1.17	if test -f rank_$rnkO/STDOUT.0000
295	jmc	1.1	then echo '==> check Ocean output:'
296	jmc	1.18	$CompRes rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
297	jmc	1.10	mv -f comp_res.log comp_res.ocn
298			echo ' '
299	jmc	1.17	else echo "No Ocean output file in rank_$rnkO" ; fi
300	jmc	1.1
301	jmc	1.17	if test -f rank_$rnkA/STDOUT.0000
302			then
303	jmc	1.1	echo '==> check Atmos output:'
304	jmc	1.18	$CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
305	jmc	1.10	mv -f comp_res.log comp_res.atm
306	jmc	1.1	echo '==> check Land output:'
307	jmc	1.18	$CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
308	jmc	1.10	mv -f comp_res.log comp_res.land
309	jmc	1.1	echo '==> check thSIce output:'
310	jmc	1.18	$CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
311	jmc	1.10	mv -f comp_res.log comp_res.sice
312			echo ' '
313	jmc	1.17	else echo "No Atmos output file in rank_$rnkA" ; fi
314	jmc	1.1
315			fi
316
317			exit 0