Annotation of /MITgcm/verification/cpl_aim+ocn/run_cpl_test

#! /bin/sh

# $Header: /u/gcmpack/MITgcm/verification/cpl_aim+ocn/run_cpl_test,v 1.16 2010/11/22 19:03:43 jmc Exp $
# $Name:  $

#- default:
 Npr=3 ; NpOc=1 ;
#Npr=25; NpOc=12;
MTH=
MTHo=
MTHa=
GMKopt='-ieee'
#GMKopt=
rnkO=1 ; rnkA=`expr $rnkO + $NpOc`

#- parse options:
if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
  MTH='-omp' ; shift
  if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
  if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
fi ; fi
if [ $# -ne 1 ]
then
  echo 'Usage:'`basename $0`' [opt] step'
  echo ' => test coupled set-up on linux box (1.cpu)'
  echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
  echo ' step = 0 : clean all directories'
  echo ' step = 1 : compile the 3 executables (cpl,ocn,atm)'
  echo ' step = 2 : copy input files and dir(s)'
  echo " step = 3 : run with $Npr mpi processes"
  echo ' step = 4 : check the results'
  echo ' step = 5 : remove output files in rank_0,1,2 dir.'
  exit
fi
kpr=$1
dir=`pwd`

#============================================================================

if test $kpr = 0
then
 rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
 rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
 /bin/rm -r -f rank_? rank_1? rank_2?
 if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
 if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
 if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
fi
if test $kpr = 5
then
 echo 'remove output files in rank_0,1,2 dir.'
 rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
 if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
 if test -d rank_$rnkO
 then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
 if test -d rank_$rnkA
 then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
fi

if test $kpr = 1
then

#- choice of the optfile: take a local one in dir verification with sufix '+mpi'
 nbOpF=`ls ../linux_* | grep '+mpi' 2> /dev/null | wc -l`
 if test $nbOpF = 1 ; then
   OPTFILE=`ls ../linux_* | grep '+mpi'`
   zz=`grep '^FC=' $OPTFILE`
   echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"
 elif [ $nbOpF -ge 2 ] ; then
   echo "pick the 1rst of these ( $nbOpF ) optfiles:"
   ls ../linux_* | grep '+mpi'
   OPTFILE=`ls ../linux_* | grep '+mpi' | head -1`
   zz=`grep '^FC=' $OPTFILE`
   echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"
 else
   echo "Pb in finding optfile: found $nbOpF :"
   ls ../linux_* | grep '+mpi' ; exit
 fi
 zz=`echo $OPTFILE | grep -c '^\/'`
 if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
#---
 echo '==== compile coupler:'
 cd build_cpl
 echo ' --- genmake2 (cpl):'
 ../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt >  TTT.genmake.$$
 tail -5 TTT.genmake.$$
 echo ' --- make depend (cpl):'
 make depend > TTT.mkdepend.$$
 tail -5 TTT.mkdepend.$$
 echo ' --- make (cpl):'
 make > TTT.make.$$ 2>&1
 tail -10 TTT.make.$$
 echo ' ' ; cd $dir

 echo '==== compile OGCM:'
 cd build_ocn
 echo ' --- genmake2 (ocn):'
 ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt >  TTT.genmake.$$
 tail -5 TTT.genmake.$$
 echo ' --- make depend (ocn):'
 make depend > TTT.mkdepend.$$
 tail -10 TTT.mkdepend.$$
 echo ' --- make (ocn):'
 make > TTT.make.$$ 2>&1
 tail -10 TTT.make.$$
 echo ' ' ; cd $dir

 echo '==== compile AGCM:'
 cd build_atm
 echo ' --- genmake2 (atm):'
 ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt >  TTT.genmake.$$
 tail -5 TTT.genmake.$$
 echo ' --- make depend (atm):'
 make depend > TTT.mkdepend.$$
 tail -10 TTT.mkdepend.$$
 echo ' --- make (atm):'
 make > TTT.make.$$ 2>&1
 tail -10 TTT.make.$$
 echo ' ' ; cd $dir

 ls -l build_???/mitgcmuv

fi

if test $kpr = 2
then
  echo 'rm dir:' rank_? rank_1? rank_2?
  /bin/rm -r -f rank_? rank_1? rank_2?
  echo 'Link files from dir:' input_cpl '->' rank_0
  mkdir rank_0
  ( cd rank_0 ; ln -s ../input_cpl/* . )

  n=$rnkO
  echo 'Link files from dir:' input_ocn '->' rank_$n
  mkdir rank_$n
  cd rank_$n
  ln -s ../input_ocn/* .
  if test -x prepare_run ; then ./prepare_run ; fi
  if test "x$MTHo" != x ; then
    echo " MTH run: mv -f eedata.mth eedata"
    if test -h eedata ; then rm -f eedata ; fi
    mv -f eedata.mth eedata
  fi
  cd $dir
  n=`expr $n + 1`
  while [ $n -le $NpOc ] ; do
    ln -s rank_$rnkO rank_$n
    n=`expr $n + 1`
  done

  n=$rnkA
  echo 'Link files from dir:' input_atm '->' rank_$n
  mkdir rank_$n
  cd rank_$n
  ln -s ../input_atm/* .
  if test -x prepare_run ; then ./prepare_run ; fi
  if test "x$MTHa" != x ; then
    echo " MTH run: mv -f eedata.mth eedata"
    if test -h eedata ; then rm -f eedata ; fi
    mv -f eedata.mth eedata
  fi
  cd $dir
  n=`expr $n + 1`
  while [ $n -lt $Npr ] ; do
    ln -s rank_$rnkA rank_$n
    n=`expr $n + 1`
  done

fi

if test $kpr = 3
then
  ROOTDIR=$dir
# rm -f rank_?/pickup*.ckptA.00?.00?.??ta
  echo $ROOTDIR
  tmpfil=TTT.$$

#--- running on the same node:
  list='' ; nc=0; xx=`hostname`
  while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
#-- On darwin cluster node (from qrsh session):
# JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
# NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
# echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
#-- On ACES cluster (in PBS batch job):
# NODEFILE=$PBS_NODEFILE
#--- running on different nodes:
# ls -l $NODEFILE
# nprc=`cat $NODEFILE | uniq | wc -l`
# if [ $nprc -ge $Npr ] ; then
#   list=`cat $NODEFILE | uniq | head -$Npr`
# else
#   list=`cat $NODEFILE | head -$Npr`
# fi

  nc=0; nn=0; dd1=cpl ;
  rm -f pr_group ; touch pr_group
  for xx in $list
  do
    echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
    nc=`expr $nc + 1`
    if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
    nn=1
  done

  cd $ROOTDIR
  if test "x$MTH" != x ; then
    export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
    if test "x$MTHo" != x ; then
      echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
    fi
    if test "x$MTHa" != x ; then
      echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
    fi
    echo ""
  fi
# /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1
#- with mpich-1 (on danton, ACES):
  mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
#- with mpich-mx (on beagle):
# mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
  tail -20 std_outp
  ls -l rank_$rnkO/pickup*.ckptA.001.001.data
  ls -l rank_$rnkA/pickup*.ckptA.001.001.data

fi

if test $kpr = 4
then

 if test -f rank_$rnkO/STDOUT.0000
 then echo '==> check Ocean output:'
    /home/jmc/bin/comp_res rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
    mv -f comp_res.log comp_res.ocn
    echo ' '
 else echo "No Ocean output file in rank_$rnkO" ; fi

 if test -f rank_$rnkA/STDOUT.0000
 then
    echo '==> check Atmos output:'
    /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
    mv -f comp_res.log comp_res.atm
    echo '==> check Land output:'
    /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
    mv -f comp_res.log comp_res.land
    echo '==> check thSIce output:'
    /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
    mv -f comp_res.log comp_res.sice
    echo ' '
 else echo "No Atmos output file in rank_$rnkA" ; fi

fi

exit 0
1	jmc	1.1	#! /bin/sh
2
3	jmc	1.17	# $Header: /u/gcmpack/MITgcm/verification/cpl_aim+ocn/run_cpl_test,v 1.16 2010/11/22 19:03:43 jmc Exp $
4	jmc	1.1	# $Name: $
5
6	jmc	1.16	#- default:
7	jmc	1.17	Npr=3 ; NpOc=1 ;
8			#Npr=25; NpOc=12;
9	jmc	1.13	MTH=
10			MTHo=
11			MTHa=
12	jmc	1.16	GMKopt='-ieee'
13	jmc	1.17	#GMKopt=
14			rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
15	jmc	1.16
16			#- parse options:
17	jmc	1.15	if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
18	jmc	1.14	MTH='-omp' ; shift
19			if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
20			if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
21	jmc	1.15	fi ; fi
22	jmc	1.14	if [ $# -ne 1 ]
23	jmc	1.1	then
24	jmc	1.14	echo 'Usage:'`basename $0`' [opt] step'
25	jmc	1.10	echo ' => test coupled set-up on linux box (1.cpu)'
26	jmc	1.14	echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
27			echo ' step = 0 : clean all directories'
28			echo ' step = 1 : compile the 3 executables (cpl,ocn,atm)'
29			echo ' step = 2 : copy input files and dir(s)'
30	jmc	1.17	echo " step = 3 : run with $Npr mpi processes"
31	jmc	1.14	echo ' step = 4 : check the results'
32			echo ' step = 5 : remove output files in rank_0,1,2 dir.'
33	jmc	1.1	exit
34			fi
35			kpr=$1
36			dir=`pwd`
37
38			#============================================================================
39
40			if test $kpr = 0
41			then
42	jmc	1.11	rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
43	jmc	1.1	rm -f build_???/TTT.make. build_???/TTT.mkdepend.*
44	jmc	1.17	/bin/rm -r -f rank_? rank_1? rank_2?
45			if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
46			if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
47			if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
48	jmc	1.1	fi
49	jmc	1.8	if test $kpr = 5
50			then
51			echo 'remove output files in rank_0,1,2 dir.'
52	jmc	1.11	rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
53	jmc	1.8	if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
54	jmc	1.17	if test -d rank_$rnkO
55			then cd rank_$rnkO ; rm -f .data .meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
56			if test -d rank_$rnkA
57			then cd rank_$rnkA ; rm -f .data .meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
58	jmc	1.8	fi
59	jmc	1.1
60			if test $kpr = 1
61			then
62
63	jmc	1.12	#- choice of the optfile: take a local one in dir verification with sufix '+mpi'
64	jmc	1.16	nbOpF=`ls ../linux_* \| grep '+mpi' 2> /dev/null \| wc -l`
65	jmc	1.17	if test $nbOpF = 1 ; then
66	jmc	1.16	OPTFILE=`ls ../linux_* \| grep '+mpi'`
67	jmc	1.11	zz=`grep '^FC=' $OPTFILE`
68	jmc	1.13	echo " Using optfile: $OPTFILE (compiler=$zz) $MTH"
69	jmc	1.17	elif [ $nbOpF -ge 2 ] ; then
70			echo "pick the 1rst of these ( $nbOpF ) optfiles:"
71			ls ../linux_* \| grep '+mpi'
72			OPTFILE=`ls ../linux_* \| grep '+mpi' \| head -1`
73			zz=`grep '^FC=' $OPTFILE`
74			echo " Using optfile: $OPTFILE (compiler=$zz) $MTH"
75			else
76	jmc	1.16	echo "Pb in finding optfile: found $nbOpF :"
77	jmc	1.17	ls ../linux_* \| grep '+mpi' ; exit
78	jmc	1.10	fi
79	jmc	1.11	zz=`echo $OPTFILE \| grep -c '^\/'`
80			if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
81	jmc	1.10	#---
82	jmc	1.1	echo '==== compile coupler:'
83			cd build_cpl
84			echo ' --- genmake2 (cpl):'
85	jmc	1.16	../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt > TTT.genmake.$$
86	jmc	1.1	tail -5 TTT.genmake.$$
87			echo ' --- make depend (cpl):'
88			make depend > TTT.mkdepend.$$
89			tail -5 TTT.mkdepend.$$
90			echo ' --- make (cpl):'
91			make > TTT.make.$$ 2>&1
92			tail -10 TTT.make.$$
93			echo ' ' ; cd $dir
94
95			echo '==== compile OGCM:'
96			cd build_ocn
97			echo ' --- genmake2 (ocn):'
98	jmc	1.16	../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt > TTT.genmake.$$
99	jmc	1.1	tail -5 TTT.genmake.$$
100			echo ' --- make depend (ocn):'
101			make depend > TTT.mkdepend.$$
102			tail -10 TTT.mkdepend.$$
103			echo ' --- make (ocn):'
104			make > TTT.make.$$ 2>&1
105			tail -10 TTT.make.$$
106			echo ' ' ; cd $dir
107
108			echo '==== compile AGCM:'
109			cd build_atm
110			echo ' --- genmake2 (atm):'
111	jmc	1.16	../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt > TTT.genmake.$$
112	jmc	1.1	tail -5 TTT.genmake.$$
113			echo ' --- make depend (atm):'
114			make depend > TTT.mkdepend.$$
115			tail -10 TTT.mkdepend.$$
116			echo ' --- make (atm):'
117			make > TTT.make.$$ 2>&1
118			tail -10 TTT.make.$$
119			echo ' ' ; cd $dir
120
121			ls -l build_???/mitgcmuv
122
123			fi
124
125			if test $kpr = 2
126			then
127	jmc	1.17	echo 'rm dir:' rank_? rank_1? rank_2?
128			/bin/rm -r -f rank_? rank_1? rank_2?
129			echo 'Link files from dir:' input_cpl '->' rank_0
130			mkdir rank_0
131			( cd rank_0 ; ln -s ../input_cpl/* . )
132
133			n=$rnkO
134			echo 'Link files from dir:' input_ocn '->' rank_$n
135			mkdir rank_$n
136			cd rank_$n
137			ln -s ../input_ocn/* .
138			if test -x prepare_run ; then ./prepare_run ; fi
139	jmc	1.13	if test "x$MTHo" != x ; then
140			echo " MTH run: mv -f eedata.mth eedata"
141	jmc	1.17	if test -h eedata ; then rm -f eedata ; fi
142	jmc	1.13	mv -f eedata.mth eedata
143			fi
144	jmc	1.1	cd $dir
145	jmc	1.17	n=`expr $n + 1`
146			while [ $n -le $NpOc ] ; do
147			ln -s rank_$rnkO rank_$n
148			n=`expr $n + 1`
149			done
150
151			n=$rnkA
152			echo 'Link files from dir:' input_atm '->' rank_$n
153			mkdir rank_$n
154			cd rank_$n
155			ln -s ../input_atm/* .
156			if test -x prepare_run ; then ./prepare_run ; fi
157	jmc	1.13	if test "x$MTHa" != x ; then
158			echo " MTH run: mv -f eedata.mth eedata"
159	jmc	1.17	if test -h eedata ; then rm -f eedata ; fi
160	jmc	1.13	mv -f eedata.mth eedata
161			fi
162	jmc	1.1	cd $dir
163	jmc	1.17	n=`expr $n + 1`
164			while [ $n -lt $Npr ] ; do
165			ln -s rank_$rnkA rank_$n
166			n=`expr $n + 1`
167			done
168	jmc	1.1
169			fi
170
171			if test $kpr = 3
172			then
173	jmc	1.17	ROOTDIR=$dir
174			# rm -f rank_?/pickup*.ckptA.00?.00?.??ta
175			echo $ROOTDIR
176			tmpfil=TTT.$$
177
178			#--- running on the same node:
179			list='' ; nc=0; xx=`hostname`
180			while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
181			#-- On darwin cluster node (from qrsh session):
182			# JOB_ID=`qstat \| sed -n '3,$ p' \| grep " $USER " \| awk '{print $1}'`
183			# NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
184			# echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
185			#-- On ACES cluster (in PBS batch job):
186			# NODEFILE=$PBS_NODEFILE
187			#--- running on different nodes:
188			# ls -l $NODEFILE
189			# nprc=`cat $NODEFILE \| uniq \| wc -l`
190			# if [ $nprc -ge $Npr ] ; then
191			# list=`cat $NODEFILE \| uniq \| head -$Npr`
192			# else
193			# list=`cat $NODEFILE \| head -$Npr`
194			# fi
195
196			nc=0; nn=0; dd1=cpl ;
197			rm -f pr_group ; touch pr_group
198			for xx in $list
199			do
200			echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
201			nc=`expr $nc + 1`
202			if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
203			nn=1
204			done
205
206			cd $ROOTDIR
207			if test "x$MTH" != x ; then
208			export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
209			if test "x$MTHo" != x ; then
210			echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
211			fi
212			if test "x$MTHa" != x ; then
213			echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
214			fi
215			echo ""
216			fi
217	jmc	1.10	# /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v ./build_cpl/mitgcmuv > std_outp 2>&1
218	jmc	1.17	#- with mpich-1 (on danton, ACES):
219			mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
220			#- with mpich-mx (on beagle):
221			# mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
222			tail -20 std_outp
223			ls -l rank_$rnkO/pickup*.ckptA.001.001.data
224			ls -l rank_$rnkA/pickup*.ckptA.001.001.data
225	jmc	1.1
226			fi
227
228			if test $kpr = 4
229			then
230
231	jmc	1.17	if test -f rank_$rnkO/STDOUT.0000
232	jmc	1.1	then echo '==> check Ocean output:'
233	jmc	1.17	/home/jmc/bin/comp_res rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
234	jmc	1.10	mv -f comp_res.log comp_res.ocn
235			echo ' '
236	jmc	1.17	else echo "No Ocean output file in rank_$rnkO" ; fi
237	jmc	1.1
238	jmc	1.17	if test -f rank_$rnkA/STDOUT.0000
239			then
240	jmc	1.1	echo '==> check Atmos output:'
241	jmc	1.17	/home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
242	jmc	1.10	mv -f comp_res.log comp_res.atm
243	jmc	1.1	echo '==> check Land output:'
244	jmc	1.17	/home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
245	jmc	1.10	mv -f comp_res.log comp_res.land
246	jmc	1.1	echo '==> check thSIce output:'
247	jmc	1.17	/home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
248	jmc	1.10	mv -f comp_res.log comp_res.sice
249			echo ' '
250	jmc	1.17	else echo "No Atmos output file in rank_$rnkA" ; fi
251	jmc	1.1
252			fi
253
254			exit 0