/[MITgcm]/MITgcm/verification/cpl_aim+ocn/run_cpl_test
ViewVC logotype

Diff of /MITgcm/verification/cpl_aim+ocn/run_cpl_test

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.11 by jmc, Tue Nov 27 00:54:58 2007 UTC revision 1.18 by jmc, Tue Nov 8 22:56:52 2011 UTC
# Line 3  Line 3 
3  # $Header$  # $Header$
4  # $Name$  # $Name$
5    
6  Np=3  #- default:
7  if [ $# -lt 1 ]   Npr=3 ; NpOc=1 ;
8    #Npr=25; NpOc=12;
9    MTH=
10    MTHo=
11    MTHa=
12    #GMKopt='-devel'
13    GMKopt='-ieee'
14    #GMKopt=
15    rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
16    
17    #- parse options:
18    if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
19      MTH='-omp' ; shift
20      if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
21      if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
22    fi ; fi
23    if [ $# -ne 1 ]
24  then  then
25    echo 'Usage:'`basename $0`' step '    echo 'Usage:'`basename $0`' [opt] step'
26    echo ' => test coupled set-up on linux box (1.cpu)'    echo ' => test coupled set-up on linux box (1.cpu)'
27    echo 'step = 0 : clean all directories'    echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
28    echo 'step = 1 : compile the 3 executables (cpl,ocn,atm)'    echo ' step = 0 : clean all directories'
29    echo 'step = 2 : copy input files and dir(s)'    echo ' step = 1 : compile the 3 executables (cpl,ocn,atm)'
30    echo "step = 3 : run with $Np mpi processes"    echo ' step = 2 : copy input files and dir(s)'
31    echo 'step = 4 : check the results'    echo " step = 3 : run with $Npr mpi processes"
32    echo 'step = 5 : remove output files in rank_0,1,2 dir.'    echo ' step = 4 : check the results'
33      echo ' step = 5 : remove output files in rank_0,1,2 dir.'
34    exit    exit
35  fi  fi
36  kpr=$1  kpr=$1
# Line 25  if test $kpr = 0 Line 42  if test $kpr = 0
42  then  then
43   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
44   rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*   rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
45   /bin/rm -r -f rank_0 rank_1 rank_2   /bin/rm -r -f rank_? rank_1? rank_2?
46   if test -f build_cpl/Makefile ; then cd build_cpl ; make CLEAN ; cd .. ; fi   if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
47   if test -f build_ocn/Makefile ; then cd build_ocn ; make CLEAN ; cd .. ; fi   if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
48   if test -f build_atm/Makefile ; then cd build_atm ; make CLEAN ; cd .. ; fi   if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
49  fi  fi
50  if test $kpr = 5  if test $kpr = 5
51  then  then
52   echo 'remove output files in rank_0,1,2 dir.'   echo 'remove output files in rank_0,1,2 dir.'
53   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
54   if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi   if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
55   if test -d rank_1   if test -d rank_$rnkO
56   then cd rank_1 ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi   then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
57   if test -d rank_2   if test -d rank_$rnkA
58   then cd rank_2 ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi   then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
59  fi  fi
60    
61  if test $kpr = 1  if test $kpr = 1
62  then  then
63    
64  #- choice of the optfile: take a local one in dir verification with sufix '.mpi'  #- choice of the optfile: take a local one in dir verification with sufix '+mpi'
65   nbOpF=`ls ../linux_*.mpi 2> /dev/null | wc -l`   nbOpF=`ls ../linux_* | grep '+mpi' 2> /dev/null | wc -l`
66   if test $nbOpF = 1   if test $nbOpF = 1 ; then
67   then     OPTFILE=`ls ../linux_* | grep '+mpi'`
    OPTFILE=`ls ../linux_*.mpi`  
68     zz=`grep '^FC=' $OPTFILE`     zz=`grep '^FC=' $OPTFILE`
69     echo " Using optfile: $OPTFILE  (compiler=$zz)"     echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"
70   else echo 'Pb in finding optfile' ; exit;   elif [ $nbOpF -ge 2 ] ; then
71       echo "pick the 1rst of these ( $nbOpF ) optfiles:"
72       ls ../linux_* | grep '+mpi'
73       OPTFILE=`ls ../linux_* | grep '+mpi' | head -1`
74       zz=`grep '^FC=' $OPTFILE`
75       echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"
76     else
77       echo "Pb in finding optfile: found $nbOpF :"
78       ls ../linux_* | grep '+mpi' ; exit
79   fi   fi
80   zz=`echo $OPTFILE | grep -c '^\/'`   zz=`echo $OPTFILE | grep -c '^\/'`
81   if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi   if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
# Line 59  then Line 83  then
83   echo '==== compile coupler:'   echo '==== compile coupler:'
84   cd build_cpl   cd build_cpl
85   echo ' --- genmake2 (cpl):'   echo ' --- genmake2 (cpl):'
86   ../../../tools/genmake2 -of $OPTFILE -mpi -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt >  TTT.genmake.$$
87   tail -5 TTT.genmake.$$   tail -5 TTT.genmake.$$
88   echo ' --- make depend (cpl):'   echo ' --- make depend (cpl):'
89   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$
# Line 72  then Line 96  then
96   echo '==== compile OGCM:'   echo '==== compile OGCM:'
97   cd build_ocn   cd build_ocn
98   echo ' --- genmake2 (ocn):'   echo ' --- genmake2 (ocn):'
99   ../../../tools/genmake2 -of $OPTFILE -mpi -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt >  TTT.genmake.$$
100   tail -5 TTT.genmake.$$   tail -5 TTT.genmake.$$
101   echo ' --- make depend (ocn):'   echo ' --- make depend (ocn):'
102   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$
# Line 85  then Line 109  then
109   echo '==== compile AGCM:'   echo '==== compile AGCM:'
110   cd build_atm   cd build_atm
111   echo ' --- genmake2 (atm):'   echo ' --- genmake2 (atm):'
112   ../../../tools/genmake2 -of $OPTFILE -mpi -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt >  TTT.genmake.$$
113   tail -5 TTT.genmake.$$   tail -5 TTT.genmake.$$
114   echo ' --- make depend (atm):'   echo ' --- make depend (atm):'
115   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$
# Line 101  fi Line 125  fi
125    
126  if test $kpr = 2  if test $kpr = 2
127  then  then
128    /bin/rm -r -f rank_0 rank_1 rank_2    echo 'rm dir:' rank_? rank_1? rank_2?
129    echo 'CP dir:' input_cpl '->' rank_0    /bin/rm -r -f rank_? rank_1? rank_2?
130    cp -p -r input_cpl rank_0    echo 'Link files from dir:' input_cpl '->' rank_0
131      mkdir rank_0
132    echo 'CP dir:' input_ocn '->' rank_1    ( cd rank_0 ; ln -s ../input_cpl/* . )
133    cp -p -r input_ocn rank_1  
134    cd rank_1    n=$rnkO
135    ./prepare_run    echo 'Link files from dir:' input_ocn '->' rank_$n
136      mkdir rank_$n
137      cd rank_$n
138      ln -s ../input_ocn/* .
139      if test -x prepare_run ; then ./prepare_run ; fi
140      if test "x$MTHo" != x ; then
141        echo " MTH run: mv -f eedata.mth eedata"
142        if test -h eedata ; then rm -f eedata ; fi
143        mv -f eedata.mth eedata
144      fi
145    cd $dir    cd $dir
146      n=`expr $n + 1`
147    echo 'CP dir:' input_atm '->' rank_2    while [ $n -le $NpOc ] ; do
148    cp -p -r input_atm rank_2      ln -s rank_$rnkO rank_$n
149    cd rank_2      n=`expr $n + 1`
150    ./prepare_run    done
151    
152      n=$rnkA
153      echo 'Link files from dir:' input_atm '->' rank_$n
154      mkdir rank_$n
155      cd rank_$n
156      ln -s ../input_atm/* .
157      if test -x prepare_run ; then ./prepare_run ; fi
158      if test "x$MTHa" != x ; then
159        echo " MTH run: mv -f eedata.mth eedata"
160        if test -h eedata ; then rm -f eedata ; fi
161        mv -f eedata.mth eedata
162      fi
163    cd $dir    cd $dir
164      n=`expr $n + 1`
165      while [ $n -lt $Npr ] ; do
166        ln -s rank_$rnkA rank_$n
167        n=`expr $n + 1`
168      done
169    
170  fi  fi
171    
172  if test $kpr = 3  if test $kpr = 3
173  then  then
174   ROOTDIR=$dir    ROOTDIR=$dir
175   rm -f rank_?/pickup*.ckptA.00?.00?.??ta  # rm -f rank_?/pickup*.ckptA.00?.00?.??ta
176   echo $ROOTDIR    echo $ROOTDIR
177   tmpfil=TTT.$$    tmpfil=TTT.$$
178    
179   rm -f pr_group ; touch pr_group  #--- running on the same node:
180   nc=0; xx=`hostname`    list='' ; nc=0; xx=`hostname`
181   for dd1 in cpl ocn atm    while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
182   do  #-- On darwin cluster node (from qrsh session):
183    echo $xx $nc $dir/build_$dd1/mitgcmuv >> pr_group  # JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
184    nc=1  # NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
185   done  # echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
186    #-- On ACES cluster (in PBS batch job):
187   cd $ROOTDIR  # NODEFILE=$PBS_NODEFILE
188  # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1  #--- running on different nodes:
189   mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1  # ls -l $NODEFILE
190   tail -20 std_outp  # nprc=`cat $NODEFILE | uniq | wc -l`
191   ls -l rank_?/pickup*.ckptA.001.001.data  # if [ $nprc -ge $Npr ] ; then
192    #   list=`cat $NODEFILE | uniq | head -$Npr`
193    # else
194    #   list=`cat $NODEFILE | head -$Npr`
195    # fi
196    
197      nc=0; nn=0; dd1=cpl ;
198      rm -f pr_group ; touch pr_group
199      for xx in $list
200      do
201        echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
202        nc=`expr $nc + 1`
203        if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
204        nn=1
205      done
206      NpAt=`expr $Npr - 1 - $NpOc`
207      RunOpt="-np 1 ./build_cpl/mitgcmuv"
208      RunOpt="$RunOpt : -np $NpOc ./build_ocn/mitgcmuv"
209      RunOpt="$RunOpt : -np $NpAt ./build_atm/mitgcmuv"
210    
211      cd $ROOTDIR
212      if test "x$MTH" != x ; then
213        export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
214        if test "x$MTHo" != x ; then
215          echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
216        fi
217        if test "x$MTHa" != x ; then
218          echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
219        fi
220        echo ""
221      fi
222      mpich=`which mpirun`
223      echo $mpich | grep 'mpich-mx' > /dev/null 2>&1
224      mpichmx=$?
225      echo $mpich | grep 'mpich2' > /dev/null 2>&1
226      mpich2=$?
227      if test $mpich2 == 0 ; then
228      #- with Hydra mpich2 (on baudelaire):
229        echo "execute 'mpirun $RunOpt' :"
230        mpirun $RunOpt  > std_outp 2>&1
231      elif test $mpichmx == 0 ; then
232      #- with mpich-mx (on beagle):
233        echo "execute 'mpirun -pg pr_group -v ./build_cpl/mitgcmuv' :"
234        mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
235      else
236      # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1
237      #- with mpich-1 (on danton, ACES):
238        echo "execute 'mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv' :"
239        mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
240      fi
241      tail -20 std_outp
242      ls -l rank_$rnkO/pickup*.ckptA.001.001.data
243      ls -l rank_$rnkA/pickup*.ckptA.001.001.data
244    
245  fi  fi
246    
247  if test $kpr = 4  if test $kpr = 4
248  then  then
249     CompRes="$HOME/bin/comp_res"
250    
251   if test -f rank_1/STDOUT.0000   if test -f rank_$rnkO/STDOUT.0000
252   then echo '==> check Ocean output:'   then echo '==> check Ocean output:'
253      /home/jmc/bin/comp_res rank_1/STDOUT.0000 results/ocnSTDOUT.0000      $CompRes rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
254      mv -f comp_res.log comp_res.ocn      mv -f comp_res.log comp_res.ocn
255      echo ' '      echo ' '
256   else echo 'No Ocean output file in rank_1' ; fi   else echo "No Ocean output file in rank_$rnkO" ; fi
257    
258   if test -f rank_2/STDOUT.0000   if test -f rank_$rnkA/STDOUT.0000
259   then   then
260      echo '==> check Atmos output:'      echo '==> check Atmos output:'
261      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
262      mv -f comp_res.log comp_res.atm      mv -f comp_res.log comp_res.atm
263      echo '==> check Land output:'      echo '==> check Land output:'
264      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 L      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
265      mv -f comp_res.log comp_res.land      mv -f comp_res.log comp_res.land
266      echo '==> check thSIce output:'      echo '==> check thSIce output:'
267      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 I      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
268      mv -f comp_res.log comp_res.sice      mv -f comp_res.log comp_res.sice
269      echo ' '      echo ' '
270   else echo 'No Atmos output file in rank_1' ; fi   else echo "No Atmos output file in rank_$rnkA" ; fi
271    
272  fi  fi
273    

Legend:
Removed from v.1.11  
changed lines
  Added in v.1.18

  ViewVC Help
Powered by ViewVC 1.1.22