/[MITgcm]/MITgcm/verification/cpl_aim+ocn/run_cpl_test
ViewVC logotype

Diff of /MITgcm/verification/cpl_aim+ocn/run_cpl_test

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.2 by edhill, Wed Oct 27 23:19:43 2004 UTC revision 1.23 by jmc, Mon Oct 26 21:04:35 2015 UTC
# Line 1  Line 1 
1  #! /bin/sh  #! /usr/bin/env bash
2    
3  # $Header$  # $Header$
4  # $Name$  # $Name$
5    
6  Np=3  #- default:
7  if [ $# -lt 1 ]   Npr=3 ; NpOc=1 ;
8  then  #Npr=25; NpOc=12;
9    echo 'Usage:'`basename $0`' step '  MTH=
10    echo ' => test coupled set-up on myrinet cluster'  MTHo=
11    echo 'step = 0 : clean all directories'  MTHa=
12    echo 'step = 1 : compile the 3 executables (cpl,ocn,atm)'  GMKopt='-devel'
13    echo 'step = 2 : copy input files and dir(s)'  #GMKopt='-ieee'
14    echo "step = 3 : run on $Np processor"  #GMKopt=
15    echo 'step = 4 : check the results'  rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
16    
17    #- parse options:
18    if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
19      MTH='-omp' ; shift
20      if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
21      if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
22    fi ; fi
23    #- allows more argument for building step (step=1)
24    chkArg=$# ; if [ $# -gt 1 ]; then chkArg=$1 ; fi
25    if [ $chkArg -ne 1 ]; then
26      echo 'Usage:'`basename $0`' [opt] step [bld-arg]'
27      echo ' => test coupled set-up on linux box (1.cpu)'
28      echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
29      echo ' step = 0 : clean all directories'
30      echo ' step = 1 : compile the 3 executables (cpl,ocn,atm); optional arg:'
31      echo '       -of Optfile_Name : using option-file "Optfile_Name"'
32      echo ' step = 2 : copy input files and dir(s)'
33      echo " step = 3 : run with $Npr mpi processes"
34      echo ' step = 4 : check the results'
35      echo ' step = 5 : remove output files in rank_0,1,2 dir.'
36    exit    exit
37  fi  fi
38  kpr=$1  kpr=$1
39  dir=`pwd`  dir=`pwd`
40    
 #- only works from a 3 nodes PBS bach session ( qsub -I -l nodes=3:p4 )  
 #  on myrinet cluster:  
 xx=`hostname | sed 's/[0-9]/X/g'`  
 if test $xx != 'myrinet-X-XX'  
 then  
   echo ' hostname='`hostname`  
   echo 'for now, works only on myrinet cluster'  
   exit 9  
 fi  
 if test 'dummy'$PBS_NODEFILE = 'dummy'  
 then  
   echo ' $PBS_NODEFILE not defined'  
   echo "for now, works only from a $Np nodes PBS bach session"  
   exit 8  
 else ncpus=`wc -l $PBS_NODEFILE | awk '{print $1}'`  
  if test $ncpus != $Np ; then  
   echo ' Nb of nodes from $PBS_NODEFILE =' $ncpus  
   echo "for now, works only from a $Np nodes PBS bach session"  
   exit 8  
  fi  
 fi  
   
41  #============================================================================  #============================================================================
42    
43  if test $kpr = 0  if test $kpr = 0
44  then  then
45   rm -f genmake_optfile pr_group std_outp   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
46   rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*   rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
47   /bin/rm -r -f rank_0 rank_1 rank_2   /bin/rm -r -f rank_? rank_1? rank_2?
48   if test -f build_cpl/Makefile ; then cd build_cpl ; make CLEAN ; cd .. ; fi   if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
49   if test -f build_ocn/Makefile ; then cd build_ocn ; make CLEAN ; cd .. ; fi   if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
50   if test -f build_atm/Makefile ; then cd build_atm ; make CLEAN ; cd .. ; fi   if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
51    fi
52    if test $kpr = 5
53    then
54     echo 'remove output files in rank_0,1,2 dir.'
55     rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
56     if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
57     if test -d rank_$rnkO
58     then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
59     if test -d rank_$rnkA
60     then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
61  fi  fi
62    
63  if test $kpr = 1  if test $kpr = 1
64  then  then
65    
66   # cp -p ../../tools/build_options/linux_ia32_ifc+mpi_cg01 genmake_optfile  #- choice of the optfile:
67   cp optfile genmake_optfile  #  default: take a local one in dir verification with sufix '+mpi'
68     nbOpF=`ls ../linux_* | grep '+mpi' 2> /dev/null | wc -l`
69    #  or take the one given as argument:
70     if [ $# -ge 3 ]; then
71       if test $2 = '-of' -a -f $3 ; then nbOpF=-1 ; OPTFILE=$3 ; fi
72     fi
73     if test $nbOpF = 1 ; then
74       OPTFILE=`ls ../linux_* | grep '+mpi'`
75     elif [ $nbOpF -ge 2 ] ; then
76       echo "pick the 1rst of these ( $nbOpF ) optfiles:"
77       ls ../linux_* | grep '+mpi'
78       OPTFILE=`ls ../linux_* | grep '+mpi' | head -1`
79     elif [ $nbOpF -ne -1 ] ; then
80       echo "Pb in finding optfile: found $nbOpF :"
81       ls ../linux_* | grep '+mpi' ; exit
82     fi
83     zz=`grep '^ *FC=' $OPTFILE | tail -1`
84     echo " Using optfile: $OPTFILE  (compiler: $zz) $MTH"
85     zz=`echo $OPTFILE | grep -c '^\/'`
86     if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
87    #---
88   echo '==== compile coupler:'   echo '==== compile coupler:'
89   cd build_cpl   cd build_cpl
90   echo ' --- genmake2 (cpl):'   echo ' --- genmake2 (cpl):'
91   ../../../tools/genmake2 -of ../genmake_optfile -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt >  TTT.genmake.$$ 2>&1
92   tail -5 TTT.genmake.$$   RetVal=$? ; tail -5 TTT.genmake.$$
93     if test "x$RetVal" != x0 ; then
94       echo "Error in genmake2 (cpl)" ; exit 11
95     fi
96   echo ' --- make depend (cpl):'   echo ' --- make depend (cpl):'
97   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$ 2>&1
98   tail -5 TTT.mkdepend.$$   RetVal=$? ; tail -5 TTT.mkdepend.$$
99   echo ' --- make (cpl):'   if test "x$RetVal" != x0 ; then
100   make > TTT.make.$$ 2>&1     echo "Error in mkdepend (cpl)" ; exit 12
101   tail -10 TTT.make.$$   fi
102     echo ' --- make (cpl):' ; touch TTT.make.$$
103    #do_make_syntax.sh obj > TTT.make.$$ 2>&1
104     make >> TTT.make.$$ 2>&1
105     RetVal=$? ; tail -10 TTT.make.$$
106     if test "x$RetVal" != x0 ; then
107       echo "Error in make     (cpl)" ; exit 13
108     fi
109   echo ' ' ; cd $dir   echo ' ' ; cd $dir
110    
111   echo '==== compile OGCM:'   echo '==== compile OGCM:'
112   cd build_ocn   cd build_ocn
113   echo ' --- genmake2 (ocn):'   echo ' --- genmake2 (ocn):'
114   ../../../tools/genmake2 -of ../genmake_optfile -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt >  TTT.genmake.$$ 2>&1
115   tail -5 TTT.genmake.$$   RetVal=$? ; tail -5 TTT.genmake.$$
116     if test "x$RetVal" != x0 ; then
117       echo "Error in genmake2 (ocn)" ; exit 21
118     fi
119   echo ' --- make depend (ocn):'   echo ' --- make depend (ocn):'
120   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$ 2>&1
121   tail -10 TTT.mkdepend.$$   RetVal=$? ; tail -10 TTT.mkdepend.$$
122   echo ' --- make (ocn):'   if test "x$RetVal" != x0 ; then
123   make > TTT.make.$$ 2>&1     echo "Error in mkdepend (ocn)" ; exit 22
124   tail -10 TTT.make.$$   fi
125     echo ' --- make (ocn):' ; touch TTT.make.$$
126    #do_make_syntax.sh obj > TTT.make.$$ 2>&1
127     make >> TTT.make.$$ 2>&1
128     RetVal=$? ; tail -10 TTT.make.$$
129     if test "x$RetVal" != x0 ; then
130       echo "Error in make     (ocn)" ; exit 23
131     fi
132   echo ' ' ; cd $dir   echo ' ' ; cd $dir
133    
134   echo '==== compile AGCM:'   echo '==== compile AGCM:'
135   cd build_atm   cd build_atm
136   echo ' --- genmake2 (atm):'   echo ' --- genmake2 (atm):'
137   ../../../tools/genmake2 -of ../genmake_optfile -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt >  TTT.genmake.$$ 2>&1
138   tail -5 TTT.genmake.$$   RetVal=$? ; tail -5 TTT.genmake.$$
139     if test "x$RetVal" != x0 ; then
140       echo "Error in genmake2 (atm)" ; exit 31
141     fi
142   echo ' --- make depend (atm):'   echo ' --- make depend (atm):'
143   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$ 2>&1
144   tail -10 TTT.mkdepend.$$   RetVal=$? ; tail -10 TTT.mkdepend.$$
145   echo ' --- make (atm):'   if test "x$RetVal" != x0 ; then
146   make > TTT.make.$$ 2>&1     echo "Error in mkdepend (atm)" ; exit 32
147   tail -10 TTT.make.$$   fi
148     echo ' --- make (atm):' ; touch TTT.make.$$
149    #do_make_syntax.sh obj > TTT.make.$$ 2>&1
150     make >> TTT.make.$$ 2>&1
151     RetVal=$? ; tail -10 TTT.make.$$
152     if test "x$RetVal" != x0 ; then
153       echo "Error in make     (atm)" ; exit 33
154     fi
155   echo ' ' ; cd $dir   echo ' ' ; cd $dir
156    
157   ls -l build_???/mitgcmuv   ls -l build_???/mitgcmuv
# Line 102  fi Line 160  fi
160    
161  if test $kpr = 2  if test $kpr = 2
162  then  then
163    cp -p -r input_cpl rank_0    echo 'rm dir:' rank_? rank_1? rank_2?
164    cp -p -r input_ocn rank_1    /bin/rm -r -f rank_? rank_1? rank_2?
165    cp -p -r input_atm rank_2    echo 'Link files from dir:' input_cpl '->' rank_0
166      mkdir rank_0
167    cd rank_1    ( cd rank_0 ; ln -s ../input_cpl/* . )
168    ./IMPORT_INPUT_FILES  
169      n=$rnkO
170      echo 'Link files from dir:' input_ocn '->' rank_$n
171      mkdir rank_$n
172      cd rank_$n
173      ln -s ../input_ocn/* .
174      if test -x prepare_run ; then ./prepare_run ; fi
175      if test "x$MTHo" != x ; then
176        echo " MTH run: mv -f eedata.mth eedata"
177        if test -h eedata ; then rm -f eedata ; fi
178        mv -f eedata.mth eedata
179      fi
180    cd $dir    cd $dir
181      n=`expr $n + 1`
182    cd rank_2    while [ $n -le $NpOc ] ; do
183    ./SHARE_OCN_GRID '../rank_1'      ln -s rank_$rnkO rank_$n
184        n=`expr $n + 1`
185      done
186    
187      n=$rnkA
188      echo 'Link files from dir:' input_atm '->' rank_$n
189      mkdir rank_$n
190      cd rank_$n
191      ln -s ../input_atm/* .
192      if test -x prepare_run ; then ./prepare_run ; fi
193      if test "x$MTHa" != x ; then
194        echo " MTH run: mv -f eedata.mth eedata"
195        if test -h eedata ; then rm -f eedata ; fi
196        mv -f eedata.mth eedata
197      fi
198    cd $dir    cd $dir
199      n=`expr $n + 1`
200    mkdir cpl ocn atm    while [ $n -lt $Npr ] ; do
201    if test -f build_cpl/mitgcmuv ; then cp -p build_cpl/mitgcmuv cpl      ln -s rank_$rnkA rank_$n
202    else echo 'file: build_cpl/mitgcmuv not found => stop'; exit; fi      n=`expr $n + 1`
203    if test -f build_ocn/mitgcmuv ; then cp -p build_ocn/mitgcmuv ocn    done
   else echo 'file: build_ocn/mitgcmuv not found => stop'; exit; fi  
   if test -f build_atm/mitgcmuv ; then cp -p build_atm/mitgcmuv atm  
   else echo 'file: build_atm/mitgcmuv not found => stop'; exit; fi  
204    
205  fi  fi
206    
207  if test $kpr = 3  if test $kpr = 3
208  then  then
209   ROOTDIR=$dir    ROOTDIR=$dir
210   rm -f rank_?/pickup*.ckptA.00?.00?.??ta  # rm -f rank_?/pickup*.ckptA.00?.00?.??ta
211   echo $ROOTDIR    echo $ROOTDIR
212   tmpfil=TTT.$$    tmpfil=TTT.$$
213    
214   echo $PBS_NODEFILE  #--- running on the same node:
215   list=`cat $PBS_NODEFILE`    list='' ; nc=0; xx=`hostname`
216   rm -f pr_group ; touch pr_group    while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
217   nc=0 ; dd1=cpl ;  #-- On darwin cluster node (from qrsh session):
218   for xx in $list  # JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
219   do  # NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
220    echo $xx $nc $dir/build_$dd1/mitgcmuv >> pr_group  # echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
221    if test $nc = 0 ; then dd1=ocn ; else dd1=atm ; fi  #-- On ACES cluster (in PBS batch job):
222    nc=1  # NODEFILE=$PBS_NODEFILE
223   done  #--- running on different nodes:
224    # ls -l $NODEFILE
225   cd $ROOTDIR  # nprc=`cat $NODEFILE | uniq | wc -l`
226   /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1  # if [ $nprc -ge $Npr ] ; then
227   tail -20 std_outp  #   list=`cat $NODEFILE | uniq | head -$Npr`
228   ls -l rank_?/pickup*.ckptA.001.001.data  # else
229    #   list=`cat $NODEFILE | head -$Npr`
230    # fi
231    
232      nc=0; nn=0; dd1=cpl ;
233      rm -f pr_group ; touch pr_group
234      for xx in $list
235      do
236        echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
237        nc=`expr $nc + 1`
238        if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
239        nn=1
240      done
241      NpAt=`expr $Npr - 1 - $NpOc`
242      RunOpt="-np 1 ./build_cpl/mitgcmuv"
243      RunOpt="$RunOpt : -np $NpOc ./build_ocn/mitgcmuv"
244      RunOpt="$RunOpt : -np $NpAt ./build_atm/mitgcmuv"
245    
246      cd $ROOTDIR
247      if test "x$MTH" != x ; then
248        export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
249        if test "x$MTHo" != x ; then
250          echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
251        fi
252        if test "x$MTHa" != x ; then
253          echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
254        fi
255        echo ""
256      fi
257      mpich=`which mpirun`
258      echo $mpich | grep 'mpich-mx' > /dev/null 2>&1
259      mpichmx=$?
260      echo $mpich | grep 'mpich-1' > /dev/null 2>&1
261      mpich1=$?
262      echo $mpich | grep 'mpich2' > /dev/null 2>&1
263      mpich2=$?
264      echo $mpich | grep 'openmpi' > /dev/null 2>&1
265      opnmpi=$?
266      if test $mpich1 == 0 ; then
267      # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1
268      #- with mpich-1 (on danton, old aces: ao, geo, itrda):
269        echo "execute 'mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv' :"
270        mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
271      elif test $mpichmx == 0 ; then
272      #- with mpich-mx (on beagle):
273        echo "execute 'mpirun -pg pr_group -v ./build_cpl/mitgcmuv' :"
274        mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
275      elif test $mpich2 == 0 -o $opnmpi == 0 ; then
276      #- with Hydra mpich2 (on baudelaire) or with openmpi:
277        echo "execute 'mpirun $RunOpt' :"
278        mpirun $RunOpt  > std_outp 2>&1
279      else
280      #- new mpich (mpich2) installation often just put in "mpich" dir
281        echo "execute 'mpirun $RunOpt' :"
282        mpirun $RunOpt  > std_outp 2>&1
283      fi
284      tail -20 std_outp
285      ls -l rank_$rnkO/pickup*.ckptA.001.001.data
286      ls -l rank_$rnkA/pickup*.ckptA.001.001.data
287    
288  fi  fi
289    
290  if test $kpr = 4  if test $kpr = 4
291  then  then
292     CompRes="$HOME/bin/comp_res"
293    
294   if test -f rank_1/STDOUT.0000   if test -f rank_$rnkO/STDOUT.0000
295   then echo '==> check Ocean output:'   then echo '==> check Ocean output:'
296      /home/jmc/bin/comp_res rank_1/STDOUT.0000 results/ocnSTDOUT.0000      $CompRes rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
297     echo ' '      mv -f comp_res.log comp_res.ocn
298   else echo 'No Ocean output file in rank_1' ; fi      echo ' '
299     else echo "No Ocean output file in rank_$rnkO" ; fi
300    
301   if test -f rank_2/STDOUT.0000   if test -f rank_$rnkA/STDOUT.0000
302   then   then
303      echo '==> check Atmos output:'      echo '==> check Atmos output:'
304      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
305        mv -f comp_res.log comp_res.atm
306      echo '==> check Land output:'      echo '==> check Land output:'
307      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 L      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
308        mv -f comp_res.log comp_res.land
309      echo '==> check thSIce output:'      echo '==> check thSIce output:'
310      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 I      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
311     echo ' '      mv -f comp_res.log comp_res.sice
312   else echo 'No Atmos output file in rank_1' ; fi      echo ' '
313     else echo "No Atmos output file in rank_$rnkA" ; fi
314    
315  fi  fi
316    
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.23
  ViewVC Help
Powered by ViewVC 1.1.22