/[MITgcm]/MITgcm/verification/cpl_aim+ocn/run_cpl_test
ViewVC logotype

Diff of /MITgcm/verification/cpl_aim+ocn/run_cpl_test

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.13 by jmc, Sat Dec 26 01:59:17 2009 UTC revision 1.22 by jmc, Sat Jul 19 22:54:20 2014 UTC
# Line 1  Line 1 
1  #! /bin/sh  #! /usr/bin/env bash
2    
3  # $Header$  # $Header$
4  # $Name$  # $Name$
5    
6  Np=3  #- default:
7     Npr=3 ; NpOc=1 ;
8    #Npr=25; NpOc=12;
9  MTH=  MTH=
10  MTHo=  MTHo=
11  MTHa=  MTHa=
12  if [ $# -lt 1 ]  GMKopt='-devel'
13  then  #GMKopt='-ieee'
14    echo 'Usage:'`basename $0`' step [-mth]'  #GMKopt=
15    rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
16    
17    #- parse options:
18    if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
19      MTH='-omp' ; shift
20      if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
21      if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
22    fi ; fi
23    #- allows more argument for building step (step=1)
24    chkArg=$# ; if [ $# -gt 1 ]; then chkArg=$1 ; fi
25    if [ $chkArg -ne 1 ]; then
26      echo 'Usage:'`basename $0`' [opt] step [bld-arg]'
27    echo ' => test coupled set-up on linux box (1.cpu)'    echo ' => test coupled set-up on linux box (1.cpu)'
28    echo 'step = 0 : clean all directories'    echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
29    echo 'step = 1 : compile the 3 executables (cpl,ocn,atm)'    echo ' step = 0 : clean all directories'
30    echo 'step = 2 : copy input files and dir(s)'    echo ' step = 1 : compile the 3 executables (cpl,ocn,atm); optional arg:'
31    echo "step = 3 : run with $Np mpi processes"    echo '       -of Optfile_Name : using option-file "Optfile_Name"'
32    echo 'step = 4 : check the results'    echo ' step = 2 : copy input files and dir(s)'
33    echo 'step = 5 : remove output files in rank_0,1,2 dir.'    echo " step = 3 : run with $Npr mpi processes"
34    echo '  -mth   : compile and run 2 threads for ocn & atm'    echo ' step = 4 : check the results'
35      echo ' step = 5 : remove output files in rank_0,1,2 dir.'
36    exit    exit
 elif [ $# -eq 2 ] ; then  
   if test $2 = '-mth' ; then  
     MTH='-omp'  
     if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi  
     if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi  
   fi  
37  fi  fi
38  kpr=$1  kpr=$1
39  dir=`pwd`  dir=`pwd`
# Line 35  if test $kpr = 0 Line 44  if test $kpr = 0
44  then  then
45   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
46   rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*   rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
47   /bin/rm -r -f rank_0 rank_1 rank_2   /bin/rm -r -f rank_? rank_1? rank_2?
48   if test -f build_cpl/Makefile ; then cd build_cpl ; make CLEAN ; cd .. ; fi   if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
49   if test -f build_ocn/Makefile ; then cd build_ocn ; make CLEAN ; cd .. ; fi   if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
50   if test -f build_atm/Makefile ; then cd build_atm ; make CLEAN ; cd .. ; fi   if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
51  fi  fi
52  if test $kpr = 5  if test $kpr = 5
53  then  then
54   echo 'remove output files in rank_0,1,2 dir.'   echo 'remove output files in rank_0,1,2 dir.'
55   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}   rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
56   if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi   if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
57   if test -d rank_1   if test -d rank_$rnkO
58   then cd rank_1 ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi   then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
59   if test -d rank_2   if test -d rank_$rnkA
60   then cd rank_2 ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi   then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
61  fi  fi
62    
63  if test $kpr = 1  if test $kpr = 1
64  then  then
65    
66  #- choice of the optfile: take a local one in dir verification with sufix '+mpi'  #- choice of the optfile:
67   nbOpF=`ls ../linux_*+mpi 2> /dev/null | wc -l`  #  default: take a local one in dir verification with sufix '+mpi'
68   if test $nbOpF = 1   nbOpF=`ls ../linux_* | grep '+mpi' 2> /dev/null | wc -l`
69   then  #  or take the one given as argument:
70     OPTFILE=`ls ../linux_*+mpi`   if [ $# -ge 3 ]; then
71     zz=`grep '^FC=' $OPTFILE`     if test $2 = '-of' -a -f $3 ; then nbOpF=-1 ; OPTFILE=$3 ; fi
    echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"  
  else echo 'Pb in finding optfile' ; exit;  
72   fi   fi
73     if test $nbOpF = 1 ; then
74       OPTFILE=`ls ../linux_* | grep '+mpi'`
75     elif [ $nbOpF -ge 2 ] ; then
76       echo "pick the 1rst of these ( $nbOpF ) optfiles:"
77       ls ../linux_* | grep '+mpi'
78       OPTFILE=`ls ../linux_* | grep '+mpi' | head -1`
79     elif [ $nbOpF -ne -1 ] ; then
80       echo "Pb in finding optfile: found $nbOpF :"
81       ls ../linux_* | grep '+mpi' ; exit
82     fi
83     zz=`grep '^ *FC=' $OPTFILE | tail -1`
84     echo " Using optfile: $OPTFILE  (compiler: $zz) $MTH"
85   zz=`echo $OPTFILE | grep -c '^\/'`   zz=`echo $OPTFILE | grep -c '^\/'`
86   if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi   if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
87  #---  #---
88   echo '==== compile coupler:'   echo '==== compile coupler:'
89   cd build_cpl   cd build_cpl
90   echo ' --- genmake2 (cpl):'   echo ' --- genmake2 (cpl):'
91   ../../../tools/genmake2 -of $OPTFILE -mpi -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt >  TTT.genmake.$$ 2>&1
92   tail -5 TTT.genmake.$$   tail -5 TTT.genmake.$$
93   echo ' --- make depend (cpl):'   echo ' --- make depend (cpl):'
94   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$ 2>&1
95   tail -5 TTT.mkdepend.$$   tail -5 TTT.mkdepend.$$
96   echo ' --- make (cpl):'   echo ' --- make (cpl):' ; touch TTT.make.$$
97   make > TTT.make.$$ 2>&1  #do_make_syntax.sh obj > TTT.make.$$ 2>&1
98     make >> TTT.make.$$ 2>&1
99   tail -10 TTT.make.$$   tail -10 TTT.make.$$
100   echo ' ' ; cd $dir   echo ' ' ; cd $dir
101    
102   echo '==== compile OGCM:'   echo '==== compile OGCM:'
103   cd build_ocn   cd build_ocn
104   echo ' --- genmake2 (ocn):'   echo ' --- genmake2 (ocn):'
105   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt >  TTT.genmake.$$ 2>&1
106   tail -5 TTT.genmake.$$   tail -5 TTT.genmake.$$
107   echo ' --- make depend (ocn):'   echo ' --- make depend (ocn):'
108   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$ 2>&1
109   tail -10 TTT.mkdepend.$$   tail -10 TTT.mkdepend.$$
110   echo ' --- make (ocn):'   echo ' --- make (ocn):' ; touch TTT.make.$$
111   make > TTT.make.$$ 2>&1  #do_make_syntax.sh obj > TTT.make.$$ 2>&1
112     make >> TTT.make.$$ 2>&1
113   tail -10 TTT.make.$$   tail -10 TTT.make.$$
114   echo ' ' ; cd $dir   echo ' ' ; cd $dir
115    
116   echo '==== compile AGCM:'   echo '==== compile AGCM:'
117   cd build_atm   cd build_atm
118   echo ' --- genmake2 (atm):'   echo ' --- genmake2 (atm):'
119   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa -ieee >  TTT.genmake.$$   ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt >  TTT.genmake.$$ 2>&1
120   tail -5 TTT.genmake.$$   tail -5 TTT.genmake.$$
121   echo ' --- make depend (atm):'   echo ' --- make depend (atm):'
122   make depend > TTT.mkdepend.$$   make depend > TTT.mkdepend.$$ 2>&1
123   tail -10 TTT.mkdepend.$$   tail -10 TTT.mkdepend.$$
124   echo ' --- make (atm):'   echo ' --- make (atm):' ; touch TTT.make.$$
125   make > TTT.make.$$ 2>&1  #do_make_syntax.sh obj > TTT.make.$$ 2>&1
126     make >> TTT.make.$$ 2>&1
127   tail -10 TTT.make.$$   tail -10 TTT.make.$$
128   echo ' ' ; cd $dir   echo ' ' ; cd $dir
129    
# Line 111  fi Line 133  fi
133    
134  if test $kpr = 2  if test $kpr = 2
135  then  then
136    /bin/rm -r -f rank_0 rank_1 rank_2    echo 'rm dir:' rank_? rank_1? rank_2?
137    echo 'CP dir:' input_cpl '->' rank_0    /bin/rm -r -f rank_? rank_1? rank_2?
138    cp -p -r input_cpl rank_0    echo 'Link files from dir:' input_cpl '->' rank_0
139      mkdir rank_0
140    echo 'CP dir:' input_ocn '->' rank_1    ( cd rank_0 ; ln -s ../input_cpl/* . )
141    cp -p -r input_ocn rank_1  
142    cd rank_1    n=$rnkO
143    ./prepare_run    echo 'Link files from dir:' input_ocn '->' rank_$n
144      mkdir rank_$n
145      cd rank_$n
146      ln -s ../input_ocn/* .
147      if test -x prepare_run ; then ./prepare_run ; fi
148    if test "x$MTHo" != x ; then    if test "x$MTHo" != x ; then
149      echo " MTH run: mv -f eedata.mth eedata"      echo " MTH run: mv -f eedata.mth eedata"
150        if test -h eedata ; then rm -f eedata ; fi
151      mv -f eedata.mth eedata      mv -f eedata.mth eedata
152    fi    fi
153    cd $dir    cd $dir
154      n=`expr $n + 1`
155    echo 'CP dir:' input_atm '->' rank_2    while [ $n -le $NpOc ] ; do
156    cp -p -r input_atm rank_2      ln -s rank_$rnkO rank_$n
157    cd rank_2      n=`expr $n + 1`
158    ./prepare_run    done
159    
160      n=$rnkA
161      echo 'Link files from dir:' input_atm '->' rank_$n
162      mkdir rank_$n
163      cd rank_$n
164      ln -s ../input_atm/* .
165      if test -x prepare_run ; then ./prepare_run ; fi
166    if test "x$MTHa" != x ; then    if test "x$MTHa" != x ; then
167      echo " MTH run: mv -f eedata.mth eedata"      echo " MTH run: mv -f eedata.mth eedata"
168        if test -h eedata ; then rm -f eedata ; fi
169      mv -f eedata.mth eedata      mv -f eedata.mth eedata
170    fi    fi
171    cd $dir    cd $dir
172      n=`expr $n + 1`
173      while [ $n -lt $Npr ] ; do
174        ln -s rank_$rnkA rank_$n
175        n=`expr $n + 1`
176      done
177    
178  fi  fi
179    
180  if test $kpr = 3  if test $kpr = 3
181  then  then
182   ROOTDIR=$dir    ROOTDIR=$dir
183   rm -f rank_?/pickup*.ckptA.00?.00?.??ta  # rm -f rank_?/pickup*.ckptA.00?.00?.??ta
184   echo $ROOTDIR    echo $ROOTDIR
185   tmpfil=TTT.$$    tmpfil=TTT.$$
186    
187   rm -f pr_group ; touch pr_group  #--- running on the same node:
188   nc=0; xx=`hostname`    list='' ; nc=0; xx=`hostname`
189   for dd1 in cpl ocn atm    while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
190   do  #-- On darwin cluster node (from qrsh session):
191    echo $xx $nc $dir/build_$dd1/mitgcmuv >> pr_group  # JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
192    nc=1  # NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
193   done  # echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
194    #-- On ACES cluster (in PBS batch job):
195   cd $ROOTDIR  # NODEFILE=$PBS_NODEFILE
196   if test "x$MTH" != x ; then  #--- running on different nodes:
197     export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m  # ls -l $NODEFILE
198     if test "x$MTHo" != x ; then  # nprc=`cat $NODEFILE | uniq | wc -l`
199       echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"  # if [ $nprc -ge $Npr ] ; then
200     fi  #   list=`cat $NODEFILE | uniq | head -$Npr`
201     if test "x$MTHa" != x ; then  # else
202       echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"  #   list=`cat $NODEFILE | head -$Npr`
203     fi  # fi
204     echo ""  
205   fi    nc=0; nn=0; dd1=cpl ;
206  # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1    rm -f pr_group ; touch pr_group
207   mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1    for xx in $list
208   tail -20 std_outp    do
209   ls -l rank_?/pickup*.ckptA.001.001.data      echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
210        nc=`expr $nc + 1`
211        if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
212        nn=1
213      done
214      NpAt=`expr $Npr - 1 - $NpOc`
215      RunOpt="-np 1 ./build_cpl/mitgcmuv"
216      RunOpt="$RunOpt : -np $NpOc ./build_ocn/mitgcmuv"
217      RunOpt="$RunOpt : -np $NpAt ./build_atm/mitgcmuv"
218    
219      cd $ROOTDIR
220      if test "x$MTH" != x ; then
221        export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
222        if test "x$MTHo" != x ; then
223          echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
224        fi
225        if test "x$MTHa" != x ; then
226          echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
227        fi
228        echo ""
229      fi
230      mpich=`which mpirun`
231      echo $mpich | grep 'mpich-mx' > /dev/null 2>&1
232      mpichmx=$?
233      echo $mpich | grep 'mpich-1' > /dev/null 2>&1
234      mpich1=$?
235      echo $mpich | grep 'mpich2' > /dev/null 2>&1
236      mpich2=$?
237      echo $mpich | grep 'openmpi' > /dev/null 2>&1
238      opnmpi=$?
239      if test $mpich1 == 0 ; then
240      # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1
241      #- with mpich-1 (on danton, old aces: ao, geo, itrda):
242        echo "execute 'mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv' :"
243        mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
244      elif test $mpichmx == 0 ; then
245      #- with mpich-mx (on beagle):
246        echo "execute 'mpirun -pg pr_group -v ./build_cpl/mitgcmuv' :"
247        mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
248      elif test $mpich2 == 0 -o $opnmpi == 0 ; then
249      #- with Hydra mpich2 (on baudelaire) or with openmpi:
250        echo "execute 'mpirun $RunOpt' :"
251        mpirun $RunOpt  > std_outp 2>&1
252      else
253      #- new mpich (mpich2) installation often just put in "mpich" dir
254        echo "execute 'mpirun $RunOpt' :"
255        mpirun $RunOpt  > std_outp 2>&1
256      fi
257      tail -20 std_outp
258      ls -l rank_$rnkO/pickup*.ckptA.001.001.data
259      ls -l rank_$rnkA/pickup*.ckptA.001.001.data
260    
261  fi  fi
262    
263  if test $kpr = 4  if test $kpr = 4
264  then  then
265     CompRes="$HOME/bin/comp_res"
266    
267   if test -f rank_1/STDOUT.0000   if test -f rank_$rnkO/STDOUT.0000
268   then echo '==> check Ocean output:'   then echo '==> check Ocean output:'
269      /home/jmc/bin/comp_res rank_1/STDOUT.0000 results/ocnSTDOUT.0000      $CompRes rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
270      mv -f comp_res.log comp_res.ocn      mv -f comp_res.log comp_res.ocn
271      echo ' '      echo ' '
272   else echo 'No Ocean output file in rank_1' ; fi   else echo "No Ocean output file in rank_$rnkO" ; fi
273    
274   if test -f rank_2/STDOUT.0000   if test -f rank_$rnkA/STDOUT.0000
275   then   then
276      echo '==> check Atmos output:'      echo '==> check Atmos output:'
277      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
278      mv -f comp_res.log comp_res.atm      mv -f comp_res.log comp_res.atm
279      echo '==> check Land output:'      echo '==> check Land output:'
280      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 L      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
281      mv -f comp_res.log comp_res.land      mv -f comp_res.log comp_res.land
282      echo '==> check thSIce output:'      echo '==> check thSIce output:'
283      /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 I      $CompRes rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
284      mv -f comp_res.log comp_res.sice      mv -f comp_res.log comp_res.sice
285      echo ' '      echo ' '
286   else echo 'No Atmos output file in rank_2' ; fi   else echo "No Atmos output file in rank_$rnkA" ; fi
287    
288  fi  fi
289    
Legend:
Removed from v.1.13  
changed lines
  Added in v.1.22
  ViewVC Help
Powered by ViewVC 1.1.22