--- MITgcm/verification/cpl_aim+ocn/run_cpl_test	2010/11/22 19:03:43	1.16
+++ MITgcm/verification/cpl_aim+ocn/run_cpl_test	2010/12/06 02:52:40	1.17
@@ -1,14 +1,17 @@
 #! /bin/sh
 
-# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/cpl_aim+ocn/run_cpl_test,v 1.16 2010/11/22 19:03:43 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/cpl_aim+ocn/run_cpl_test,v 1.17 2010/12/06 02:52:40 jmc Exp $
 # $Name:  $
 
 #- default:
-Np=3
+ Npr=3 ; NpOc=1 ;
+#Npr=25; NpOc=12;
 MTH=
 MTHo=
 MTHa=
 GMKopt='-ieee'
+#GMKopt=
+rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
 
 #- parse options:
 if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
@@ -24,7 +27,7 @@
   echo ' step = 0 : clean all directories'
   echo ' step = 1 : compile the 3 executables (cpl,ocn,atm)'
   echo ' step = 2 : copy input files and dir(s)'
-  echo " step = 3 : run with $Np mpi processes"
+  echo " step = 3 : run with $Npr mpi processes"
   echo ' step = 4 : check the results'
   echo ' step = 5 : remove output files in rank_0,1,2 dir.'
   exit
@@ -38,20 +41,20 @@
 then
  rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
  rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
- /bin/rm -r -f rank_0 rank_1 rank_2
- if test -f build_cpl/Makefile ; then cd build_cpl ; make CLEAN ; cd .. ; fi
- if test -f build_ocn/Makefile ; then cd build_ocn ; make CLEAN ; cd .. ; fi
- if test -f build_atm/Makefile ; then cd build_atm ; make CLEAN ; cd .. ; fi
+ /bin/rm -r -f rank_? rank_1? rank_2?
+ if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
+ if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
+ if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
 fi
 if test $kpr = 5
 then
  echo 'remove output files in rank_0,1,2 dir.'
  rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
  if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
- if test -d rank_1
- then cd rank_1 ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
- if test -d rank_2
- then cd rank_2 ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
+ if test -d rank_$rnkO
+ then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
+ if test -d rank_$rnkA
+ then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
 fi
 
 if test $kpr = 1
@@ -59,14 +62,19 @@
 
 #- choice of the optfile: take a local one in dir verification with sufix '+mpi'
  nbOpF=`ls ../linux_* | grep '+mpi' 2> /dev/null | wc -l`
- if test $nbOpF = 1
- then 
+ if test $nbOpF = 1 ; then
    OPTFILE=`ls ../linux_* | grep '+mpi'`
    zz=`grep '^FC=' $OPTFILE`
    echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"
- else 
+ elif [ $nbOpF -ge 2 ] ; then
+   echo "pick the 1rst of these ( $nbOpF ) optfiles:"
+   ls ../linux_* | grep '+mpi'
+   OPTFILE=`ls ../linux_* | grep '+mpi' | head -1`
+   zz=`grep '^FC=' $OPTFILE`
+   echo " Using optfile: $OPTFILE  (compiler=$zz) $MTH"
+ else
    echo "Pb in finding optfile: found $nbOpF :"
-   ls ../linux_* | grep '+mpi' ; exit 
+   ls ../linux_* | grep '+mpi' ; exit
  fi
  zz=`echo $OPTFILE | grep -c '^\/'`
  if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
@@ -116,91 +124,130 @@
 
 if test $kpr = 2
 then
-  /bin/rm -r -f rank_0 rank_1 rank_2
-  echo 'CP dir:' input_cpl '->' rank_0
-  cp -p -r input_cpl rank_0
-
-  echo 'CP dir:' input_ocn '->' rank_1
-  cp -p -r input_ocn rank_1
-  cd rank_1
-  ./prepare_run
+  echo 'rm dir:' rank_? rank_1? rank_2?
+  /bin/rm -r -f rank_? rank_1? rank_2?
+  echo 'Link files from dir:' input_cpl '->' rank_0
+  mkdir rank_0
+  ( cd rank_0 ; ln -s ../input_cpl/* . )
+
+  n=$rnkO
+  echo 'Link files from dir:' input_ocn '->' rank_$n
+  mkdir rank_$n
+  cd rank_$n
+  ln -s ../input_ocn/* .
+  if test -x prepare_run ; then ./prepare_run ; fi
   if test "x$MTHo" != x ; then
     echo " MTH run: mv -f eedata.mth eedata"
+    if test -h eedata ; then rm -f eedata ; fi
     mv -f eedata.mth eedata
   fi
   cd $dir
-
-  echo 'CP dir:' input_atm '->' rank_2
-  cp -p -r input_atm rank_2
-  cd rank_2
-  ./prepare_run
+  n=`expr $n + 1`
+  while [ $n -le $NpOc ] ; do
+    ln -s rank_$rnkO rank_$n
+    n=`expr $n + 1`
+  done
+
+  n=$rnkA
+  echo 'Link files from dir:' input_atm '->' rank_$n
+  mkdir rank_$n
+  cd rank_$n
+  ln -s ../input_atm/* .
+  if test -x prepare_run ; then ./prepare_run ; fi
   if test "x$MTHa" != x ; then
     echo " MTH run: mv -f eedata.mth eedata"
+    if test -h eedata ; then rm -f eedata ; fi
     mv -f eedata.mth eedata
   fi
   cd $dir
+  n=`expr $n + 1`
+  while [ $n -lt $Npr ] ; do
+    ln -s rank_$rnkA rank_$n
+    n=`expr $n + 1`
+  done
 
 fi
 
 if test $kpr = 3
 then
- ROOTDIR=$dir
- rm -f rank_?/pickup*.ckptA.00?.00?.??ta
- echo $ROOTDIR
- tmpfil=TTT.$$
-
- rm -f pr_group ; touch pr_group 
- nc=0; xx=`hostname`
- for dd1 in cpl ocn atm
- do
-  echo $xx $nc $dir/build_$dd1/mitgcmuv >> pr_group
-  nc=1
- done
-
- cd $ROOTDIR
- if test "x$MTH" != x ; then
-   export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
-   if test "x$MTHo" != x ; then
-     echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
-   fi
-   if test "x$MTHa" != x ; then
-     echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
-   fi
-   echo ""
- fi
+  ROOTDIR=$dir
+# rm -f rank_?/pickup*.ckptA.00?.00?.??ta
+  echo $ROOTDIR
+  tmpfil=TTT.$$
+
+#--- running on the same node:
+  list='' ; nc=0; xx=`hostname`
+  while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
+#-- On darwin cluster node (from qrsh session):
+# JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
+# NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
+# echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
+#-- On ACES cluster (in PBS batch job):
+# NODEFILE=$PBS_NODEFILE
+#--- running on different nodes:
+# ls -l $NODEFILE
+# nprc=`cat $NODEFILE | uniq | wc -l`
+# if [ $nprc -ge $Npr ] ; then
+#   list=`cat $NODEFILE | uniq | head -$Npr`
+# else
+#   list=`cat $NODEFILE | head -$Npr`
+# fi
+
+  nc=0; nn=0; dd1=cpl ;
+  rm -f pr_group ; touch pr_group
+  for xx in $list
+  do
+    echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
+    nc=`expr $nc + 1`
+    if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
+    nn=1
+  done
+
+  cd $ROOTDIR
+  if test "x$MTH" != x ; then
+    export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
+    if test "x$MTHo" != x ; then
+      echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
+    fi
+    if test "x$MTHa" != x ; then
+      echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
+    fi
+    echo ""
+  fi
 # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v  ./build_cpl/mitgcmuv > std_outp 2>&1
-#- on danton (mpich-1):
- mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
-#- on beagle:
-#mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
- tail -20 std_outp
- ls -l rank_?/pickup*.ckptA.001.001.data
+#- with mpich-1 (on danton, ACES):
+  mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
+#- with mpich-mx (on beagle):
+# mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
+  tail -20 std_outp
+  ls -l rank_$rnkO/pickup*.ckptA.001.001.data
+  ls -l rank_$rnkA/pickup*.ckptA.001.001.data
 
 fi
 
 if test $kpr = 4
 then
 
- if test -f rank_1/STDOUT.0000
+ if test -f rank_$rnkO/STDOUT.0000
  then echo '==> check Ocean output:'
-    /home/jmc/bin/comp_res rank_1/STDOUT.0000 results/ocnSTDOUT.0000
+    /home/jmc/bin/comp_res rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
     mv -f comp_res.log comp_res.ocn
     echo ' '
- else echo 'No Ocean output file in rank_1' ; fi
+ else echo "No Ocean output file in rank_$rnkO" ; fi
 
- if test -f rank_2/STDOUT.0000
- then 
+ if test -f rank_$rnkA/STDOUT.0000
+ then
     echo '==> check Atmos output:'
-    /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000
+    /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
     mv -f comp_res.log comp_res.atm
     echo '==> check Land output:'
-    /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 L
+    /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
     mv -f comp_res.log comp_res.land
     echo '==> check thSIce output:'
-    /home/jmc/bin/comp_res rank_2/STDOUT.0000 results/atmSTDOUT.0000 I
+    /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
     mv -f comp_res.log comp_res.sice
     echo ' '
- else echo 'No Atmos output file in rank_2' ; fi
+ else echo "No Atmos output file in rank_$rnkA" ; fi
 
 fi