Annotation of /MITgcm_contrib/test_scripts/engaging/test_engag_op64_adm

#!/bin/bash
#SBATCH -J o64Adm_tst
#SBATCH -p sched_mit_hill
#SBATCH -t 06:00:00
#SBATCH --mem-per-cpu 4000
#SBATCH -n 6
#SBATCH -N 2
# #SBATCH -x node[360,365]
#SBATCH -e /home/jm_c/test_engaging/output/o64Adm_tst.stderr
#SBATCH -o /home/jm_c/test_engaging/output/o64Adm_tst.stdout
#SBATCH --no-requeue

# $Header: /u/gcmpack/MITgcm_contrib/test_scripts/engaging/test_engag_op64_adm,v 1.29 2023/11/06 14:23:01 jmc Exp $
# $Name:  $

if test -f /etc/profile.d/modules.sh ; then . /etc/profile.d/modules.sh ; fi
# Note: added "ulimit -s unlimited" in file "~/.bashrc"
# to pass big test (the 2 fizhi-cs-* test & adjoint tests) with MPI

umask 0022
#- to get case insensitive "ls" (and order of tested experiments)
export LC_ALL="en_US.UTF-8"
echo " running on: "`hostname`
headNode='eofe8'

dNam='engaging'
HERE="$HOME/test_${dNam}"
OUTP="$HERE/output"; SavD="$HERE/send"
SEND="ssh $headNode $SavD/mpack"
TST_DIR="/pool001/jm_c/test_$dNam"
tmpFil="/tmp/"`basename $0`".$$"
  cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q'
#- which GitHub repository to use and how to access it:
  git_repo='MITgcm'; git_code='MITgcm' ; git_other='verification_other'
 #git_repo='altMITgcm'; #git_code='MITgcm66h'
#--
  git_repo="https://github.com/$git_repo"
 #git_repo="git://github.com/$git_repo"
 #git_repo="git@github.com:$git_repo"

dblTr=0 ; typ='' ; addExp='' ; skipExp=''
sfx='o64Adm'; typ='-adm' ; dblTr=1
#- currently, no NetCDF => no pkg/profiles
skipExp="$skipExp global_oce_biogeo_bling"
 module add open64
 module add mvapich2/open64/64/2.0b
 export MPI_INC_DIR="$MPI_HOME/include"
 OPTFILE="../tools/build_options/linux_amd64_open64"
#options="-j 2 -devel -gsl"
 options="$typ -MPI 6"
#export OMP_NUM_THREADS=2
#export OMP_SLAVE_STACK_SIZE=400m
#export GSL_IEEE_MODE=double-precision,mask-underflow,mask-denormalized
 ulimit -s unlimited
#- need this to get "staf":
 export PATH="$PATH:$HOME/bin"

gcmDIR="MITgcm_$sfx"
dAlt=`date +%d` ; dAlt=`expr $dAlt % 3`
if [ $dAlt -eq 1 ] ; then options="$options -fast"
else options="$options -devel" ; fi
if test "x$skipExp" != x ; then skipExp=`echo $skipExp | sed 's/^ *//'` ; fi

checkOut=1 ; #options="$options -do"
#options="$options -nc" ; checkOut=1
#options="$options -q"  ; checkOut=0 ; dblTr=0
# dblTr=-1 #- skip testreport completely (only run "do_tst_2+2")

if test -d $TST_DIR ; then
  echo "start from TST_DIR='$TST_DIR' at: "`date`
else
 #if test ! -d $TST_DIR ; then mkdir $TST_DIR ; fi
 #if test ! -d $TST_DIR ; then
 #  echo "ERROR: Can't create directory \"$TST_DIR\""
 #  exit 1
 #fi
 #echo "start in new dir TST_DIR='$TST_DIR' at: "`date`
  echo "ERROR: missing directory \"$TST_DIR\""
  exit 1
fi
cd $TST_DIR
pwd
df .

NSLOTS=$SLURM_NTASKS
THEDATE=`date`
echo '********************************************************************************'
echo 'Start job '$THEDATE
echo 'NSLOTS = '$NSLOTS
echo '======= NODELIST ==============================================================='
echo $SLURM_NODELIST
cat /etc/redhat-release
echo '======= env ===================================================================='
env | grep SLURM
echo '======= modules ================================================================'
module list 2>&1
echo '================================================================================'

#- check for disk space: relative space (99%) or absolute (10.G):
dsp=`df -P . | tail -1 | awk '{print $5}' | sed 's/%$//'`
if [ $dsp -gt 99 ] ; then
#dsp=`df -P . | tail -1 | awk '{print $4}'`
#if [ $dsp -le 100000000 ] ; then
  echo 'Not enough space on this disk => do not run testreport.'
  df .
  exit
fi
if [ $checkOut -eq 1 ] ; then
  if test ! -e $gcmDIR/.git/config ; then
    echo "no file: $gcmDIR/.git/config => try to download a fresh clone"
    checkOut=2
  fi
  if test "x$addExp" != x ; then
    if test ! -e $gcmDIR/$git_other/.git/config ; then
      echo "no file: $gcmDIR/$git_other/.git/config => try a fresh clone"
      checkOut=2
    fi
  fi
fi
if [ $checkOut -eq 1 ] ; then
    echo "cleaning output from $gcmDIR/verification @ "`date +"%H:%M:%S"`" :"
#- remove previous output tar files and tar & remove previous output-dir
    /bin/rm -f $gcmDIR/verification/??_${dNam}-${sfx}_????????_?.tar.gz
    ( cd $gcmDIR/verification
      listD=`ls -1 -d ??_${dNam}-${sfx}_????????_? 2> /dev/null`
      for dd in $listD
      do
        if test -d $dd ; then
          tar -cf ${dd}".tar" $dd > /dev/null 2>&1 && gzip ${dd}".tar" && /bin/rm -rf $dd
          retVal=$?
          if test "x$retVal" != x0 ; then
             echo "ERROR in tar+gzip prev outp-dir: $dd"
             echo " on '"`hostname`"' (return val=$retVal) but continue"
          fi
        fi
      done )
    echo "clean testreport output"
  # ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
    ( cd $gcmDIR/verification ; ./testreport $typ -clean )
    if test "x$addExp" != x ; then
      ( cd $gcmDIR/verification
        listD=`ls -o | grep '^l' | awk '{print $8}' 2> /dev/null`
        echo " + remove local links: $listD"
        /bin/rm -f $listD
      )
    fi
    echo "Update $git_code code in dir $gcmDIR @ "`date +"%H:%M:%S"`" :"
    ( cd $gcmDIR ; git pull ) 2>&1
    retVal=$?
    if test "x$retVal" != x0 ; then
       echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
       exit
    fi
    echo "  and checkout master @ "`date +"%H:%M:%S"`" :"
    ( cd $gcmDIR ; git checkout master -- . ) 2>&1
  if test "x$addExp" != x ; then
    echo "Update $git_other code in dir $gcmDIR/$git_other @ "`date +"%H:%M:%S"`" :"
    ( cd $gcmDIR/$git_other ; git pull ) 2>&1
    retVal=$?
    if test "x$retVal" != x0 ; then
       echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
       exit
    fi
    echo "  and checkout master @ "`date +"%H:%M:%S"`" :"
    ( cd $gcmDIR/$git_other ; git checkout master -- . ) 2>&1
  fi
fi

if [ $checkOut -eq 2 ] ; then
  if test -e $gcmDIR ; then
    echo -n "Removing working copy: $gcmDIR ..."
    rm -rf $gcmDIR
    echo "  done"
  fi
  echo "Make a clone of $git_code from repo: $git_repo ..."
  git clone $git_repo/${git_code}.git $gcmDIR 2> $tmpFil
  retVal=$?
  if test $retVal = 0 ; then
      echo '  done' ; rm -f $tmpFil
  else
      echo " Error: 'git clone' returned: $retVal"
      cat $tmpFil ; rm -f $tmpFil
      exit
  fi
  if test "x$addExp" != x ; then
    echo "Make a clone of $git_other from repo: $git_repo ..."
    ( cd $gcmDIR ; git clone $git_repo/${git_other}.git 2> $tmpFil )
    retVal=$?
    if test $retVal = 0 ; then
      echo '  done' ; rm -f $tmpFil
    else
      echo " Error: 'git clone' returned: $retVal"
      cat $tmpFil ; rm -f $tmpFil
      exit
    fi
  fi
  if test -d $gcmDIR/verification ; then
    /usr/bin/find $gcmDIR -type d | xargs chmod g+rxs
    /usr/bin/find $gcmDIR -type f | xargs chmod g+r
  fi
fi

#- change dir to $gcmDIR/verification + add link for additional experiments:
if test -e $gcmDIR/verification ; then
  if [ $checkOut -lt 2 ] ; then
    echo " dir $gcmDIR/verification exist" ; fi
  cd $gcmDIR/verification
  for exp2add in $addExp ; do
    test -r $exp2add && /bin/rm -rf $exp2add
    if test -d  ../$git_other/$exp2add ; then
      echo "  add $exp2add link from $git_other"
      ln -s ../$git_other/$exp2add .
    else
      echo "  missing dir: $git_other/$exp2add"
      continue
    fi
  done
else
  echo "no dir: $gcmDIR/verification => exit"
  exit
fi

if [ $dblTr -eq 1 ] ; then
  echo ''
#- 1) just compile ("-nr"), using "-j 2" to speed up
  echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
    -j 2 -nr -odir ${dNam}-$sfx
  ./testreport $options -of $OPTFILE -skd "$skipExp" \
    -j 2 -nr -odir ${dNam}-$sfx
  options="$options -q"
fi

if [ $dblTr -ge 0 ] ; then
  echo ''
#- 2) run and report results ; also finish to compile those who failed with "-j"
  echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
    -odir ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jm_c@mitgcm.org
  ./testreport $options -of $OPTFILE -skd "$skipExp" \
    -odir ${dNam}-$sfx -send "$SEND" -sd $SavD -a jm_c@mitgcm.org
  retVal=$?
else retVal=0 ; fi

if test "x$retVal" != x0 ; then
  echo "<== testreport returned retVal=${retVal}, expecting 0"
# echo " -> skip restart test 'do_tst_2+2'"
#else
# echo ''
#- 3) test restart and report results
# echo ../tools/do_tst_2+2 -mpi \
#   -o ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jm_c@mitgcm.org
# ../tools/do_tst_2+2 -mpi \
#   -o ${dNam}-$sfx -send "$SEND" -sd $SavD -a jm_c@mitgcm.org
fi

# exit 0
  echo -n "-- SLURM_TASKS_PER_NODE= $SLURM_TASKS_PER_NODE ; " >> ${OUTP}/tracer_adj.log
  date >> ${OUTP}/tracer_adj.log
  grep 'My Processor Name' tutorial_tracer_adjsens/run/STDOUT.000? \
     | sed 's/tutorial_tracer_adjsens\//    /' >> ${OUTP}/tracer_adj.log
  grep 'My Processor Name' tutorial_tracer_adjsens/tr_run.som81/STDOUT.000? \
     | sed 's/tutorial_tracer_adjsens\// /'    >> ${OUTP}/tracer_adj.log
1	jmc	1.1	#!/bin/bash
2	jmc	1.13	#SBATCH -J o64Adm_tst
3	jmc	1.1	#SBATCH -p sched_mit_hill
4	jmc	1.26	#SBATCH -t 06:00:00
5	jmc	1.1	#SBATCH --mem-per-cpu 4000
6			#SBATCH -n 6
7			#SBATCH -N 2
8	jmc	1.23	# #SBATCH -x node[360,365]
9	jmc	1.13	#SBATCH -e /home/jm_c/test_engaging/output/o64Adm_tst.stderr
10			#SBATCH -o /home/jm_c/test_engaging/output/o64Adm_tst.stdout
11	jmc	1.19	#SBATCH --no-requeue
12	jmc	1.1
13	jmc	1.30	# $Header: /u/gcmpack/MITgcm_contrib/test_scripts/engaging/test_engag_op64_adm,v 1.29 2023/11/06 14:23:01 jmc Exp $
14	jmc	1.1	# $Name: $
15
16	jmc	1.27	if test -f /etc/profile.d/modules.sh ; then . /etc/profile.d/modules.sh ; fi
17	jmc	1.3	# Note: added "ulimit -s unlimited" in file "~/.bashrc"
18	jmc	1.1	# to pass big test (the 2 fizhi-cs-* test & adjoint tests) with MPI
19
20			umask 0022
21			#- to get case insensitive "ls" (and order of tested experiments)
22			export LC_ALL="en_US.UTF-8"
23			echo " running on: "`hostname`
24	jmc	1.27	headNode='eofe8'
25	jmc	1.1
26			dNam='engaging'
27	jmc	1.27	HERE="$HOME/test_${dNam}"
28	jmc	1.1	OUTP="$HERE/output"; SavD="$HERE/send"
29	jmc	1.27	SEND="ssh $headNode $SavD/mpack"
30	jmc	1.1	TST_DIR="/pool001/jm_c/test_$dNam"
31	jmc	1.11	tmpFil="/tmp/"`basename $0`".$$"
32			cmdCVS='cvs -d :pserver:cvsanon@mitgcm.org:/u/gcmpack -q'
33	jmc	1.13	#- which GitHub repository to use and how to access it:
34	jmc	1.17	git_repo='MITgcm'; git_code='MITgcm' ; git_other='verification_other'
35	jmc	1.11	#git_repo='altMITgcm'; #git_code='MITgcm66h'
36	jmc	1.13	#--
37			git_repo="https://github.com/$git_repo"
38			#git_repo="git://github.com/$git_repo"
39			#git_repo="git@github.com:$git_repo"
40	jmc	1.1
41	jmc	1.21	dblTr=0 ; typ='' ; addExp='' ; skipExp=''
42			sfx='o64Adm'; typ='-adm' ; dblTr=1
43	jmc	1.25	#- currently, no NetCDF => no pkg/profiles
44			skipExp="$skipExp global_oce_biogeo_bling"
45	jmc	1.1	module add open64
46			module add mvapich2/open64/64/2.0b
47			export MPI_INC_DIR="$MPI_HOME/include"
48			OPTFILE="../tools/build_options/linux_amd64_open64"
49			#options="-j 2 -devel -gsl"
50			options="$typ -MPI 6"
51			#export OMP_NUM_THREADS=2
52			#export OMP_SLAVE_STACK_SIZE=400m
53			#export GSL_IEEE_MODE=double-precision,mask-underflow,mask-denormalized
54			ulimit -s unlimited
55			#- need this to get "staf":
56	jmc	1.27	export PATH="$PATH:$HOME/bin"
57	jmc	1.1
58	jmc	1.11	gcmDIR="MITgcm_$sfx"
59	jmc	1.1	dAlt=`date +%d` ; dAlt=`expr $dAlt % 3`
60			if [ $dAlt -eq 1 ] ; then options="$options -fast"
61			else options="$options -devel" ; fi
62	jmc	1.25	if test "x$skipExp" != x ; then skipExp=`echo $skipExp \| sed 's/^ *//'` ; fi
63	jmc	1.1
64	jmc	1.3	checkOut=1 ; #options="$options -do"
65	jmc	1.1	#options="$options -nc" ; checkOut=1
66	jmc	1.21	#options="$options -q" ; checkOut=0 ; dblTr=0
67	jmc	1.29	# dblTr=-1 #- skip testreport completely (only run "do_tst_2+2")
68	jmc	1.1
69	jmc	1.11	if test -d $TST_DIR ; then
70			echo "start from TST_DIR='$TST_DIR' at: "`date`
71			else
72			#if test ! -d $TST_DIR ; then mkdir $TST_DIR ; fi
73			#if test ! -d $TST_DIR ; then
74			# echo "ERROR: Can't create directory \"$TST_DIR\""
75			# exit 1
76			#fi
77			#echo "start in new dir TST_DIR='$TST_DIR' at: "`date`
78			echo "ERROR: missing directory \"$TST_DIR\""
79			exit 1
80			fi
81			cd $TST_DIR
82			pwd
83	jmc	1.20	df .
84	jmc	1.11
85	jmc	1.1	NSLOTS=$SLURM_NTASKS
86	jmc	1.2	THEDATE=`date`
87	jmc	1.1	echo '********************************************************************************'
88			echo 'Start job '$THEDATE
89			echo 'NSLOTS = '$NSLOTS
90			echo '======= NODELIST ==============================================================='
91			echo $SLURM_NODELIST
92	jmc	1.4	cat /etc/redhat-release
93	jmc	1.1	echo '======= env ===================================================================='
94	jmc	1.2	env \| grep SLURM
95			echo '======= modules ================================================================'
96			module list 2>&1
97	jmc	1.1	echo '================================================================================'
98
99			#- check for disk space: relative space (99%) or absolute (10.G):
100			dsp=`df -P . \| tail -1 \| awk '{print $5}' \| sed 's/%$//'`
101			if [ $dsp -gt 99 ] ; then
102			#dsp=`df -P . \| tail -1 \| awk '{print $4}'`
103			#if [ $dsp -le 100000000 ] ; then
104			echo 'Not enough space on this disk => do not run testreport.'
105			df .
106			exit
107			fi
108			if [ $checkOut -eq 1 ] ; then
109	jmc	1.15	if test ! -e $gcmDIR/.git/config ; then
110			echo "no file: $gcmDIR/.git/config => try to download a fresh clone"
111			checkOut=2
112			fi
113			if test "x$addExp" != x ; then
114			if test ! -e $gcmDIR/$git_other/.git/config ; then
115			echo "no file: $gcmDIR/$git_other/.git/config => try a fresh clone"
116			checkOut=2
117			fi
118			fi
119			fi
120			if [ $checkOut -eq 1 ] ; then
121	jmc	1.30	echo "cleaning output from $gcmDIR/verification @ "`date +"%H:%M:%S"`" :"
122	jmc	1.1	#- remove previous output tar files and tar & remove previous output-dir
123			/bin/rm -f $gcmDIR/verification/??_${dNam}-${sfx}_????????_?.tar.gz
124			( cd $gcmDIR/verification
125			listD=`ls -1 -d ??_${dNam}-${sfx}_????????_? 2> /dev/null`
126			for dd in $listD
127			do
128			if test -d $dd ; then
129			tar -cf ${dd}".tar" $dd > /dev/null 2>&1 && gzip ${dd}".tar" && /bin/rm -rf $dd
130	jmc	1.11	retVal=$?
131			if test "x$retVal" != x0 ; then
132	jmc	1.1	echo "ERROR in tar+gzip prev outp-dir: $dd"
133	jmc	1.11	echo " on '"`hostname`"' (return val=$retVal) but continue"
134	jmc	1.1	fi
135			fi
136			done )
137	jmc	1.27	echo "clean testreport output"
138			# ( cd $gcmDIR/verification ; ../tools/do_tst_2+2 -clean )
139	jmc	1.1	( cd $gcmDIR/verification ; ./testreport $typ -clean )
140	jmc	1.27	if test "x$addExp" != x ; then
141			( cd $gcmDIR/verification
142			listD=`ls -o \| grep '^l' \| awk '{print $8}' 2> /dev/null`
143			echo " + remove local links: $listD"
144			/bin/rm -f $listD
145			)
146			fi
147	jmc	1.30	echo "Update $git_code code in dir $gcmDIR @ "`date +"%H:%M:%S"`" :"
148	jmc	1.14	( cd $gcmDIR ; git pull ) 2>&1
149	jmc	1.11	retVal=$?
150			if test "x$retVal" != x0 ; then
151			echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
152	jmc	1.1	exit
153			fi
154	jmc	1.30	echo " and checkout master @ "`date +"%H:%M:%S"`" :"
155	jmc	1.14	( cd $gcmDIR ; git checkout master -- . ) 2>&1
156	jmc	1.15	if test "x$addExp" != x ; then
157	jmc	1.30	echo "Update $git_other code in dir $gcmDIR/$git_other @ "`date +"%H:%M:%S"`" :"
158	jmc	1.15	( cd $gcmDIR/$git_other ; git pull ) 2>&1
159			retVal=$?
160			if test "x$retVal" != x0 ; then
161			echo "git pull on '"`hostname`"' fail (return val=$retVal) => exit"
162			exit
163			fi
164	jmc	1.30	echo " and checkout master @ "`date +"%H:%M:%S"`" :"
165	jmc	1.15	( cd $gcmDIR/$git_other ; git checkout master -- . ) 2>&1
166	jmc	1.1	fi
167			fi
168	jmc	1.17
169	jmc	1.1	if [ $checkOut -eq 2 ] ; then
170			if test -e $gcmDIR ; then
171	jmc	1.3	echo -n "Removing working copy: $gcmDIR ..."
172			rm -rf $gcmDIR
173			echo " done"
174	jmc	1.1	fi
175	jmc	1.17	echo "Make a clone of $git_code from repo: $git_repo ..."
176	jmc	1.13	git clone $git_repo/${git_code}.git $gcmDIR 2> $tmpFil
177	jmc	1.11	retVal=$?
178			if test $retVal = 0 ; then
179			echo ' done' ; rm -f $tmpFil
180			else
181			echo " Error: 'git clone' returned: $retVal"
182			cat $tmpFil ; rm -f $tmpFil
183	jmc	1.1	exit
184	jmc	1.11	fi
185	jmc	1.15	if test "x$addExp" != x ; then
186			echo "Make a clone of $git_other from repo: $git_repo ..."
187			( cd $gcmDIR ; git clone $git_repo/${git_other}.git 2> $tmpFil )
188			retVal=$?
189			if test $retVal = 0 ; then
190			echo ' done' ; rm -f $tmpFil
191			else
192			echo " Error: 'git clone' returned: $retVal"
193			cat $tmpFil ; rm -f $tmpFil
194			exit
195			fi
196			fi
197	jmc	1.1	if test -d $gcmDIR/verification ; then
198			/usr/bin/find $gcmDIR -type d \| xargs chmod g+rxs
199			/usr/bin/find $gcmDIR -type f \| xargs chmod g+r
200			fi
201			fi
202
203	jmc	1.17	#- change dir to $gcmDIR/verification + add link for additional experiments:
204	jmc	1.1	if test -e $gcmDIR/verification ; then
205			if [ $checkOut -lt 2 ] ; then
206			echo " dir $gcmDIR/verification exist" ; fi
207			cd $gcmDIR/verification
208	jmc	1.7	for exp2add in $addExp ; do
209	jmc	1.15	test -r $exp2add && /bin/rm -rf $exp2add
210			if test -d ../$git_other/$exp2add ; then
211			echo " add $exp2add link from $git_other"
212			ln -s ../$git_other/$exp2add .
213	jmc	1.7	else
214	jmc	1.15	echo " missing dir: $git_other/$exp2add"
215			continue
216	jmc	1.7	fi
217			done
218	jmc	1.1	else
219			echo "no dir: $gcmDIR/verification => exit"
220			exit
221			fi
222
223	jmc	1.21	if [ $dblTr -eq 1 ] ; then
224	jmc	1.8	echo ''
225	jmc	1.27	#- 1) just compile ("-nr"), using "-j 2" to speed up
226	jmc	1.25	echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
227	jmc	1.27	-j 2 -nr -odir ${dNam}-$sfx
228	jmc	1.25	./testreport $options -of $OPTFILE -skd "$skipExp" \
229	jmc	1.27	-j 2 -nr -odir ${dNam}-$sfx
230	jmc	1.21	options="$options -q"
231			fi
232
233	jmc	1.29	if [ $dblTr -ge 0 ] ; then
234	jmc	1.21	echo ''
235	jmc	1.27	#- 2) run and report results ; also finish to compile those who failed with "-j"
236	jmc	1.25	echo ./testreport $options -of $OPTFILE -skd \'$skipExp\' \
237			-odir ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jm_c@mitgcm.org
238			./testreport $options -of $OPTFILE -skd "$skipExp" \
239			-odir ${dNam}-$sfx -send "$SEND" -sd $SavD -a jm_c@mitgcm.org
240	jmc	1.28	retVal=$?
241	jmc	1.29	else retVal=0 ; fi
242	jmc	1.8
243	jmc	1.28	if test "x$retVal" != x0 ; then
244			echo "<== testreport returned retVal=${retVal}, expecting 0"
245			# echo " -> skip restart test 'do_tst_2+2'"
246			#else
247	jmc	1.8	# echo ''
248	jmc	1.25	#- 3) test restart and report results
249	jmc	1.8	# echo ../tools/do_tst_2+2 -mpi \
250	jmc	1.10	# -o ${dNam}-$sfx -send \'$SEND\' -sd $SavD -a jm_c@mitgcm.org
251	jmc	1.8	# ../tools/do_tst_2+2 -mpi \
252	jmc	1.10	# -o ${dNam}-$sfx -send "$SEND" -sd $SavD -a jm_c@mitgcm.org
253	jmc	1.28	fi
254	jmc	1.1
255	jmc	1.24	# exit 0
256			echo -n "-- SLURM_TASKS_PER_NODE= $SLURM_TASKS_PER_NODE ; " >> ${OUTP}/tracer_adj.log
257			date >> ${OUTP}/tracer_adj.log
258			grep 'My Processor Name' tutorial_tracer_adjsens/run/STDOUT.000? \
259			\| sed 's/tutorial_tracer_adjsens\// /' >> ${OUTP}/tracer_adj.log
260			grep 'My Processor Name' tutorial_tracer_adjsens/tr_run.som81/STDOUT.000? \
261			\| sed 's/tutorial_tracer_adjsens\// /' >> ${OUTP}/tracer_adj.log