3 |
# $Header$ |
# $Header$ |
4 |
# $Name$ |
# $Name$ |
5 |
|
|
|
run_mpi() |
|
|
{ |
|
|
# run_mpi output_file |
|
|
echo "execute: '$RUNMPI'" |
|
|
$RUNMPI |
|
|
if test $# = 1 |
|
|
then mv STDOUT.0000 $1 |
|
|
echo 'mv STDOUT.0000' $1 |
|
|
fi |
|
|
} |
|
|
|
|
6 |
rnp_loc() |
rnp_loc() |
7 |
{ |
{ |
8 |
# rnp_loc arg_1 arg_2 |
# rnp_loc arg_1 arg_2 |
37 |
action=1 |
action=1 |
38 |
cmdMV='mv -f' |
cmdMV='mv -f' |
39 |
tmpDir='temp_tst' |
tmpDir='temp_tst' |
40 |
|
wait2run=0 # if needs sometime for file system syncro on all nodes |
41 |
|
|
42 |
#- default executable, command-line , using MPI: |
#- default executable, command-line , using MPI: |
43 |
gcmEXE=./mitgcmuv |
gcmEXE=./mitgcmuv |
63 |
*) flag=$xx ;; |
*) flag=$xx ;; |
64 |
esac |
esac |
65 |
done |
done |
66 |
echo "cmdEXE='$cmdEXE' ; RUNMPI='$RUNMPI'" |
#echo "cmdEXE='$cmdEXE'" |
67 |
echo "nArg=$nArg ; mpi=$mpi ; flag='$flag'" |
#echo "nArg=$nArg ; mpi=$mpi ; flag='$flag'" |
68 |
|
|
69 |
case $nArg in |
case $nArg in |
70 |
1) ;; |
1) ;; |
84 |
#-- set the run-command : |
#-- set the run-command : |
85 |
if test "x$cmdEXE" = x |
if test "x$cmdEXE" = x |
86 |
then |
then |
87 |
RUNMPI="mpirun -machinefile ../../mf -v -np 2 $gcmEXE" |
if [ $mpi -ge 1 ] ; then |
88 |
|
cmdEXE="mpirun -machinefile ../../mf -v -np 2 $gcmEXE" |
89 |
|
else |
90 |
cmdEXE=$gcmEXE |
cmdEXE=$gcmEXE |
91 |
else |
fi |
|
RUNMPI="$cmdEXE" |
|
92 |
fi |
fi |
93 |
if [ $mpi -ge 1 ] ; then cmdEXE=run_mpi ; fi |
echo "cmdEXE='$cmdEXE'" |
|
echo "cmdEXE='$cmdEXE' ; RUNMPI='$RUNMPI'" |
|
94 |
#========================= |
#========================= |
95 |
|
|
96 |
if test $flag = '0' -o $flag = 'All' ; then |
if test $flag = '0' -o $flag = 'All' ; then |
135 |
listF=`ls -1 $yy.${zz}*` |
listF=`ls -1 $yy.${zz}*` |
136 |
for xx in $listF |
for xx in $listF |
137 |
do |
do |
138 |
new=`echo $xx | sed "s/^$yy\.$zz/$yy\.$sufx/"` |
new=`echo $xx | sed "s/^$yy\.$zz/$yy.$sufx/"` |
139 |
if [ $prt -ge 2 ] ; then echo ' '$cmdMV $xx $new ; fi |
if [ $prt -ge 2 ] ; then echo ' '$cmdMV $xx $new ; fi |
140 |
if test $action = 1 ; then $cmdMV $xx $new ; fi |
if test $action = 1 ; then $cmdMV $xx $new ; fi |
141 |
done |
done |
146 |
echo "prepare parameter file 'data.tst' :" |
echo "prepare parameter file 'data.tst' :" |
147 |
cp -p -f data data.tst |
cp -p -f data data.tst |
148 |
listOut='startTime endTime nIter0 nTimeSteps nEndIter chkPtFreq pChkPtFreq globalFiles' |
listOut='startTime endTime nIter0 nTimeSteps nEndIter chkPtFreq pChkPtFreq globalFiles' |
149 |
listOut=$listOut' useSingleCpuIO pickupSuff usePickupBeforeC54 mdsioLocalDir' |
listOut=$listOut' pickupSuff usePickupBeforeC54 mdsioLocalDir' |
150 |
#- comment out (in data.tst) all parameters from list: 'listOut' |
#- comment out (in data.tst) all parameters from list: 'listOut' |
151 |
for yy in $listOut |
for yy in $listOut |
152 |
do |
do |
182 |
mkdir $tmpDir |
mkdir $tmpDir |
183 |
mv -f *.data $tmpDir |
mv -f *.data $tmpDir |
184 |
mv -f *.meta $tmpDir |
mv -f *.meta $tmpDir |
185 |
mv -f *.txt $tmpDir |
mv -f *.txt $tmpDir 2> /dev/null |
186 |
mv -f STD???.0[0-9][0-9][0-9] $tmpDir |
mv -f STD???.0[0-9][0-9][0-9] $tmpDir |
187 |
listS=`ls $tmpDir/pickup*.${sufx}* 2> /dev/null | head -1` |
listS=`ls $tmpDir/pickup*.${sufx}* 2> /dev/null | head -1` |
188 |
if test "x$listS" != x ; then mv $tmpDir/pickup*.${sufx}* . ; fi |
if test "x$listS" != x ; then mv $tmpDir/pickup*.${sufx}* . ; fi |
263 |
cp -p data.tst data |
cp -p data.tst data |
264 |
echo '==> START RUN 2 x' $Nit 'it' |
echo '==> START RUN 2 x' $Nit 'it' |
265 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
266 |
then $cmdEXE std_outp.2it > TTT.out.2it 2>&1 |
then |
267 |
|
sleep $wait2run |
268 |
|
$cmdEXE > TTT.out.2it 2>&1 |
269 |
|
mv STDOUT.0000 std_outp.2it |
270 |
|
cat TTT.out.2it |
271 |
else $cmdEXE > std_outp.2it |
else $cmdEXE > std_outp.2it |
272 |
fi |
fi |
273 |
mv STDERR.0000 std__err.2it |
mv STDERR.0000 std__err.2it |
274 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.2it` |
out=`egrep -c 'STOP ABNORMAL END' std_outp.2it` |
275 |
if test $out != 0 |
if test $out != 0 |
276 |
then echo "==> RUN 2 x $Nit it : ABNORMAL END => exit" ; exit 1 ; fi |
then echo "==> RUN 2 x $Nit it : ABNORMAL END => exit" ; exit 1 ; fi |
277 |
if test -f $pref.ckptA.001.001.data |
listF=`ls -1 $pref.ckptA*.data 2> /dev/null` |
278 |
then echo "==> END RUN 2 x $Nit it" |
if test "x$listF" = x |
279 |
else echo "==> RUN 2 x $Nit it STOP without writing $pref => exit" ; exit 1 ; fi |
then echo "==> RUN 2 x $Nit it STOP without writing $pref => exit" ; exit 1 |
280 |
|
else echo "==> END RUN 2 x $Nit it" ; fi |
281 |
listP=`ls pickup*.ckptA* | sed 's/\..*//g' | uniq` |
listP=`ls pickup*.ckptA* | sed 's/\..*//g' | uniq` |
282 |
echo ' listP=' $listP |
echo ' listP=' $listP |
283 |
for xx in $listP |
for xx in $listP |
284 |
do |
do |
285 |
rnp_loc $xx.ckptA $xx.$num2c |
rnp_loc $xx.ckptA $xx.$num2c |
286 |
done |
done |
287 |
mv *.0??.0??.* $dir1 |
mv *.data *.meta $dir1 |
288 |
txtF=`ls -1 *.txt 2> /dev/null` |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
289 |
mv R?.data R?.meta DR?.data DR?.meta $txtF $dir1 |
if test "x$txtF" != x ; then mv $txtF $dir1 ; fi |
290 |
listS=`ls $dir1/pickup*.${num0c}* 2> /dev/null | head -1` |
listS=`ls $dir1/pickup*.${num0c}* 2> /dev/null | head -1` |
291 |
if test "x$listS" != x ; then mv $dir1/pickup*.${num0c}* . ; fi |
if test "x$listS" != x ; then mv $dir1/pickup*.${num0c}* . ; fi |
292 |
#--- 2nd run ( Nit iter , restart from same Iter = num0 as 1rst run ) |
#--- 2nd run ( Nit iter , restart from same Iter = num0 as 1rst run ) |
293 |
sed "s/^ nTimeSteps=$Dbl/ nTimeSteps=$Nit/g" data.tst > data |
sed "s/^ nTimeSteps=$Dbl/ nTimeSteps=$Nit/g" data.tst > data.tmp |
294 |
|
mv -f data.tmp data |
295 |
echo '==> START RUN 1iA' |
echo '==> START RUN 1iA' |
296 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
297 |
then $cmdEXE std_outp.1iA > TTT.out.1iA 2>&1 |
then |
298 |
|
sleep $wait2run |
299 |
|
$cmdEXE > TTT.out.1iA 2>&1 |
300 |
|
mv STDOUT.0000 std_outp.1iA |
301 |
|
cat TTT.out.1iA |
302 |
else $cmdEXE > std_outp.1iA |
else $cmdEXE > std_outp.1iA |
303 |
fi |
fi |
304 |
mv STDERR.0000 std__err.1iA |
mv STDERR.0000 std__err.1iA |
305 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iA` |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iA` |
306 |
if test $out != 0 |
if test $out != 0 |
307 |
then echo "==> RUN 1iA : ABNORMAL END => exit" ; exit 2 ; fi |
then echo "==> RUN 1iA : ABNORMAL END => exit" ; exit 2 ; fi |
308 |
if test -f $pref.ckptA.001.001.data |
listF=`ls -1 $pref.ckptA*.data 2> /dev/null` |
309 |
then echo "==> END RUN 1iA" |
if test "x$listF" = x |
310 |
else echo "==> RUN 1iA STOP without writing $pref => exit" ; exit 2 ; fi |
then echo "==> RUN 1iA STOP without writing $pref => exit" ; exit 2 |
311 |
|
else echo "==> END RUN 1iA" ; fi |
312 |
for xx in $listP |
for xx in $listP |
313 |
do |
do |
314 |
rnp_loc $xx.ckptA $xx.$num1c |
rnp_loc $xx.ckptA $xx.$num1c |
315 |
done |
done |
316 |
mv *.0??.0??.* $dir2 |
mv *.data *.meta $dir2 |
317 |
txtF=`ls -1 *.txt 2> /dev/null` |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
318 |
mv R?.data R?.meta DR?.data DR?.meta $txtF $dir2 |
if test "x$txtF" != x ; then mv $txtF $dir2 ; fi |
319 |
listS=`ls $dir2/pickup*.${num0c}* 2> /dev/null | head -1` |
listS=`ls $dir2/pickup*.${num0c}* 2> /dev/null | head -1` |
320 |
if test "x$listS" != x ; then mv $dir2/pickup*.${num0c}* . ; fi |
if test "x$listS" != x ; then mv $dir2/pickup*.${num0c}* . ; fi |
321 |
#--- 3rd run ( Nit iter , restart from 2nd ) |
#--- 3rd run ( Nit iter , restart from 2nd ) |
324 |
mv -f data.tmp data |
mv -f data.tmp data |
325 |
echo '==> START RUN 1iB' |
echo '==> START RUN 1iB' |
326 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
327 |
then $cmdEXE std_outp.1iB > TTT.out.1iB 2>&1 |
then |
328 |
|
sleep $wait2run |
329 |
|
$cmdEXE > TTT.out.1iB 2>&1 |
330 |
|
mv STDOUT.0000 std_outp.1iB |
331 |
|
cat TTT.out.1iB |
332 |
else $cmdEXE > std_outp.1iB |
else $cmdEXE > std_outp.1iB |
333 |
fi |
fi |
334 |
mv STDERR.0000 std__err.1iB |
mv STDERR.0000 std__err.1iB |
335 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iB` |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iB` |
336 |
if test $out != 0 |
if test $out != 0 |
337 |
then echo "==> RUN 1iB : ABNORMAL END => exit" ; exit 3 ; fi |
then echo "==> RUN 1iB : ABNORMAL END => exit" ; exit 3 ; fi |
338 |
if test -f $pref.ckptA.001.001.data |
listF=`ls -1 $pref.ckptA*.data 2> /dev/null` |
339 |
then echo "==> END RUN 1iB" |
if test "x$listF" = x |
340 |
else echo "==> RUN 1iB STOP without writing $pref => exit" ; exit 3 ; fi |
then echo "==> RUN 1iB STOP without writing $pref => exit" ; exit 3 |
341 |
|
else echo "==> END RUN 1iB" ; fi |
342 |
for xx in $listP |
for xx in $listP |
343 |
do |
do |
344 |
rnp_loc $xx.ckptA $xx.$num2c |
rnp_loc $xx.ckptA $xx.$num2c |
345 |
done |
done |
346 |
mv *.0??.0??.* $dir3 |
mv *.data *.meta $dir3 |
347 |
txtF=`ls -1 *.txt 2> /dev/null` |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
348 |
mv R?.data R?.meta DR?.data DR?.meta $txtF $dir3 |
if test "x$txtF" != x ; then mv $txtF $dir3 ; fi |
349 |
listS=`ls $dir3/pickup*.${num0c}* 2> /dev/null | head -1` |
listS=`ls $dir3/pickup*.${num0c}* 2> /dev/null | head -1` |
350 |
if test "x$listS" != x ; then mv $dir3/pickup*.${num0c}* . ; fi |
if test "x$listS" != x ; then mv $dir3/pickup*.${num0c}* . ; fi |
351 |
rm $dir3/pickup*.$num1c.* |
rm $dir3/pickup*.$num1c.* |
376 |
chkmeta='Y' |
chkmeta='Y' |
377 |
for yy in $listP |
for yy in $listP |
378 |
do |
do |
379 |
listY=`(cd $dir1 ; ls $yy.$num2c.*.data | sed "s/$yy\.$num2c\.//" | sed "s/\.data//")` |
listY=`(cd $dir1 ; ls $yy.$num2c*.data | sed "s/$yy\.$num2c//" | sed "s/data$//")` |
380 |
zz=$yy.$num2c |
zz=$yy.$num2c |
381 |
echo '--> file='$zz', listY='$listY |
echo '--> file='$zz', listY='$listY |
382 |
for xx in $listY |
for xx in $listY |
383 |
do |
do |
384 |
echo ' diff' $dir1/$zz.$xx.data $dir3 |
echo ' diff' $dir1/$zz${xx}data $dir3 |
385 |
diff $dir1/$zz.$xx.data $dir3 |
diff $dir1/$zz${xx}data $dir3 |
386 |
out=$? |
out=$? |
387 |
if test $out != 0 ; then |
if test $out != 0 ; then |
388 |
echo 'Diff outp=' $out ' ==> stop' |
echo 'Diff outp=' $out ' ==> stop' |
389 |
exit 4 |
exit 4 |
390 |
fi |
fi |
391 |
if test $chkmeta = 'Y' -a -f $dir1/$zz.$xx.meta ; then |
if test $chkmeta = 'Y' -a -f $dir1/$zz${xx}meta ; then |
392 |
echo ' diff' $dir1/$zz.$xx.meta $dir3 |
echo ' diff' $dir1/$zz${xx}meta $dir3 |
393 |
diff $dir1/$zz.$xx.meta $dir3 |
diff $dir1/$zz${xx}meta $dir3 |
394 |
out=$? |
out=$? |
395 |
if test $out != 0 ; then |
if test $out != 0 ; then |
396 |
echo 'Diff outp=' $out ' ==> stop' |
echo 'Diff outp=' $out ' ==> stop' |
411 |
rm -rf $dir1 $dir2 $dir3 |
rm -rf $dir1 $dir2 $dir3 |
412 |
echo ' rm -f' std_????.?i? |
echo ' rm -f' std_????.?i? |
413 |
rm -f std_????.?i? |
rm -f std_????.?i? |
414 |
|
rm -f TTT.out.?i? 2> /dev/null |
415 |
echo ' rm -f' data.pkg data.tst |
echo ' rm -f' data.pkg data.tst |
416 |
rm -f data.pkg data.tst |
rm -f data.pkg data.tst |
417 |
#- move back files from temp_tst dir: |
#- move back files from temp_tst dir: |