19 |
|
|
20 |
#--------------------------- |
#--------------------------- |
21 |
|
|
|
#- default: |
|
|
Nit=2 |
|
|
prt=1 # 1 = print some informations ; 2 = print more informations |
|
|
action=1 |
|
|
cmd='mv -f' |
|
|
tmpDir='temp_tst' |
|
|
|
|
|
#- executable: |
|
|
mpi=0 |
|
|
if [ $mpi -ge 1 ] |
|
|
then gcmExc=run_ifc_mpi |
|
|
else gcmExc=mitgcmuv |
|
|
fi |
|
|
|
|
22 |
#- exit status: |
#- exit status: |
23 |
# 0 -> test is successful |
# 0 -> test is successful |
24 |
# 1 -> fail in 1rst run |
# 1 -> fail in 1rst run |
25 |
# 2 -> fail in 2nd run |
# 2 -> fail in 2nd run |
26 |
# 3 -> fail in 3rd run |
# 3 -> fail in 3rd run |
27 |
# 4 -> pickup*.data (written at the end of 2+2 & 4 it run) are different |
# 4 -> pickup*.data (written at the end of 2+2 & 4 it run) are different |
28 |
# 5 -> pickup*.meta (written at the end of 2+2 & 4 it run) are different |
# 5 -> pickup*.meta (written at the end of 2+2 & 4 it run) are different |
29 |
# 6 -> missing 'data.tst' file (removed or not yet generated ?) |
# 6 -> missing 'data.tst' file (removed or not yet generated ?) |
31 |
# 8 -> missing information (in standard output of previous run) about pickup |
# 8 -> missing information (in standard output of previous run) about pickup |
32 |
# 9 -> error in argument list |
# 9 -> error in argument list |
33 |
|
|
34 |
|
#- default: |
35 |
|
Nit=2 |
36 |
|
prt=1 # 1 = print some informations ; 2 = print more informations |
37 |
|
action=1 |
38 |
|
cmdMV='mv -f' |
39 |
|
tmpDir='temp_tst' |
40 |
|
wait2run=0 # if needs sometime for file system syncro on all nodes |
41 |
|
|
42 |
|
#- default executable, command-line , using MPI: |
43 |
|
gcmEXE=./mitgcmuv |
44 |
|
cmdEXE= |
45 |
|
mpi=0 |
46 |
|
|
47 |
nArg=$# |
nArg=$# |
48 |
if [ $nArg -ge 3 ] |
prev= |
49 |
then |
for xx ; do |
50 |
if test $2 = '-exe' ; then gcmExc=$3 ; nArg=`expr $nArg - 2` ; fi |
#echo "xx='$xx'" |
51 |
if test $1 = '-exe' ; then gcmExc=$2 ; shift ; shift ; nArg=$# ; fi |
# If the previous option needs an argument, assign it. |
52 |
fi |
if test -n "$prev"; then |
53 |
|
eval "$prev=\$xx" |
54 |
|
prev= |
55 |
|
nArg=`expr $nArg - 2` |
56 |
|
continue |
57 |
|
fi |
58 |
|
|
59 |
|
case $xx in |
60 |
|
-mpi ) mpi=1 ; nArg=`expr $nArg - 1` ;; |
61 |
|
-exe ) prev=gcmEXE ;; |
62 |
|
-command ) prev=cmdEXE ;; |
63 |
|
*) flag=$xx ;; |
64 |
|
esac |
65 |
|
done |
66 |
|
#echo "cmdEXE='$cmdEXE'" |
67 |
|
#echo "nArg=$nArg ; mpi=$mpi ; flag='$flag'" |
68 |
|
|
69 |
case $nArg in |
case $nArg in |
70 |
1) ;; |
1) ;; |
71 |
*) echo "Usage: `basename $0` flag [-exe EXECUTABLE]" |
*) echo "Usage: `basename $0` flag [-mpi] [-exe EXECUTABLE] [-command COMMAND]" |
72 |
echo " Check restart: compare 1 run of 2 x $Nit it long" |
echo " Check restart: compare 1 run of 2 x $Nit it long" |
73 |
echo " with 2 consecutive runs of $Nit it long each" |
echo " with 2 consecutive runs of $Nit it long each" |
74 |
echo " where: flag = 0 -> prepare (from a previous run) pickup & data files" |
echo " where: flag = 0 -> prepare (from a previous run) pickup & data files" |
75 |
echo " flag = 1 -> do the 3 runs (with EXECUTABLE, default=mitgcmuv)" |
echo " flag = 1 -> do the 3 runs (using COMMAND" |
76 |
|
echo " or simply using EXECUTABLE, default=$gcmEXE)" |
77 |
echo " flag = 2 -> compare std_outp" |
echo " flag = 2 -> compare std_outp" |
78 |
echo " flag = 3 -> compare pickup files" |
echo " flag = 3 -> compare pickup files" |
79 |
echo " flag = 4 -> clean-up output files" |
echo " flag = 4 -> clean-up output files" |
80 |
echo " flag = All -> do 0,1,2,3,4 sequentially" |
echo " flag = All -> do 0,1,2,3,4 sequentially" |
81 |
exit 9 ;; |
exit 9 ;; |
82 |
esac |
esac |
|
flag=$1 |
|
83 |
|
|
84 |
|
#-- set the run-command : |
85 |
|
if test "x$cmdEXE" = x |
86 |
|
then |
87 |
|
if [ $mpi -ge 1 ] ; then |
88 |
|
cmdEXE="mpirun -machinefile ../../mf -v -np 2 $gcmEXE" |
89 |
|
else |
90 |
|
cmdEXE=$gcmEXE |
91 |
|
fi |
92 |
|
fi |
93 |
|
echo "cmdEXE='$cmdEXE'" |
94 |
#========================= |
#========================= |
95 |
|
|
96 |
if test $flag = '0' -o $flag = 'All' ; then |
if test $flag = '0' -o $flag = 'All' ; then |
97 |
#---------------------------------------------------------------------- |
#---------------------------------------------------------------------- |
98 |
# last pickup written (from previous run): |
# last pickup written (from previous run): |
99 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
100 |
then outpRun='STDOUT.0000' |
then outpRun='STDOUT.0000' |
101 |
else outpRun='output.txt' |
else outpRun='output.txt' |
102 |
fi |
fi |
103 |
if test -r $outpRun |
if test -r $outpRun |
104 |
then |
then |
105 |
# grep ' %CHECKPOINT ' $outpRun | tail -1 |
# grep ' %CHECKPOINT ' $outpRun | tail -1 |
106 |
lastPick=`grep ' %CHECKPOINT ' $outpRun | sed 's/^.*%CHECKPOINT //' | tail -1` |
lastPick=`grep ' %CHECKPOINT ' $outpRun | sed 's/^.*%CHECKPOINT //' | tail -1` |
107 |
if test "x$lastPick" = x |
if test "x$lastPick" = x |
108 |
then |
then |
109 |
echo "no pickup information written in '$outpRun' from previous run" |
echo "no pickup information written in '$outpRun' from previous run" |
110 |
exit 8 |
exit 8 |
111 |
else |
else |
113 |
zz=`echo $lastPick | awk '{print $2}'` |
zz=`echo $lastPick | awk '{print $2}'` |
114 |
echo "from previous run std.outp, lastPick='$lastPick' ; iter='$iter' ; zz='$zz'" |
echo "from previous run std.outp, lastPick='$lastPick' ; iter='$iter' ; zz='$zz'" |
115 |
fi |
fi |
116 |
else |
else |
117 |
echo "no standard output file '$outpRun' from previous run" |
echo "no standard output file '$outpRun' from previous run" |
118 |
exit 7 |
exit 7 |
119 |
fi |
fi |
136 |
for xx in $listF |
for xx in $listF |
137 |
do |
do |
138 |
new=`echo $xx | sed "s/^$yy\.$zz/$yy\.$sufx/"` |
new=`echo $xx | sed "s/^$yy\.$zz/$yy\.$sufx/"` |
139 |
if [ $prt -ge 2 ] ; then echo ' '$cmd $xx $new ; fi |
if [ $prt -ge 2 ] ; then echo ' '$cmdMV $xx $new ; fi |
140 |
if test $action = 1 ; then $cmd $xx $new ; fi |
if test $action = 1 ; then $cmdMV $xx $new ; fi |
141 |
done |
done |
142 |
done |
done |
143 |
fi |
fi |
173 |
sed "/^ *\&PARM03/a \ nIter0=$iter," data.tst > data.tmp_$$ |
sed "/^ *\&PARM03/a \ nIter0=$iter," data.tst > data.tmp_$$ |
174 |
mv -f data.tmp_$$ data.tst |
mv -f data.tmp_$$ data.tst |
175 |
echo "prepare file 'data.tst' : done" |
echo "prepare file 'data.tst' : done" |
176 |
if [ $prt -ge 1 ] ; then |
if [ $prt -ge 1 ] ; then |
177 |
echo ' diff data.tst data' ; diff data.tst data ; echo ' ' ; |
echo ' diff data.tst data' ; diff data.tst data ; echo ' ' ; |
178 |
fi |
fi |
179 |
#---- clean up current dir: move output to temp_tst dir: |
#---- clean up current dir: move output to temp_tst dir: |
182 |
mkdir $tmpDir |
mkdir $tmpDir |
183 |
mv -f *.data $tmpDir |
mv -f *.data $tmpDir |
184 |
mv -f *.meta $tmpDir |
mv -f *.meta $tmpDir |
185 |
mv -f *.txt $tmpDir |
mv -f *.txt $tmpDir 2> /dev/null |
186 |
|
mv -f STD???.0[0-9][0-9][0-9] $tmpDir |
187 |
listS=`ls $tmpDir/pickup*.${sufx}* 2> /dev/null | head -1` |
listS=`ls $tmpDir/pickup*.${sufx}* 2> /dev/null | head -1` |
188 |
if test "x$listS" != x ; then mv $tmpDir/pickup*.${sufx}* . ; fi |
if test "x$listS" != x ; then mv $tmpDir/pickup*.${sufx}* . ; fi |
189 |
#- move main parameter file "data": |
#- move main parameter file "data": |
202 |
for ll in $listN |
for ll in $listN |
203 |
do |
do |
204 |
sed "$ll s/^./#&/" data.pkg > data.tmp_$$ |
sed "$ll s/^./#&/" data.pkg > data.tmp_$$ |
205 |
if [ $prt -ge 1 ] ; then echo ' changes in data.pkg' |
if [ $prt -ge 1 ] ; then echo ' changes in data.pkg' |
206 |
diff data.tmp_$$ data.pkg ; echo ' ' ; fi |
diff data.tmp_$$ data.pkg ; echo ' ' ; fi |
207 |
mv -f data.tmp_$$ data.pkg |
mv -f data.tmp_$$ data.pkg |
208 |
done |
done |
238 |
num2c=`printf "%10.10i\n" $num2` |
num2c=`printf "%10.10i\n" $num2` |
239 |
echo " start-end iter: $num0 , $num1 , $num2" |
echo " start-end iter: $num0 , $num1 , $num2" |
240 |
echo " sufix: '$num0c' '$num1c' '$num2c'" |
echo " sufix: '$num0c' '$num1c' '$num2c'" |
241 |
echo ' gcmExc='$gcmExc |
echo ' cmdEXE='$cmdEXE |
242 |
#exit |
#exit |
243 |
#--------------------------- |
#--------------------------- |
244 |
|
|
263 |
cp -p data.tst data |
cp -p data.tst data |
264 |
echo '==> START RUN 2 x' $Nit 'it' |
echo '==> START RUN 2 x' $Nit 'it' |
265 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
266 |
then ./$gcmExc std_outp.2it > TTT.out.2it 2>&1 |
then |
267 |
else ./$gcmExc > std_outp.2it |
sleep $wait2run |
268 |
|
$cmdEXE > TTT.out.2it 2>&1 |
269 |
|
mv STDOUT.0000 std_outp.2it |
270 |
|
cat TTT.out.2it |
271 |
|
else $cmdEXE > std_outp.2it |
272 |
fi |
fi |
273 |
mv STDERR.0000 std__err.2it |
mv STDERR.0000 std__err.2it |
274 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.2it` |
out=`egrep -c 'STOP ABNORMAL END' std_outp.2it` |
284 |
rnp_loc $xx.ckptA $xx.$num2c |
rnp_loc $xx.ckptA $xx.$num2c |
285 |
done |
done |
286 |
mv *.0??.0??.* $dir1 |
mv *.0??.0??.* $dir1 |
287 |
txtF=`ls -1 *.txt 2> /dev/null` |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
288 |
mv R?.data R?.meta DR?.data DR?.meta $txtF $dir1 |
mv R?.data R?.meta DR?.data DR?.meta PHref{C,F}.??ta $txtF $dir1 |
289 |
listS=`ls $dir1/pickup*.${num0c}* 2> /dev/null | head -1` |
listS=`ls $dir1/pickup*.${num0c}* 2> /dev/null | head -1` |
290 |
if test "x$listS" != x ; then mv $dir1/pickup*.${num0c}* . ; fi |
if test "x$listS" != x ; then mv $dir1/pickup*.${num0c}* . ; fi |
291 |
#--- 2nd run ( Nit iter , restart from same Iter = num0 as 1rst run ) |
#--- 2nd run ( Nit iter , restart from same Iter = num0 as 1rst run ) |
292 |
sed "s/^ nTimeSteps=$Dbl/ nTimeSteps=$Nit/g" data.tst > data |
sed "s/^ nTimeSteps=$Dbl/ nTimeSteps=$Nit/g" data.tst > data.tmp |
293 |
|
mv -f data.tmp data |
294 |
echo '==> START RUN 1iA' |
echo '==> START RUN 1iA' |
295 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
296 |
then ./$gcmExc std_outp.1iA > TTT.out.1iA 2>&1 |
then |
297 |
else ./$gcmExc > std_outp.1iA |
sleep $wait2run |
298 |
|
$cmdEXE > TTT.out.1iA 2>&1 |
299 |
|
mv STDOUT.0000 std_outp.1iA |
300 |
|
cat TTT.out.1iA |
301 |
|
else $cmdEXE > std_outp.1iA |
302 |
fi |
fi |
303 |
mv STDERR.0000 std__err.1iA |
mv STDERR.0000 std__err.1iA |
304 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iA` |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iA` |
312 |
rnp_loc $xx.ckptA $xx.$num1c |
rnp_loc $xx.ckptA $xx.$num1c |
313 |
done |
done |
314 |
mv *.0??.0??.* $dir2 |
mv *.0??.0??.* $dir2 |
315 |
txtF=`ls -1 *.txt 2> /dev/null` |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
316 |
mv R?.data R?.meta DR?.data DR?.meta $txtF $dir2 |
mv R?.data R?.meta DR?.data DR?.meta PHref{C,F}.??ta $txtF $dir2 |
317 |
listS=`ls $dir2/pickup*.${num0c}* 2> /dev/null | head -1` |
listS=`ls $dir2/pickup*.${num0c}* 2> /dev/null | head -1` |
318 |
if test "x$listS" != x ; then mv $dir2/pickup*.${num0c}* . ; fi |
if test "x$listS" != x ; then mv $dir2/pickup*.${num0c}* . ; fi |
319 |
#--- 3rd run ( Nit iter , restart from 2nd ) |
#--- 3rd run ( Nit iter , restart from 2nd ) |
322 |
mv -f data.tmp data |
mv -f data.tmp data |
323 |
echo '==> START RUN 1iB' |
echo '==> START RUN 1iB' |
324 |
if [ $mpi -ge 1 ] |
if [ $mpi -ge 1 ] |
325 |
then ./$gcmExc std_outp.1iB > TTT.out.1iB 2>&1 |
then |
326 |
else ./$gcmExc > std_outp.1iB |
sleep $wait2run |
327 |
|
$cmdEXE > TTT.out.1iB 2>&1 |
328 |
|
mv STDOUT.0000 std_outp.1iB |
329 |
|
cat TTT.out.1iB |
330 |
|
else $cmdEXE > std_outp.1iB |
331 |
fi |
fi |
332 |
mv STDERR.0000 std__err.1iB |
mv STDERR.0000 std__err.1iB |
333 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iB` |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iB` |
341 |
rnp_loc $xx.ckptA $xx.$num2c |
rnp_loc $xx.ckptA $xx.$num2c |
342 |
done |
done |
343 |
mv *.0??.0??.* $dir3 |
mv *.0??.0??.* $dir3 |
344 |
txtF=`ls -1 *.txt 2> /dev/null` |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
345 |
mv R?.data R?.meta DR?.data DR?.meta $txtF $dir3 |
mv R?.data R?.meta DR?.data DR?.meta PHref{C,F}.??ta $txtF $dir3 |
346 |
listS=`ls $dir3/pickup*.${num0c}* 2> /dev/null | head -1` |
listS=`ls $dir3/pickup*.${num0c}* 2> /dev/null | head -1` |
347 |
if test "x$listS" != x ; then mv $dir3/pickup*.${num0c}* . ; fi |
if test "x$listS" != x ; then mv $dir3/pickup*.${num0c}* . ; fi |
348 |
rm $dir3/pickup*.$num1c.* |
rm $dir3/pickup*.$num1c.* |
367 |
|
|
368 |
if test $flag = '3' -o $flag = 'All' ; then |
if test $flag = '3' -o $flag = 'All' ; then |
369 |
#---------------------------------------------------------------------- |
#---------------------------------------------------------------------- |
370 |
echo ' ' |
echo ' ' |
371 |
echo "== diff pickup files : end of 1rst run (2x$Nit it) & end of 3rd run (2nd $Nit it)" |
echo "== diff pickup files : end of 1rst run (2x$Nit it) & end of 3rd run (2nd $Nit it)" |
372 |
listP=`( cd $dir1 ; ls pickup*.$num2c* | sed 's/\..*//g' | uniq )` |
listP=`( cd $dir1 ; ls pickup*.$num2c* | sed 's/\..*//g' | uniq )` |
373 |
chkmeta='Y' |
chkmeta='Y' |
389 |
echo ' diff' $dir1/$zz.$xx.meta $dir3 |
echo ' diff' $dir1/$zz.$xx.meta $dir3 |
390 |
diff $dir1/$zz.$xx.meta $dir3 |
diff $dir1/$zz.$xx.meta $dir3 |
391 |
out=$? |
out=$? |
392 |
if test $out != 0 ; then |
if test $out != 0 ; then |
393 |
echo 'Diff outp=' $out ' ==> stop' |
echo 'Diff outp=' $out ' ==> stop' |
394 |
exit 5 |
exit 5 |
395 |
fi |
fi |
401 |
|
|
402 |
if test $flag = '4' -o $flag = 'All' ; then |
if test $flag = '4' -o $flag = 'All' ; then |
403 |
#------------- |
#------------- |
404 |
echo ' ' |
echo ' ' |
405 |
echo "== clean-up and reset :" |
echo "== clean-up and reset :" |
406 |
echo '--> remove output dir and output files:' |
echo '--> remove output dir and output files:' |
407 |
echo ' rm -rf' $dir1 $dir2 $dir3 |
echo ' rm -rf' $dir1 $dir2 $dir3 |
408 |
rm -rf $dir1 $dir2 $dir3 |
rm -rf $dir1 $dir2 $dir3 |
409 |
echo ' rm -f' std_????.?i? |
echo ' rm -f' std_????.?i? |
410 |
rm -f std_????.?i? |
rm -f std_????.?i? |
411 |
|
rm -f TTT.out.?i? 2> /dev/null |
412 |
echo ' rm -f' data.pkg data.tst |
echo ' rm -f' data.pkg data.tst |
413 |
rm -f data.pkg data.tst |
rm -f data.pkg data.tst |
414 |
#- move back files from temp_tst dir: |
#- move back files from temp_tst dir: |