1 |
#! /usr/bin/env bash |
2 |
|
3 |
# $Header: /u/gcmpack/MITgcm/tools/tst_2+2,v 1.5 2009/03/03 20:59:40 jmc Exp $ |
4 |
# $Name: $ |
5 |
|
6 |
rnp_loc() |
7 |
{ |
8 |
# rnp_loc arg_1 arg_2 |
9 |
# rename files with prefix = 'arg_1' to files with prefix = 'arg_2' |
10 |
echo ' rnp_loc:' $1 $2 |
11 |
# rnp -s $1 $2 |
12 |
listY=`ls -1 $1*` |
13 |
for yy in $listY |
14 |
do |
15 |
zz=`echo $yy | sed "s/^$1/$2/"` |
16 |
mv $yy $zz |
17 |
done |
18 |
} |
19 |
|
20 |
#--------------------------- |
21 |
|
22 |
#- exit status: |
23 |
# 0 -> test is successful |
24 |
# 1 -> fail in 1rst run |
25 |
# 2 -> fail in 2nd run |
26 |
# 3 -> fail in 3rd run |
27 |
# 4 -> pickup*.data (written at the end of 2+2 & 4 it run) are different |
28 |
# 5 -> pickup*.meta (written at the end of 2+2 & 4 it run) are different |
29 |
# 6 -> missing 'data.tst' file (removed or not yet generated ?) |
30 |
# 7 -> missing standard output from previous run |
31 |
# 8 -> missing information (in standard output of previous run) about pickup |
32 |
# 9 -> error in argument list |
33 |
|
34 |
#- default: |
35 |
Nit=2 |
36 |
prt=1 # 1 = print some informations ; 2 = print more informations |
37 |
action=1 |
38 |
cmdMV='mv -f' |
39 |
tmpDir='temp_tst' |
40 |
wait2run=0 # if needs sometime for file system syncro on all nodes |
41 |
|
42 |
#- default executable, command-line , using MPI: |
43 |
gcmEXE=./mitgcmuv |
44 |
cmdEXE= |
45 |
mpi=0 |
46 |
|
47 |
nArg=$# |
48 |
prev= |
49 |
for xx ; do |
50 |
#echo "xx='$xx'" |
51 |
# If the previous option needs an argument, assign it. |
52 |
if test -n "$prev"; then |
53 |
eval "$prev=\$xx" |
54 |
prev= |
55 |
nArg=`expr $nArg - 2` |
56 |
continue |
57 |
fi |
58 |
|
59 |
case $xx in |
60 |
-mpi ) mpi=1 ; nArg=`expr $nArg - 1` ;; |
61 |
-exe ) prev=gcmEXE ;; |
62 |
-command ) prev=cmdEXE ;; |
63 |
*) flag=$xx ;; |
64 |
esac |
65 |
done |
66 |
#echo "cmdEXE='$cmdEXE'" |
67 |
#echo "nArg=$nArg ; mpi=$mpi ; flag='$flag'" |
68 |
|
69 |
case $nArg in |
70 |
1) ;; |
71 |
*) echo "Usage: `basename $0` flag [-mpi] [-exe EXECUTABLE] [-command COMMAND]" |
72 |
echo " Check restart: compare 1 run of 2 x $Nit it long" |
73 |
echo " with 2 consecutive runs of $Nit it long each" |
74 |
echo " where: flag = 0 -> prepare (from a previous run) pickup & data files" |
75 |
echo " flag = 1 -> do the 3 runs (using COMMAND" |
76 |
echo " or simply using EXECUTABLE, default=$gcmEXE)" |
77 |
echo " flag = 2 -> compare std_outp" |
78 |
echo " flag = 3 -> compare pickup files" |
79 |
echo " flag = 4 -> clean-up output files" |
80 |
echo " flag = All -> do 0,1,2,3,4 sequentially" |
81 |
exit 9 ;; |
82 |
esac |
83 |
|
84 |
#-- set the run-command : |
85 |
if test "x$cmdEXE" = x |
86 |
then |
87 |
if [ $mpi -ge 1 ] ; then |
88 |
cmdEXE="mpirun -machinefile ../../mf -v -np 2 $gcmEXE" |
89 |
else |
90 |
cmdEXE=$gcmEXE |
91 |
fi |
92 |
fi |
93 |
echo "cmdEXE='$cmdEXE'" |
94 |
#========================= |
95 |
|
96 |
if test $flag = '0' -o $flag = 'All' ; then |
97 |
#---------------------------------------------------------------------- |
98 |
# last pickup written (from previous run): |
99 |
if [ $mpi -ge 1 ] |
100 |
then outpRun='STDOUT.0000' |
101 |
else outpRun='output.txt' |
102 |
fi |
103 |
if test -r $outpRun |
104 |
then |
105 |
# grep ' %CHECKPOINT ' $outpRun | tail -1 |
106 |
lastPick=`grep ' %CHECKPOINT ' $outpRun | sed 's/^.*%CHECKPOINT //' | tail -1` |
107 |
if test "x$lastPick" = x |
108 |
then |
109 |
echo "no pickup information written in '$outpRun' from previous run" |
110 |
exit 8 |
111 |
else |
112 |
iter=`echo $lastPick | awk '{print $1}'` |
113 |
zz=`echo $lastPick | awk '{print $2}'` |
114 |
echo "from previous run std.outp, lastPick='$lastPick' ; iter='$iter' ; zz='$zz'" |
115 |
fi |
116 |
else |
117 |
echo "no standard output file '$outpRun' from previous run" |
118 |
exit 7 |
119 |
fi |
120 |
|
121 |
#- rename pickup (if needed) |
122 |
sufx=`printf "%10.10i\n" $iter` |
123 |
if test $zz = $sufx |
124 |
then |
125 |
echo "no need to change pickup sufix" |
126 |
else |
127 |
# echo "rename pickup sufix: $zz -> $sufx" |
128 |
listP=`ls pickup*.${zz}* 2> /dev/null | sed 's/\..*//g' | uniq` |
129 |
if test "x$listP" = x |
130 |
then echo ' rename' $zz '->' $sufx ': no file to rename' |
131 |
else |
132 |
if [ $prt -ge 1 ] ; then echo ' rename' $zz '->' $sufx 'for all:' $listP ; fi |
133 |
for yy in $listP |
134 |
do |
135 |
listF=`ls -1 $yy.${zz}*` |
136 |
for xx in $listF |
137 |
do |
138 |
new=`echo $xx | sed "s/^$yy\.$zz/$yy.$sufx/"` |
139 |
if [ $prt -ge 2 ] ; then echo ' '$cmdMV $xx $new ; fi |
140 |
if test $action = 1 ; then $cmdMV $xx $new ; fi |
141 |
done |
142 |
done |
143 |
fi |
144 |
fi |
145 |
#------------------------- |
146 |
echo "prepare parameter file 'data.tst' :" |
147 |
cp -p -f data data.tst |
148 |
listOut='startTime endTime nIter0 nTimeSteps nEndIter chkPtFreq pChkPtFreq globalFiles' |
149 |
listOut=$listOut' pickupSuff usePickupBeforeC54 mdsioLocalDir' |
150 |
#- comment out (in data.tst) all parameters from list: 'listOut' |
151 |
for yy in $listOut |
152 |
do |
153 |
if [ $prt -ge 2 ] ; then echo -n ' commenting out:' $yy ; fi |
154 |
nn=`grep -i -c "^ *$yy *=" data.tst` |
155 |
if [ $nn -ge 1 ] |
156 |
then |
157 |
listN=`grep -i -n "^ *$yy *=" data.tst | sed 's/:.*$//'` |
158 |
if [ $prt -ge 2 ] ; then echo " : found on line: $listN" ; fi |
159 |
for ll in $listN |
160 |
do |
161 |
sed "$ll s/^./#&/" data.tst > data.tmp_$$ |
162 |
# diff data.tmp_$$ data.tst |
163 |
mv -f data.tmp_$$ data.tst |
164 |
done |
165 |
else if [ $prt -ge 2 ] ; then echo ' : not found' ; fi |
166 |
fi |
167 |
done |
168 |
if [ $prt -ge 2 ] ; then echo ' ' ; fi |
169 |
# add nIter0 & nTimeSteps in namelist "PARM03": |
170 |
Dbl=`expr $Nit \* 2` |
171 |
sed "/^ *\&PARM03/a \ nTimeSteps=$Dbl," data.tst > data.tmp_$$ |
172 |
mv -f data.tmp_$$ data.tst |
173 |
sed "/^ *\&PARM03/a \ nIter0=$iter," data.tst > data.tmp_$$ |
174 |
mv -f data.tmp_$$ data.tst |
175 |
echo "prepare file 'data.tst' : done" |
176 |
if [ $prt -ge 1 ] ; then |
177 |
echo ' diff data.tst data' ; diff data.tst data ; echo ' ' ; |
178 |
fi |
179 |
#---- clean up current dir: move output to temp_tst dir: |
180 |
if test $action = 1 ; then |
181 |
rm -rf $tmpDir |
182 |
mkdir $tmpDir |
183 |
mv -f *.data $tmpDir |
184 |
mv -f *.meta $tmpDir |
185 |
mv -f *.txt $tmpDir 2> /dev/null |
186 |
mv -f STD???.0[0-9][0-9][0-9] $tmpDir |
187 |
listS=`ls $tmpDir/pickup*.${sufx}* 2> /dev/null | head -1` |
188 |
if test "x$listS" != x ; then mv $tmpDir/pickup*.${sufx}* . ; fi |
189 |
#- move main parameter file "data": |
190 |
mv -f data $tmpDir |
191 |
#- do not deal with MNC pickup at all: |
192 |
cp -p -f data.pkg data.tmp_$$ |
193 |
mv -f data.pkg $tmpDir |
194 |
mv -f data.tmp_$$ data.pkg |
195 |
yy='useMNC' |
196 |
if [ $prt -ge 2 ] ; then echo -n " commenting out: $yy in data.pkg" ; fi |
197 |
nn=`grep -i -c "^ *$yy *=" data.pkg` |
198 |
if [ $nn -ge 1 ] |
199 |
then |
200 |
listN=`grep -i -n "^ *$yy *=" data.pkg | sed 's/:.*$//'` |
201 |
if [ $prt -ge 2 ] ; then echo " : found on line: $listN" ; fi |
202 |
for ll in $listN |
203 |
do |
204 |
sed "$ll s/^./#&/" data.pkg > data.tmp_$$ |
205 |
if [ $prt -ge 1 ] ; then echo ' changes in data.pkg' |
206 |
diff data.tmp_$$ data.pkg ; echo ' ' ; fi |
207 |
mv -f data.tmp_$$ data.pkg |
208 |
done |
209 |
else if [ $prt -ge 2 ] ; then echo ' : not found' ; echo ' ' ; fi |
210 |
fi |
211 |
fi |
212 |
#---------------------------------------------------------------------- |
213 |
fi |
214 |
|
215 |
#========================= |
216 |
|
217 |
if test -r data.tst ; then |
218 |
#- extract length of runs & starting iter number from file "data.tst" |
219 |
num0=`sed -n 's/^ nIter0=//p' data.tst | sed 's/,//g' | sed 's/ //g'` |
220 |
Dbl=`sed -n 's/^ nTimeSteps=//p' data.tst | sed 's/,//g'` |
221 |
Nit=`expr $Dbl / 2` |
222 |
# Nit = Nb of iter of 2nd & 3rd run (1rst one is 2 x Nit iter long) |
223 |
else |
224 |
echo '=> Need a readable "data.tst" file for the 2 x Nit iterations run' |
225 |
exit 6 |
226 |
fi |
227 |
|
228 |
#- dir where to put the results : |
229 |
dir1=res_2it |
230 |
dir2=res_1iA |
231 |
dir3=res_1iB |
232 |
|
233 |
#--------------------------- |
234 |
num1=`expr $num0 + $Nit` |
235 |
num2=`expr $num0 + $Dbl` |
236 |
num0c=`printf "%10.10i\n" $num0` |
237 |
num1c=`printf "%10.10i\n" $num1` |
238 |
num2c=`printf "%10.10i\n" $num2` |
239 |
echo " start-end iter: $num0 , $num1 , $num2" |
240 |
echo " sufix: '$num0c' '$num1c' '$num2c'" |
241 |
echo ' cmdEXE='$cmdEXE |
242 |
#exit |
243 |
#--------------------------- |
244 |
|
245 |
if test $flag = '1' -o $flag = 'All' ; then |
246 |
#---------------------------------------------------------------------- |
247 |
#- check at the end of each run for pickup with prefix: |
248 |
pref='pickup' |
249 |
offLine=`grep -i -c '^ *useOffLine *= *\.TRUE\.' data.pkg` |
250 |
if [ $offLine -ge 1 ] ; then pref='pickup_ptracers' ; fi |
251 |
#--------------------------- |
252 |
if test -d $dir1 -o -d $dir2 -o -d $dir3 |
253 |
then echo -n 'remove dir: ' |
254 |
if test -d $dir1 ; then echo -n $dir1 ' , ' ; rm -r $dir1 ; fi |
255 |
if test -d $dir2 ; then echo -n $dir2 ' , ' ; rm -r $dir2 ; fi |
256 |
if test -d $dir3 ; then echo -n $dir3 ' , ' ; rm -r $dir3 ; fi |
257 |
echo ' ' |
258 |
fi |
259 |
mkdir $dir1 $dir2 $dir3 |
260 |
rm -f STDERR.0000 |
261 |
|
262 |
#--- 1rst run ( 2 x Nit iter ) |
263 |
cp -p data.tst data |
264 |
echo '==> START RUN 2 x' $Nit 'it' |
265 |
if [ $mpi -ge 1 ] |
266 |
then |
267 |
sleep $wait2run |
268 |
$cmdEXE > TTT.out.2it 2>&1 |
269 |
mv STDOUT.0000 std_outp.2it |
270 |
cat TTT.out.2it |
271 |
else $cmdEXE > std_outp.2it |
272 |
fi |
273 |
mv STDERR.0000 std__err.2it |
274 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.2it` |
275 |
if test $out != 0 |
276 |
then echo "==> RUN 2 x $Nit it : ABNORMAL END => exit" ; exit 1 ; fi |
277 |
listF=`ls -1 $pref.ckptA*.data 2> /dev/null` |
278 |
if test "x$listF" = x |
279 |
then echo "==> RUN 2 x $Nit it STOP without writing $pref => exit" ; exit 1 |
280 |
else echo "==> END RUN 2 x $Nit it" ; fi |
281 |
listP=`ls pickup*.ckptA* | sed 's/\..*//g' | uniq` |
282 |
echo ' listP=' $listP |
283 |
for xx in $listP |
284 |
do |
285 |
rnp_loc $xx.ckptA $xx.$num2c |
286 |
done |
287 |
mv *.data *.meta $dir1 |
288 |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
289 |
if test "x$txtF" != x ; then mv $txtF $dir1 ; fi |
290 |
listS=`ls $dir1/pickup*.${num0c}* 2> /dev/null | head -1` |
291 |
if test "x$listS" != x ; then mv $dir1/pickup*.${num0c}* . ; fi |
292 |
#--- 2nd run ( Nit iter , restart from same Iter = num0 as 1rst run ) |
293 |
sed "s/^ nTimeSteps=$Dbl/ nTimeSteps=$Nit/g" data.tst > data.tmp |
294 |
mv -f data.tmp data |
295 |
echo '==> START RUN 1iA' |
296 |
if [ $mpi -ge 1 ] |
297 |
then |
298 |
sleep $wait2run |
299 |
$cmdEXE > TTT.out.1iA 2>&1 |
300 |
mv STDOUT.0000 std_outp.1iA |
301 |
cat TTT.out.1iA |
302 |
else $cmdEXE > std_outp.1iA |
303 |
fi |
304 |
mv STDERR.0000 std__err.1iA |
305 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iA` |
306 |
if test $out != 0 |
307 |
then echo "==> RUN 1iA : ABNORMAL END => exit" ; exit 2 ; fi |
308 |
listF=`ls -1 $pref.ckptA*.data 2> /dev/null` |
309 |
if test "x$listF" = x |
310 |
then echo "==> RUN 1iA STOP without writing $pref => exit" ; exit 2 |
311 |
else echo "==> END RUN 1iA" ; fi |
312 |
for xx in $listP |
313 |
do |
314 |
rnp_loc $xx.ckptA $xx.$num1c |
315 |
done |
316 |
mv *.data *.meta $dir2 |
317 |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
318 |
if test "x$txtF" != x ; then mv $txtF $dir2 ; fi |
319 |
listS=`ls $dir2/pickup*.${num0c}* 2> /dev/null | head -1` |
320 |
if test "x$listS" != x ; then mv $dir2/pickup*.${num0c}* . ; fi |
321 |
#--- 3rd run ( Nit iter , restart from 2nd ) |
322 |
ln -s $dir2/pickup*.$num1c.* . |
323 |
sed "s/^ nIter0=$num0/ nIter0=$num1/g" data > data.tmp |
324 |
mv -f data.tmp data |
325 |
echo '==> START RUN 1iB' |
326 |
if [ $mpi -ge 1 ] |
327 |
then |
328 |
sleep $wait2run |
329 |
$cmdEXE > TTT.out.1iB 2>&1 |
330 |
mv STDOUT.0000 std_outp.1iB |
331 |
cat TTT.out.1iB |
332 |
else $cmdEXE > std_outp.1iB |
333 |
fi |
334 |
mv STDERR.0000 std__err.1iB |
335 |
out=`egrep -c 'STOP ABNORMAL END' std_outp.1iB` |
336 |
if test $out != 0 |
337 |
then echo "==> RUN 1iB : ABNORMAL END => exit" ; exit 3 ; fi |
338 |
listF=`ls -1 $pref.ckptA*.data 2> /dev/null` |
339 |
if test "x$listF" = x |
340 |
then echo "==> RUN 1iB STOP without writing $pref => exit" ; exit 3 |
341 |
else echo "==> END RUN 1iB" ; fi |
342 |
for xx in $listP |
343 |
do |
344 |
rnp_loc $xx.ckptA $xx.$num2c |
345 |
done |
346 |
mv *.data *.meta $dir3 |
347 |
txtF=`ls -1 *.txt STD???.00?? 2> /dev/null` |
348 |
if test "x$txtF" != x ; then mv $txtF $dir3 ; fi |
349 |
listS=`ls $dir3/pickup*.${num0c}* 2> /dev/null | head -1` |
350 |
if test "x$listS" != x ; then mv $dir3/pickup*.${num0c}* . ; fi |
351 |
rm $dir3/pickup*.$num1c.* |
352 |
rm data |
353 |
#---------------------------------------------------------------------- |
354 |
fi |
355 |
|
356 |
if test $flag = '2' -o $flag = 'All' ; then |
357 |
#---------------------------------------------------------------------- |
358 |
echo ' ' ; echo '== compare cg2d_init_res :' |
359 |
echo ' run 1iA:' |
360 |
grep "cg2d_init_res" std_outp.1iA \ |
361 |
| sed 's/.* cg2d_init_res =//' |
362 |
echo ' run 1iB:' |
363 |
grep "cg2d_init_res" std_outp.1iB \ |
364 |
| sed 's/.* cg2d_init_res =//' |
365 |
echo ' run 2it:' |
366 |
grep "cg2d_init_res" std_outp.2it \ |
367 |
| sed 's/.* cg2d_init_res =//' |
368 |
#---------------------------------------------------------------------- |
369 |
fi |
370 |
|
371 |
if test $flag = '3' -o $flag = 'All' ; then |
372 |
#---------------------------------------------------------------------- |
373 |
echo ' ' |
374 |
echo "== diff pickup files : end of 1rst run (2x$Nit it) & end of 3rd run (2nd $Nit it)" |
375 |
listP=`( cd $dir1 ; ls pickup*.$num2c* | sed 's/\..*//g' | uniq )` |
376 |
chkmeta='Y' |
377 |
for yy in $listP |
378 |
do |
379 |
listY=`(cd $dir1 ; ls $yy.$num2c*.data | sed "s/$yy\.$num2c//" | sed "s/data$//")` |
380 |
zz=$yy.$num2c |
381 |
echo '--> file='$zz', listY='$listY |
382 |
for xx in $listY |
383 |
do |
384 |
echo ' diff' $dir1/$zz${xx}data $dir3 |
385 |
diff $dir1/$zz${xx}data $dir3 |
386 |
out=$? |
387 |
if test $out != 0 ; then |
388 |
echo 'Diff outp=' $out ' ==> stop' |
389 |
exit 4 |
390 |
fi |
391 |
if test $chkmeta = 'Y' -a -f $dir1/$zz${xx}meta ; then |
392 |
echo ' diff' $dir1/$zz${xx}meta $dir3 |
393 |
diff $dir1/$zz${xx}meta $dir3 |
394 |
out=$? |
395 |
if test $out != 0 ; then |
396 |
echo 'Diff outp=' $out ' ==> stop' |
397 |
exit 5 |
398 |
fi |
399 |
fi |
400 |
done |
401 |
done |
402 |
#---------------------------------------------------------------------- |
403 |
fi |
404 |
|
405 |
if test $flag = '4' -o $flag = 'All' ; then |
406 |
#------------- |
407 |
echo ' ' |
408 |
echo "== clean-up and reset :" |
409 |
echo '--> remove output dir and output files:' |
410 |
echo ' rm -rf' $dir1 $dir2 $dir3 |
411 |
rm -rf $dir1 $dir2 $dir3 |
412 |
echo ' rm -f' std_????.?i? |
413 |
rm -f std_????.?i? |
414 |
rm -f TTT.out.?i? 2> /dev/null |
415 |
echo ' rm -f' data.pkg data.tst |
416 |
rm -f data.pkg data.tst |
417 |
#- move back files from temp_tst dir: |
418 |
if test -d $tmpDir ; then |
419 |
echo "--> move back files from 'temp_tst' dir." |
420 |
cd $tmpDir ; mv -f * .. ; cd .. |
421 |
rmdir $tmpDir |
422 |
fi |
423 |
#------------- |
424 |
fi |
425 |
exit 0 |