/[MITgcm]/MITgcm/tools/do_tst_2+2
ViewVC logotype

Annotation of /MITgcm/tools/do_tst_2+2

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.22 - (hide annotations) (download)
Fri Oct 7 17:03:49 2016 UTC (7 years, 5 months ago) by jmc
Branch: MAIN
CVS Tags: checkpoint66g, checkpoint66f, checkpoint66e, checkpoint66d, checkpoint66c, checkpoint66b, checkpoint66a, checkpoint66o, checkpoint66n, checkpoint66m, checkpoint66l, checkpoint66k, checkpoint66j, checkpoint66i, checkpoint66h, HEAD
Changes since 1.21: +3 -3 lines
keep tar file if something went wrong when sending email (like in testreport)

1 jmc 1.2 #! /usr/bin/env bash
2 jmc 1.1
3 jmc 1.22 # $Header: /u/gcmpack/MITgcm/tools/do_tst_2+2,v 1.21 2015/11/18 22:18:41 jmc Exp $
4 jmc 1.1 # $Name: $
5    
6     usage()
7     {
8     echo "Usage: `basename $0` [OPTIONS]"
9     echo " -> perform test 2+2=4 for several experiments"
10     echo ""
11     echo "where possible OPTIONS are:"
12     echo " -help : print usage"
13     echo " -t LIST_EXP : only check experiments in LIST_EXP"
14     echo " -skd LIST_EXP : skip experiments in LIST_EXP"
15 jmc 1.3 echo " -exe COMMAND : use COMMAND to run the tests"
16     echo " -mpi : run the tests using MPI"
17 jmc 1.10 echo " -mf STRING :(MPI) file with list of possible machines to run on"
18 jmc 1.2 echo " -o STRING : used to build output directory name"
19 jmc 1.1 echo " (DEF=\"hostname\")"
20 jmc 1.2 echo " -a,-A STRING : email address to send output to"
21 jmc 1.1 echo " (DEF=\"\" no email is sent)"
22 jmc 1.2 echo " (-A: + save each log file)"
23 jmc 1.17 echo " -send STRING : sending command (instead of using mpack)"
24     echo " -sd STRING : location to save output tar file to send (DEF='$SAVDIR')"
25 jmc 1.1 echo " -clean : clean output & reset"
26     exit
27     }
28    
29     CLEAN=0
30     TESTDIRS=
31     SKIPDIRS=
32 jmc 1.3 SCRIPT='../tools/tst_2+2'
33 jmc 1.13 LOGFILE='tst_2+2_out.log'
34 jmc 1.5 scrArg='All'
35 jmc 1.20 bldDir='build'
36 jmc 1.3 COMMAND=
37     mpi=0
38 jmc 1.10 MPI_MFILE=
39 jmc 1.19 OUTDIR=`hostname -s | tr '[:upper:]' '[:lower:]'`
40 jmc 1.2 SAVELOG=0
41 jmc 1.1 ADDRESS=
42     MPACK="../tools/mpack-1.6/mpack"
43 jmc 1.17 SENDCMD=
44     SAVDIR='.'
45 jmc 1.1 here=`pwd`
46     yy=
47     for xx
48     do
49     if test -n "$yy"; then
50     eval "$yy=\$xx"
51     yy=
52     else
53     case $xx in
54     -help ) usage ;;
55     -clean) CLEAN=1 ;;
56     -t ) yy=TESTDIRS ;;
57     -skd ) yy=SKIPDIRS ;;
58 jmc 1.3 -exe ) yy=COMMAND ;;
59     -mpi ) mpi=1 ;;
60 jmc 1.10 -mf ) yy=MPI_MFILE ;;
61 jmc 1.1 -a ) yy=ADDRESS ;;
62 jmc 1.2 -A ) yy=ADDRESS ; SAVELOG=1 ;;
63 jmc 1.17 -send ) yy=SENDCMD ;;
64     -sd ) yy=SAVDIR ;;
65 jmc 1.1 -o ) yy=OUTDIR ;;
66     *) echo "Error: unrecognized option: "$xx ; usage ; exit ;;
67     esac
68     fi
69     done
70     #------------------------
71    
72 jmc 1.2 if test "x$TESTDIRS" = x ; then TESTDIRS=`ls ` ; fi
73 jmc 1.1 LIST=""
74     for xx in $TESTDIRS
75     do
76     yy=`echo $SKIPDIRS | grep -c $xx`
77     if test $yy = 0 ; then
78 jmc 1.2 if test -f $xx/results/output.txt ; then
79 jmc 1.1 LIST=${LIST}" "$xx
80     # else
81     # echo ""; echo -n " -- skip \"$xx\" (not a directory !)"
82     fi
83     # else
84     # echo ""; echo -n " -- skip \"$xx\" (excluded)"
85     fi
86     done
87    
88 jmc 1.16 if test -x $SCRIPT ; then
89     if test $CLEAN = 1 ; then
90     echo "clean output from script '$SCRIPT' for experiment in:"
91     else
92     echo "run script '$SCRIPT' for experiment in:"
93     fi
94     echo " $LIST"
95     echo ""
96     yy=`echo $SCRIPT | grep -c '^\/'`
97     if test $yy = 0 ; then SCRIPT="../../$SCRIPT" ; fi
98     else
99     echo "ERROR: script '$SCRIPT' not found or not executable"
100     exit
101     fi
102 jmc 1.1 #------------------------
103 jmc 1.16
104 jmc 1.1 if test $CLEAN = 1 ; then
105     for xx in $LIST
106     do
107     listD='run '`(cd $xx/results ; ls output.*.txt 2> /dev/null | sed 's/^output\./tr_run./g' | sed 's/\.txt$//g' )`
108     for yy in $listD
109     do
110 jmc 1.2 if test -r $xx/$yy/data.tst ; then
111 jmc 1.1 echo "clean dir:" $xx/$yy
112 jmc 1.2 cd $xx/$yy
113 jmc 1.12 echo ' ' >> $LOGFILE
114     echo $SCRIPT 4 >> $LOGFILE
115     $SCRIPT 4 >> $LOGFILE 2>&1
116 jmc 1.1 cd $here
117     fi
118     done
119     done
120     exit
121     fi
122     #------------------------
123     # Create a uniquely named directory to store results
124 jmc 1.10 CMDLINE=$0
125     for xx in "$@" ; do nw=`echo $xx | wc -w`
126     if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
127     else CMDLINE="$CMDLINE '$xx'" ; fi
128     done
129 jmc 1.14 DATE=''
130     if test -f tr_out.txt ; then
131     # try to use the date corresponding to 'Start time:' in file "tr_out.txt" :
132     sTime=`grep '^Start time: ' tr_out.txt 2>/dev/null | sed 's/Start time: //'`
133     if test "x$sTime" != x ; then DATE=`date -d "$sTime" "+%Y%m%d" 2>/dev/null` ; fi
134     fi
135     # otherwise, use current date:
136     if test "x$DATE" = x ; then DATE=`date +%Y%m%d`; fi
137 jmc 1.1 BASE="rs_"$OUTDIR"_"$DATE"_"
138     xx=0
139     DRESULTS="$BASE$xx"
140     while test -e $DRESULTS ; do
141     xx=$(( $xx + 1 ))
142     DRESULTS="$BASE$xx"
143     done
144     mkdir $DRESULTS
145     out=$?
146     if test "x$out" != x0 ; then
147     echo "ERROR: Can't create results directory \"./$DRESULTS\""
148     exit 1
149     fi
150     SUMMARY="$DRESULTS/summary.txt"
151     OPTFILE=
152    
153     printf "Start time: " > $SUMMARY
154     date >> $SUMMARY
155 jmc 1.10 echo 'run:' $CMDLINE >> $SUMMARY
156 jmc 1.1 if test -f tr_out.txt ; then
157     echo ' using output from:' >> $SUMMARY
158 jmc 1.11 sed -n '2,/^ OPTFILE=/ p' tr_out.txt >> $SUMMARY
159 jmc 1.1 echo >> $SUMMARY
160     fi
161     echo 'test 2+2=4 summary :' >> $SUMMARY
162     echo >> $SUMMARY
163     echo 'P. Run Result experiment' >> $SUMMARY
164     echo ' 1 2 3' >> $SUMMARY
165    
166 jmc 1.3 #-- For MPI test:
167 jmc 1.10 LOC_MFILE='tr_mpi_mfile'
168 jmc 1.3 RUNOUTP="output.txt"
169     if [ $mpi -ge 1 ] ; then
170     SCRIPT="$SCRIPT -mpi"
171     RUNOUTP="STDOUT.0000"
172 jmc 1.18 if test "x$COMMAND" = x ; then COMMAND="mpirun -v -np TR_NPROC ./mitgcmuv" ; fi
173 jmc 1.3 fi
174    
175 jmc 1.1 for xx in $LIST
176     do
177 jmc 1.15 echo '------------------------------------------------------------------------------'
178 jmc 1.10 if [ $mpi -le 0 ] ; then
179     rCommand=$COMMAND
180     else
181     LOC_NPROC=2
182 jmc 1.20 mpi_size=$xx/$bldDir/SIZE.h.mpi
183 jmc 1.8 if test -f $mpi_size ; then
184     px=`grep '^ & *nPx *=' $mpi_size | sed 's/^ & *nPx *= *//' | sed 's/, *$//'`
185     py=`grep '^ & *nPy *=' $mpi_size | sed 's/^ & *nPy *= *//' | sed 's/, *$//'`
186     pp=`expr $px \* $py` > /dev/null 2>&1 ; out=$?
187 jmc 1.10 if test "x$out" = x0 ; then LOC_NPROC=$pp ; fi
188     fi
189     rCommand=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"`
190     if test "x$MPI_MFILE" != x ; then
191     #- create new MPI machine-file with the right number of Procs
192     rm -f $LOC_MFILE
193     cat $MPI_MFILE | sort | uniq | head -$LOC_NPROC > $LOC_MFILE
194     nl=`wc -l $LOC_MFILE | awk '{print $1}'`
195     if [ $nl -lt $LOC_NPROC ] ; then
196     rm -f $LOC_MFILE
197     cat $MPI_MFILE | head -$LOC_NPROC > $LOC_MFILE
198     fi
199     rCommand=`echo $rCommand | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"`
200 jmc 1.8 fi
201     fi
202 jmc 1.10
203 jmc 1.1 listD='run '`(cd $xx/results ; ls output.*.txt 2> /dev/null | sed 's/^output\./tr_run./g' | sed 's/\.txt$//g' )`
204     for yy in $listD
205     do
206 jmc 1.3 if test -f $xx/$yy/$RUNOUTP ; then
207 jmc 1.20 if test "x$OPTFILE" = x -a -f $xx/$bldDir/Makefile ; then
208     comm=`grep '^# OPTFILE=' $xx/$bldDir/Makefile 2>/dev/null | head -1 | sed 's/^# //'`
209     echo "from '$xx/$bldDir/Makefile', extract:" > $DRESULTS/genmake_state
210     sed -n '/^# executed by:/,+1 p' $xx/$bldDir/Makefile >> $DRESULTS/genmake_state
211 jmc 1.7 echo " $comm" >> $DRESULTS/genmake_state
212 jmc 1.1 eval $comm
213 jmc 1.20 gmkLog=$xx/$bldDir/genmake.log
214 jmc 1.7 grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1
215     out=$?
216     if test "x$out" = x0 ; then
217     echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state
218     sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \
219     | grep -v '^... compiler version ' > tr_vers.tmp_log
220     sed -n '1,/^$/p' tr_vers.tmp_log | sed '/^$/d' | sed 's/^./ &/' \
221     >> $DRESULTS/genmake_state
222     rm -f tr_vers.tmp_log
223     fi
224 jmc 1.1 fi
225 jmc 1.2 if test $yy = 'run' ; then nam=$xx
226 jmc 1.1 else nam=$xx`echo $yy | sed 's/tr_run//'` ; fi
227     echo -n "Entering $xx/$yy :"
228     cd $xx/$yy
229 jmc 1.12 pwd > $LOGFILE
230 jmc 1.3 if test "x$COMMAND" = x ; then
231 jmc 1.12 echo $SCRIPT $scrArg >> $LOGFILE ; echo ' ' >> $LOGFILE
232     $SCRIPT $scrArg >> $LOGFILE 2>&1
233 jmc 1.3 out=$?
234     else
235 jmc 1.12 echo "$SCRIPT $scrArg -command \"$rCommand\"" >> $LOGFILE
236     echo ' ' >> $LOGFILE
237     $SCRIPT $scrArg -command "$rCommand" >> $LOGFILE 2>&1
238 jmc 1.3 out=$?
239     fi
240 jmc 1.1 case $out in
241     0 ) echo ' pass test 2+2=4'
242     echo "Y Y Y Y pass <- $nam" >> ../../$SUMMARY ;;
243     4|5) echo "Y Y Y Y FAIL ($out) - $nam" >> ../../$SUMMARY ;;
244     3 ) echo "Y Y Y N FAIL ($out) - $nam" >> ../../$SUMMARY ;;
245     2 ) echo "Y Y N N FAIL ($out) - $nam" >> ../../$SUMMARY ;;
246     1 ) echo "Y N N N FAIL ($out) - $nam" >> ../../$SUMMARY ;;
247     * ) echo "N N N N FAIL ($out) - $nam" >> ../../$SUMMARY ;;
248     esac
249 jmc 1.2 if test $out != '0' ; then
250 jmc 1.1 echo " test 2+2=4 FAIL (exit $out)"
251     echo " "
252 jmc 1.12 tail -5 $LOGFILE
253 jmc 1.1 echo " "
254 jmc 1.12 cp -p $LOGFILE ../../$DRESULTS/$nam.log
255 jmc 1.2 elif test $SAVELOG = 1 ; then
256 jmc 1.12 cp -p $LOGFILE ../../$DRESULTS/$nam.log
257 jmc 1.1 fi
258     cd $here
259     fi
260     done
261 jmc 1.10 if test "x$mpi" != x0 -a "x$MPI_MFILE" != x ; then rm -f $LOC_MFILE ; fi
262 jmc 1.1 done
263     printf "End time: " >> $SUMMARY
264     date >> $SUMMARY
265    
266 jmc 1.17 # If address was supplied, then send email using Send-Command or mpack.
267 jmc 1.1 if test "x$ADDRESS" != xNONE -a "x$ADDRESS" != x ; then
268 jmc 1.17 if test "x$SENDCMD" = x ; then
269     # if Send-Command is empty and mpack exist, then send email using mpack.
270     if test -x $MPACK ; then SENDCMD=$MPACK
271     else
272 jmc 1.21 echo
273 jmc 1.17 echo "Warning: $MPACK is not executable => no email was sent"
274     fi
275     fi
276     if test "x$SENDCMD" != x ; then
277     #echo " run: $SENDCMD -s MITgcm-test -m 3555000 ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESS"
278     tar -cf ${SAVDIR}/${DRESULTS}".tar" $DRESULTS > /dev/null 2>&1 \
279     && gzip ${SAVDIR}/${DRESULTS}".tar" \
280     && $SENDCMD -s MITgcm-test -m 3555000 ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESS
281 jmc 1.1 out=$?
282     if test "x$out" != x0 ; then
283     echo
284     echo "Warning: The tar, gzip, & mpack step failed. Please send email"
285     echo " to <MITgcm-support@mitgcm.org> for help. You may copy the "
286     echo " summary of results from the directory \"$DRESULTS\"."
287     else
288     echo
289     echo "An email containing results was sent to the following address:"
290     echo " \"$ADDRESS\""
291 jmc 1.22 test -f ${SAVDIR}/${DRESULTS}".tar" && rm -f ${SAVDIR}/${DRESULTS}".tar"
292     test -f ${SAVDIR}/${DRESULTS}".tar.gz" && rm -f ${SAVDIR}/${DRESULTS}".tar.gz"
293 jmc 1.1 fi
294     fi
295     fi
296 jmc 1.21 echo '' ; echo '===== Summary (tst_2+2_out.txt): ======'
297     cat $SUMMARY
298 jmc 1.1
299     # save output and remove output Dir (if no address was provided)
300     # note: to keep the output Dir without sending email, use option "-a NONE"
301     if test -f tst_2+2_out.txt ; then mv -f tst_2+2_out.txt tst_2+2_out.old ; fi
302     cp -p $SUMMARY tst_2+2_out.txt
303     if test "x$ADDRESS" = x ; then
304     rm -rf $DRESULTS
305     fi

  ViewVC Help
Powered by ViewVC 1.1.22