--- MITgcm/verification/testreport 2011/01/27 04:51:03 1.157 +++ MITgcm/verification/testreport 2011/05/18 22:03:57 1.163 @@ -1,6 +1,6 @@ #! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.157 2011/01/27 04:51:03 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.163 2011/05/18 22:03:57 jmc Exp $ # $Name: $ # @@ -238,12 +238,6 @@ listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"` fi if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi - #- report to this experiment local summary file --- - echo "MACH='$MACH'" > $locDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" - echo "DATE='$DATE'" >> $locDIR"/summary.txt" - grep '(PID\.TID 0000\.0001) n.. =' $1/$2/$OUTPUTFILE \ - | sed 's/(PID.TID 0000.0001) //' >> $locDIR"/summary.txt" echo "listVar='$listVar'" >> $locDIR"/summary.txt" #--- allargs="" @@ -430,7 +424,8 @@ find . -name "*.data" -exec rm {} \; find . -name "fort.*" -exec rm {} \; find . -type l -exec rm {} \; - rm -f $EXECUTABLE $RUNLOG *.txt STD* *diagnostics.log datetime + #- should remove executable only if sym-link (alredy done above) + rm -f $RUNLOG *.txt STD* *diagnostics.log datetime rm -rf mnc_test_* rm -f *_MIT_CE_000.opt0000 costfunction*0000 echo successful @@ -466,13 +461,14 @@ { # makemodel directory ( + mk_fail=0 if test "x$NOMAKE" = xt ; then cd $1; if test -x $EXECUTABLE ; then echo "make skipped!" else echo "no executable!" - return 2 + mk_fail=3 fi else cd $1; @@ -493,20 +489,32 @@ if test "x$RETVAL" != x0 ; then tail make.tr_log echo failed - cp make.tr_log genmake.log genmake.tr_log $CDIR + cp genmake.log genmake.tr_log $CDIR + tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail" rm -f $EXECUTABLE - return 1 + mk_fail=1 else echo successful fi + else + echo "no Makefile !" + mk_fail=2 fi fi + if test "x$ADM" = xt -a -f taf_ad.log ; then + head -1 taf_ad.log >> $CDIR"/summary.txt" + nerr=`grep -c 'TAF *.* ERROR ' taf_ad.log` + nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' taf_ad.log` + echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \ + >> $CDIR"/summary.txt" + fi + if test $mk_fail != 0 ; then return $mk_fail ; fi ) } mk_mpi_size() { - # mk_mpi_size input_file output_file proc_Nb + # mk_mpi_size input_file output_file proc_Nb prefer_to_split_X # # make new SIZE.h (=output_file) from SIZE.h_mpi (=input_file) # for an MPI build with no more than proc_Nb processors ; @@ -515,6 +523,7 @@ inp=$1 out=$2 np=$3 + dirX=$4 tmp=TTT.$$ px=`grep "^ & *nPx *=" $inp | sed "s/^ & *nPx *= *//" | sed 's/, *$//'` @@ -532,6 +541,13 @@ if [ `expr $py % $j` -eq 0 ] ; then ij=`expr $i \* $j` if [ $ij -gt $pp ] ; then + flag=1 + elif [ $ij -eq $pp ] ; then + flag=$dirX + else + flag=0 + fi + if test $flag = 1 ; then if [ $ij -le $np ] ; then ix=$i ; jy=$j ; pp=$ij #echo " ix,jy= $ix,$jy" @@ -682,6 +698,15 @@ ( cd $1 printf 'runmodel in %s ... ' $1 + if test "x$MPI" != x0 ; then + #- adjust the MPI run command with the right number of Procs + #echo '' ; echo " COMMAND='$COMMAND'" + COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"` + if test "x$MPI_MFILE" != x ; then + COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"` + fi + #echo " COMMAND='$COMMAND'" + fi if test -L $EXECUTABLE ; then if test -x "../"$builddir"/"$EXECUTABLE ; then cmp $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1 @@ -693,15 +718,6 @@ echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp ln -sf "../"$builddir"/"$EXECUTABLE . fi - if test "x$MPI" != x0 ; then - #- adjust the MPI run command with the right number of Procs - #echo '' ; echo " COMMAND='$COMMAND'" - COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"` - if test "x$MPI_MFILE" != x ; then - COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"` - fi - #echo " COMMAND='$COMMAND'" - fi if test ! -x $EXECUTABLE ; then rm -f $RUNLOG ; touch $RUNLOG if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi @@ -711,7 +727,7 @@ else if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then # output do not exist or is older than executable: - rm -f $RUNLOG ; touch $RUNLOG + rm -f $OUTPUTFILE $RUNLOG ; touch $RUNLOG if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi ( eval $COMMAND ) >> $RUNLOG 2>&1 RETVAL=$? @@ -730,13 +746,19 @@ fi fi rm -f run.log_tmp + #- in all cases where OutputFile exists, report SIZE (and AD time) + if test -f $OUTPUTFILE ; then + grep '(PID\.TID 0000\.0001) n.. =' $OUTPUTFILE \ + | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" + if test "x$ADM" = xt ; then + grep -A3 'Seconds in section "ALL' $OUTPUTFILE >> $CDIR"/summary.txt" + fi + fi + if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then echo successful printf '=> output from running in %s :\n' $1 1>&2 tail $RUNLOG | sed 's/^.*/> &/g' 1>&2 - # === Reduce the size of the testing emails! - #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE - if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi return 0 elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then #-- for some weird cases (run is finihed but with error code) @@ -749,7 +771,6 @@ printf '=> output from running in %s :\n' $1 1>&2 tail $RUNLOG | sed 's/^.*/> &/g' 1>&2 cp $RUNLOG $CDIR"/"$RUNLOG - if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi return 1 fi ) @@ -851,27 +872,38 @@ check_eedata() { - # check_eedata size.h eedata + # check_eedata eedata size.h if [ $# -eq 2 ] ; then - if test -f $1 -a -f $2 ; then - sx=`grep "^ & *nSx *=" $1 | sed "s/^ & *nSx *=//" | sed 's/, *$//'` - nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"` + if test -f $1 -a -f $2 ; then + nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"` + sx=`grep "^ & *nSx *=" $2 | sed "s/^ & *nSx *=//" | sed 's/, *$//'` if test "x$nx" = x ; then rx=10 else rx=`expr $sx % $nx` fi - sy=`grep "^ & *nSy *=" $1 | sed "s/^ & *nSy *=//" | sed 's/, *$//'` - ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"` + ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"` + sy=`grep "^ & *nSy *=" $2 | sed "s/^ & *nSy *=//" | sed 's/, *$//'` if test "x$ny" = x ; then ry=20 else ry=`expr $sy % $ny` fi echo `expr $rx + $ry` - else + else echo '-1' - fi + fi + elif [ $# -eq 1 ] ; then + if test -f $1 ; then + nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"` + if test "x$nx" = x ; then nx=1 ; fi + ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"` + if test "x$ny" = x ; then ny=1 ; fi + #echo $nx $ny + echo $nx + else + echo '-1' + fi else echo '-2' fi @@ -884,6 +916,7 @@ # Default properties debug=0 verbose=1 +NBLINES_MKLOG=16000 IEEE=true if test "x$MITGCM_IEEE" != x ; then @@ -1315,26 +1348,35 @@ # Check for specific files for particular type of run + if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" != "x0" ; then + echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" + continue + fi + if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then + echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir" + continue + fi + if test "x$MPI" != "x0" ; then - if test -r $CODE_DIR"/SIZE.h_mpi" ; then - #- create new SIZE.h with no more than '$MPI' Procs - mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI - LOC_NPROC=$? - ( cd $BUILD_DIR - if test -r SIZE.h.mpi ; then - cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$? - else RETVAL=1 - fi - if test "x$RETVAL" = x0 ; then - rm -f tr_size.mpi - else - rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi - fi - ) - else - echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" - continue - fi + prefer_X=0 + if test "x$MULTI_THREAD" = "xt" ; then + retv=`check_eedata $dir"/input/eedata.mth"` + if test $retv = 1 ; then prefer_X=1 ; fi + fi + #- create new SIZE.h with no more than '$MPI' Procs + mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI $prefer_X + LOC_NPROC=$? + ( cd $BUILD_DIR + if test -r SIZE.h.mpi ; then + cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$? + else RETVAL=1 + fi + if test "x$RETVAL" = x0 ; then + rm -f tr_size.mpi + else + rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi + fi + ) if test "x$MPI_MFILE" != x ; then #- create new MPI machine-file with the right number of Procs rm -f $LOC_MFILE @@ -1350,16 +1392,12 @@ echo " new LOC_MFILE=$LOC_MFILE : $nl procs for LOC_NPROC=$LOC_NPROC" fi fi - fi - if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then - echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir" - continue - fi - if test "x$MPI" != "x0" -a "x$MULTI_THREAD" = "xt" ; then - retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/input/eedata.mth"` - if test $retv != 0 ; then - echo "input/eedata.mth tiling misfit -- skipping $dir" - continue + if test "x$MULTI_THREAD" = "xt" ; then + retv=`check_eedata $dir"/input/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"` + if test $retv != 0 ; then + echo "input/eedata.mth tiling misfit -- skipping $dir" + continue + fi fi fi @@ -1379,7 +1417,7 @@ if test "x$MPI" = "x0" ; then extra_runs="$extra_runs $name" else - retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/"$exd"/eedata.mth"` + retv=`check_eedata $dir"/"$exd"/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"` if test $retv = 0 ; then extra_runs="$extra_runs $name" else @@ -1404,9 +1442,13 @@ unset genmake makedepend make run results=$EMPTY_RESULTS - # Create an output dir for each OPTFILE/tdir combination + # Create an output dir & summary.txt file for each tested experiment (tdir) locDIR=$DRESULTS"/"$dir mkdir $locDIR + #- report to this experiment local summary file --- + echo "DATE='$DATE' ; tdir='$dir'" > $locDIR"/summary.txt" + echo "MACH='$MACH'" >> $locDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" CDIR=`pwd`"/$locDIR" if test "x$NORUN" = xt ; then @@ -1432,27 +1474,20 @@ fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo 1>&2 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY - touch $locDIR"/summary.txt" echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" - echo "tdir='$dir'" >> $locDIR"/summary.txt" - if test "x$ADM" = xt ; then - head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt" - nerr=`grep -c 'TAF *.* ERROR ' $dir/$builddir/taf_ad.log` - nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' $dir/$builddir/taf_ad.log` - echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \ - >> $locDIR"/summary.txt" - grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \ - >> $locDIR"/summary.txt" - fi for ex in $extra_runs ; do unset run results=$EMPTY_RESULTS # reference output file refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"` - # Create an output dir for each OPTFILE/tdir.ex combination + # Create an output dir & summary.txt file for each extra run (tdir.ex) locDIR=$DRESULTS"/"$dir"."$ex mkdir $locDIR + #- report to this experiment local summary file --- + echo "DATE='$DATE' ; tdir='$dir.$ex'" > $locDIR"/summary.txt" + #echo "MACH='$MACH'" >> $locDIR"/summary.txt" + #echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" CDIR=`pwd`"/$locDIR" test ! -e "$dir/$pfxdir.$ex" && mkdir "$dir/$pfxdir.$ex" run_clean $dir/$pfxdir.$ex @@ -1463,13 +1498,7 @@ fres="$fres.$ex" echo 1>&2 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY - touch $locDIR"/summary.txt" echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" - echo "tdir='$dir.$ex'" >> $locDIR"/summary.txt" - if test "x$ADM" = xt ; then - grep -A3 'Seconds in section "ALL' $dir/$pfxdir.$ex/$OUTPUTFILE \ - >> $locDIR"/summary.txt" - fi if test "x$POSTCLEAN" = x2 ; then run_clean $dir/$pfxdir.$ex fi