--- MITgcm/verification/testreport 2010/11/03 19:28:18 1.147 +++ MITgcm/verification/testreport 2011/01/14 23:55:00 1.154 @@ -1,6 +1,6 @@ #! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.147 2010/11/03 19:28:18 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.154 2011/01/14 23:55:00 jmc Exp $ # $Name: $ # @@ -12,7 +12,8 @@ echo "where possible OPTIONS are:" echo " (-help|-h) print usage" echo " (-mth) run multi-threaded (using eedata.mth)" - echo " (-mpi) compile and run using MPI" + echo " (-mpi) use MPI to compile and run on 2 processors" + echo " (-MPI) NUMBER use MPI to compile and run on max NUMBER procs" echo " (-ieee/-noieee) if possible, use IEEE compiler flags" echo " (DEF=\"-ieee\")" echo " (-gsl) compile with \"-gsl\" flag" @@ -25,6 +26,8 @@ echo " (-tdir|-t) STRING list of group and/or exp. dirs to test" echo " (recognized groups: basic, tutorials)" echo " (DEF=\"\" which test all)" + echo " (if list= 'start_from THIS_EXP' then" + echo " test THIS_EXP + all the following)" echo " (-skipdir|-skd) STRING list of exp. dirs to skip" echo " (DEF=\"\" which test all)" echo " (-bash|-b) STRING preferred location of a \"bash\" or" @@ -232,7 +235,14 @@ listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"` fi if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi - echo "listVar='$listVar'" > $locDIR"/summary.txt" + #- report to this experiment local summary file --- + echo "MACH='$MACH'" > $locDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" + echo "DATE='$DATE'" >> $locDIR"/summary.txt" + grep '(PID\.TID 0000\.0001) n.. =' $1/$2/$OUTPUTFILE \ + | sed 's/(PID.TID 0000.0001) //' >> $locDIR"/summary.txt" + echo "listVar='$listVar'" >> $locDIR"/summary.txt" + #--- allargs="" for xx in $listVar do @@ -337,7 +347,7 @@ if test "x$GSL" = xt ; then command="$command -gsl" fi - if test "x$MPI" = xt ; then + if test "x$MPI" != x0 ; then command="$command -mpi" fi if test "x$MULTI_THREAD" = xt ; then @@ -365,7 +375,7 @@ if test "x$RETVAL" != x0 ; then tail genmake.tr_log echo "genmakemodel: genmake failed" - cp genmake_* genmake.tr_log $CDIR + cp genmake.log genmake_* genmake.tr_log $CDIR return 1 else echo "successful" @@ -386,13 +396,12 @@ #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi if test -r Makefile ; then printf 'clean build-dir: make Clean ... ' - rm -f Makefile.old $MAKE Clean >> make.tr_log 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.tr_log echo "makeclean: \"make Clean\" failed" - cp make.tr_log $CDIR"/make.tr_log" + cp make.tr_log genmake.log genmake.tr_log $CDIR return 1 fi echo successful @@ -441,7 +450,7 @@ if test "x$RETVAL" != x0 ; then tail make.tr_log echo "makedependmodel: make depend failed" - cp make.tr_log $CDIR"/make.tr_log" + cp make.tr_log genmake.log genmake.tr_log $CDIR return 1 else echo successful @@ -481,7 +490,7 @@ if test "x$RETVAL" != x0 ; then tail make.tr_log echo failed - cp make.tr_log $CDIR"/make.tr_log" + cp make.tr_log genmake.log genmake.tr_log $CDIR rm -f $EXECUTABLE return 1 else @@ -492,6 +501,56 @@ ) } +mk_mpi_size() +{ + # mk_mpi_size input_file output_file proc_Nb + # + # make new SIZE.h (=output_file) from SIZE.h_mpi (=input_file) + # for an MPI build with no more than proc_Nb processors ; + # return the effective number of processors. + + inp=$1 + out=$2 + np=$3 + tmp=TTT.$$ + + px=`grep "^ & *nPx *=" $inp | sed "s/^ & *nPx *= *//" | sed 's/, *$//'` + py=`grep "^ & *nPy *=" $inp | sed "s/^ & *nPy *= *//" | sed 's/, *$//'` + sx=`grep "^ & *nSx *=" $inp | sed "s/^ & *nSx *= *//" | sed 's/, *$//'` + sy=`grep "^ & *nSy *=" $inp | sed "s/^ & *nSy *= *//" | sed 's/, *$//'` + + #- find the largest divisor of input_file proc Nb, but not larger than $np + pp=0 + for i in `seq 1 $px` ; do + if [ `expr $px % $i` -eq 0 ] ; then + for j in `seq 1 $py` ; do + if [ `expr $py % $j` -eq 0 ] ; then + ij=`expr $i \* $j` + if [ $ij -gt $pp ] ; then + if [ $ij -le $np ] ; then + ix=$i ; jy=$j ; pp=$ij + #echo " ix,jy= $ix,$jy" + fi + fi + fi + done + fi + done + + #- create new SIZE.h type file: + sx=`expr $sx \* $px / $ix` + sy=`expr $sy \* $py / $jy` + if [ $verbose -gt 1 ]; then + echo " px,py,np= $px,$py,$np : New MPI size: px,py= $ix,$jy : sx,sy= $sx,$sy" + fi + sed "/^ \& *nPx *=/s/[0-9]*,/$ix,/" $inp > $tmp + sed "/^ \& *nPy *=/s/[0-9]*,/$jy,/" $tmp > $out + sed "/^ \& *nSx *=/s/[0-9]*,/$sx,/" $out > $tmp + sed "/^ \& *nSy *=/s/[0-9]*,/$sy,/" $tmp > $out + rm -f $tmp + return $pp +} + symlink_mpifiles() { # Put special links so that MPI specific files are used @@ -500,44 +559,44 @@ dir=$1 code_dir=$2 - BUILD_DIR=$dir/$3 - CODE_DIR=$dir/$code_dir + build_dir=$dir/$3 # These are files that should replace their counter-part when using -mpi - MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)` + MPI_FILES=`(cd $dir/$code_dir; find . -name "*_mpi" -print)` - # Is this an MPI run? - if test "x$MPI" = xt ; then - # YES: We symbolically link these files to the build - # dir so long as there is no real file in place - for ii in $MPI_FILES ; do - i=`echo $ii | sed 's:^\./::'` - name=`echo $i | sed 's:_mpi::' ` - cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1 - RETVAL=$? - if test "x$RETVAL" != x0 ; then - if ! test -f $BUILD_DIR/$i ; then - #echo Linking $name to $i - (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name) - fi - fi - done - else - # NO: We undo any _mpi symbolically linked files - for ii in $MPI_FILES ; do - i=`echo $ii | sed 's:^\./::'` - name=`echo $i | sed 's:_mpi::' ` - if test -L $BUILD_DIR/$name ; then - cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1 + for ii in $MPI_FILES ; do + i=`echo $ii | sed 's:^\./::'` + name=`echo $i | sed 's:_mpi::'` + file="../$code_dir/$i" + if test $name = 'SIZE.h' ; then file="SIZE.h.mpi" ; fi + + # Is this an MPI run? + if test "x$MPI" = x0 ; then + # NO: We undo any _mpi symbolically linked files + if test -L $build_dir/$name ; then + ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 ) RETVAL=$? if test "x$RETVAL" = x0 ; then - #echo Un-linking $name from $linktarg - rm -f $BUILD_DIR/$name + if [ $verbose -gt 1 ]; then + echo " Un-linking $name from ../$code_dir" ; fi + rm -f $build_dir/$name fi fi - done - fi - + else + # YES: We symbolically link these files to the build + # dir so long as there is no real file in place + ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 ) + RETVAL=$? + if [ $verbose -gt 1 ]; then echo " cmp $name $file returns: $RETVAL" ; fi + if test "x$RETVAL" != x0 ; then + if test -h $build_dir/$name ; then rm -f $build_dir/$name ; fi + if test ! -r $build_dir/$name ; then + if [ $verbose -gt 1 ]; then echo " Linking $name to $file" ; fi + (cd $build_dir; ln -sf $file $name) + fi + fi + fi + done } linkdata() @@ -554,16 +613,16 @@ if test -r "../"$1"/"$xx ; then # found 1 _mpi sfx file in 1rst input dir and it is readable yy=`echo $xx | sed 's:\.mpi$::'` - if test "x$MPI" = "xt" ; then - # mpi test: remove symbolic link & link _mpi sfx file + if test "x$MPI" = "x0" ; then + # not mpi test: remove symbolic link + if test -h $yy ; then rm -f $yy ; fi + else + # mpi test: remove symbolic link & link .mpi sfx file if test -h $yy ; then rm -f $yy ; fi if test ! -r $yy ; then ln -sf "../"$1"/"$xx $yy ; printf " $xx" 1>&2 fi - else - # not mpi test: remove symbolic link - if test -h $yy ; then rm -f $yy ; fi fi fi done @@ -611,15 +670,15 @@ { # runmodel directory # - # runs "$COMMAND in "directory" + # runs $COMMAND in "directory" # (where "$COMMAND" is relative to "directory") ( cd $1 printf 'runmodel in %s ... ' $1 if test -L $EXECUTABLE ; then if test -x "../"$builddir"/"$EXECUTABLE ; then - diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1 - outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi + cmp $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1 + outD=$? ; if test "x$outD" != x0 ; then rm -f $EXECUTABLE ; fi else rm -f $EXECUTABLE fi fi @@ -627,6 +686,12 @@ echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_tmp ln -sf "../"$builddir"/"$EXECUTABLE . fi + if test "x$MPI" != x0 ; then + #- adjust the MPI run command with the right number of Procs + #echo "COMMAND='$COMMAND'" + COMMAND=`echo $COMMAND | sed "s/-np * XX /-np $NPROC_MPI /"` + #echo "COMMAND='$COMMAND'" + fi if test ! -x $EXECUTABLE ; then rm -f $RUNLOG ; touch $RUNLOG if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi @@ -652,7 +717,8 @@ echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1 fi fi - ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'` + #ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'` + ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'` fi rm -f run.log_tmp if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then @@ -843,7 +909,7 @@ CC=cc fi JOBS= -MPI=f +MPI=0 MULTI_THREAD=f OUTDIR= DELDIR= @@ -955,7 +1021,9 @@ -postclean | --postclean | -pc | --pc) POSTCLEAN=t ;; - -mpi) MPI=t ;; + -mpi | --mpi) MPI=2 ;; + -MPI | --MPI) ac_prev=MPI ;; + -MPI=* | --MPI=*) MPI=$ac_optarg ;; -mth) MULTI_THREAD=t ;; @@ -1071,7 +1139,7 @@ if test "x$COMMAND" = x ; then COMMAND="./$EXECUTABLE > $OUTPUTFILE" fi -if test "x$MPI" = xt ; then +if test "x$MPI" != x0 ; then OUTPUTFILE="STDOUT.0000" fi @@ -1164,8 +1232,8 @@ fi else echo >> $SUMMARY - echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY - echo " so the genmake default will be used." >> $SUMMARY + echo "No \"OPTFILE\" was specified ; genmake2 found and uses:" >> $SUMMARY + #-note: to be filled later after 1rst run fi echo echo >> $SUMMARY @@ -1229,7 +1297,7 @@ ( cd $BUILD_DIR rm -f $EXECUTABLE *.bak rm -f genmake_state genmake_*optfile genmake.log - rm -f genmake.tr_log make.tr_log + rm -f SIZE.h.mpi genmake.tr_log make.tr_log ) if test -d $dir/$rundir/CVS ; then echo -n ' --- dir:' $dir/$rundir ': ' @@ -1254,20 +1322,37 @@ # Check for specific files for particular type of run - if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then - echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" - continue + if test "x$MPI" != "x0" ; then + if test -r $CODE_DIR"/SIZE.h_mpi" ; then + #- create new SIZE.h with no more than '$MPI' Procs + mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI + NPROC_MPI=$? + ( cd $BUILD_DIR + if test -r SIZE.h.mpi ; then + cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$? + else RETVAL=1 + fi + if test "x$RETVAL" = x0 ; then + rm -f tr_size.mpi + else + rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi + fi + ) + else + echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" + continue + fi fi if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir" continue fi - if test "x$MPI" = "xt" -a "x$MULTI_THREAD" = "xt" ; then - retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/input/eedata.mth"` - if test $retv != 0 ; then - echo "input/eedata.mth tiling misfit -- skipping $dir" - continue - fi + if test "x$MPI" != "x0" -a "x$MULTI_THREAD" = "xt" ; then + retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/input/eedata.mth"` + if test $retv != 0 ; then + echo "input/eedata.mth tiling misfit -- skipping $dir" + continue + fi fi # Check whether there are "extra runs" for this testdir @@ -1283,15 +1368,15 @@ if test -f $outf -a -r $outf ; then if test "x$MULTI_THREAD" = "xt" ; then if test -r $dir"/"$exd"/eedata.mth" ; then - if test "x$MPI" = "xt" ; then - retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/"$exd"/eedata.mth"` + if test "x$MPI" = "x0" ; then + extra_runs="$extra_runs $name" + else + retv=`check_eedata $BUILD_DIR"/SIZE.h.mpi" $dir"/"$exd"/eedata.mth"` if test $retv = 0 ; then extra_runs="$extra_runs $name" else echo $exd"/eedata.mth tiling misfit -- skipping $dir" fi - else - extra_runs="$extra_runs $name" fi #else echo $dir"/"$exd"/eedata.mth: not found" fi @@ -1341,9 +1426,6 @@ echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY touch $locDIR"/summary.txt" echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" - echo "MACH='$MACH'" >> $locDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" - echo "DATE='$DATE'" >> $locDIR"/summary.txt" echo "tdir='$dir'" >> $locDIR"/summary.txt" if test "x$ADM" = xt ; then head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt" @@ -1371,9 +1453,6 @@ echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY touch $locDIR"/summary.txt" echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" - echo "MACH='$MACH'" >> $locDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" - echo "DATE='$DATE'" >> $locDIR"/summary.txt" echo "tdir='$dir.$ex'" >> $locDIR"/summary.txt" if test "x$ADM" = xt ; then head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt" @@ -1390,7 +1469,28 @@ mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'` echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state - echo $mkOpt >> $DRESULTS/genmake_state + echo " $mkOpt" >> $DRESULTS/genmake_state + if test "x$OPTFILE" = xNONE ; then + eval $mkOpt + sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\ + $SUMMARY > tr_0.tmp_log + RETVAL=$? + if test "x$RETVAL" = x0 ; then rm -f $SUMMARY + cp tr_0.tmp_log $SUMMARY + else rm -f tr_0.tmp_log + fi + fi + gmkLog=$dir/$builddir/genmake.log + grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" = x0 ; then + echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state + sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \ + | grep -v '^... compiler version ' > tr_1.tmp_log + sed -n '1,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \ + >> $DRESULTS/genmake_state + rm -f tr_1.tmp_log + fi fi fi #postclean $dir/$builddir