--- MITgcm/verification/testreport 2010/11/02 22:53:11 1.146 +++ MITgcm/verification/testreport 2011/09/26 12:58:48 1.167 @@ -1,6 +1,6 @@ #! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.146 2010/11/02 22:53:11 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.167 2011/09/26 12:58:48 jmc Exp $ # $Name: $ # @@ -12,9 +12,12 @@ echo "where possible OPTIONS are:" echo " (-help|-h) print usage" echo " (-mth) run multi-threaded (using eedata.mth)" - echo " (-mpi) compile and run using MPI" - echo " (-ieee/-noieee) if possible, use IEEE compiler flags" - echo " (DEF=\"-ieee\")" + echo " (-mpi) use MPI to compile and run on 2 processors" + echo " (-MPI) NUMBER use MPI to compile and run on max NUMBER procs" + echo " (-mfile|-mf) STRING MPI: file with list of possible machines to run on" + echo " (-fast) use optfile default for compiler flags (no '-ieee')" + echo " DEF=off => use IEEE numerics option (if available)" + echo " (-devel) use optfile developement flags (if available)" echo " (-gsl) compile with \"-gsl\" flag" echo " (-use_r4|-ur4) if allowed, use real*4 type for '_RS' variable" echo " (-optfile|-of) STRING list of optfiles to use" @@ -25,6 +28,8 @@ echo " (-tdir|-t) STRING list of group and/or exp. dirs to test" echo " (recognized groups: basic, tutorials)" echo " (DEF=\"\" which test all)" + echo " (if list= 'start_from THIS_EXP' then" + echo " test THIS_EXP + all the following)" echo " (-skipdir|-skd) STRING list of exp. dirs to skip" echo " (DEF=\"\" which test all)" echo " (-bash|-b) STRING preferred location of a \"bash\" or" @@ -38,18 +43,22 @@ echo " (DEF=\"make\")" echo " (-odir) STRING used to build output directory name" echo " (DEF=\"hostname\")" - echo " (-ptracers|-ptr) STRING specify which ptracers to test" - echo " (DEF=\"1 2 3 4 5\")" +# echo " (-ptracers|-ptr) STRING specify which ptracers to test" +# echo " (DEF=\"1 2 3 4 5\")" echo " (-match) NUMBER Matching Criteria (number of digits)" echo " (DEF=\"$MATCH_CRIT\")" echo " (-j) JOBS use \"make -j JOBS\" for parallel builds" - echo " (-clean) *ONLY* run \"make CLEAN\"" + echo " (-ef) STRING used as genmake2 \"-extra_flag\" argument" + echo " (-clean) *ONLY* run \"make CLEAN\" & clean run-dir" echo " (-norun|-nr) skip the \"runmodel\" stage (stop after make)" + echo " (-obj) only produces objects (=norun & no executable)" echo " (-runonly|-ro) *ONLY* run stage (=\"-quick\" without make)" echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\"" echo " (-nogenmake|-ng) skip the genmake stage" echo " (-noclean|-nc) skip the \"make clean\" stage" echo " (-nodepend|-nd) skip the \"make depend\" stage" + echo " (-postclean|-pc) after each exp. test, clean build-dir & run-dir" + echo " (-deloutp|-do) delete output files after successful run" echo " (-deldir|-dd) on success, delete the output directory" echo " (-ts) provide timing information per timestep" echo " (-papis) provide MFlop/s per timestep using PAPI" @@ -232,7 +241,8 @@ listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"` fi if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi - echo "listVar='$listVar'" > $locDIR"/summary.txt" + echo "listVar='$listVar'" >> $locDIR"/summary.txt" + #--- allargs="" for xx in $listVar do @@ -241,8 +251,9 @@ then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$? if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;; - 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;; - 'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;; + 'Cost') testoutput_var $1 "ADM ref_cost_function" "ADM CostFct" $2 $3; yy=$? ;; + 'AdGrd') testoutput_var $1 "ADM adjoint_gradient" "ADM Ad Grad" $2 $3; yy=$? ;; + 'FDGrd') testoutput_var $1 "ADM finite-diff_grad" "ADM FD Grad" $2 $3; yy=$? ;; 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;; 'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;; 'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;; @@ -331,13 +342,16 @@ if test "x$OPTFILE" != xNONE ; then command="$command -optfile=$OPTFILE" fi - if test "x$IEEE" != x ; then - command="$command -ieee" + if test $OptLev = 1 ; then + command="$command -ieee" + fi + if test $OptLev = 0 ; then + command="$command -devel" fi if test "x$GSL" = xt ; then command="$command -gsl" fi - if test "x$MPI" = xt ; then + if test "x$MPI" != x0 ; then command="$command -mpi" fi if test "x$MULTI_THREAD" = xt ; then @@ -347,6 +361,9 @@ if test "x$USE_R4" = xt ; then command="$command -use_r4" fi + if test "x$EXTRFLG" != x ; then + command="$command -extra_flag $EXTRFLG" + fi if test "x$TS" = xt ; then command="$command -ts" fi @@ -365,7 +382,7 @@ if test "x$RETVAL" != x0 ; then tail genmake.tr_log echo "genmakemodel: genmake failed" - cp genmake_* genmake.tr_log $CDIR + cp genmake.log genmake_* genmake.tr_log $CDIR return 1 else echo "successful" @@ -386,13 +403,12 @@ #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi if test -r Makefile ; then printf 'clean build-dir: make Clean ... ' - rm -f Makefile.old $MAKE Clean >> make.tr_log 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.tr_log echo "makeclean: \"make Clean\" failed" - cp make.tr_log $CDIR"/make.tr_log" + cp make.tr_log genmake.log genmake.tr_log $CDIR return 1 fi echo successful @@ -418,7 +434,8 @@ find . -name "*.data" -exec rm {} \; find . -name "fort.*" -exec rm {} \; find . -type l -exec rm {} \; - rm -f $EXECUTABLE $RUNLOG *.txt STD* *diagnostics.log datetime + #- should remove executable only if sym-link (alredy done above) + rm -f $RUNLOG *.txt STD* *diagnostics.log datetime rm -rf mnc_test_* rm -f *_MIT_CE_000.opt0000 costfunction*0000 echo successful @@ -441,7 +458,7 @@ if test "x$RETVAL" != x0 ; then tail make.tr_log echo "makedependmodel: make depend failed" - cp make.tr_log $CDIR"/make.tr_log" + cp make.tr_log genmake.log genmake.tr_log $CDIR return 1 else echo successful @@ -454,13 +471,14 @@ { # makemodel directory ( + mk_fail=0 if test "x$NOMAKE" = xt ; then cd $1; if test -x $EXECUTABLE ; then echo "make skipped!" else echo "no executable!" - return 2 + mk_fail=3 fi else cd $1; @@ -472,26 +490,100 @@ $MAKE adAll >> make.tr_log 2>&1 else if test "x$JOBS" = x ; then - $MAKE >> make.tr_log 2>&1 + $MAKE $TARG >> make.tr_log 2>&1 else - $MAKE -j $JOBS >> make.tr_log 2>&1 + $MAKE -j $JOBS $TARG >> make.tr_log 2>&1 fi fi RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.tr_log echo failed - cp make.tr_log $CDIR"/make.tr_log" + cp genmake.log genmake.tr_log $CDIR + tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail" rm -f $EXECUTABLE - return 1 + mk_fail=1 else echo successful fi + else + echo "no Makefile !" + mk_fail=2 fi fi + if test "x$ADM" = xt -a -f taf_ad.log ; then + head -1 taf_ad.log >> $CDIR"/summary.txt" + nerr=`grep -c 'TAF *.* ERROR ' taf_ad.log` + nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' taf_ad.log` + echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \ + >> $CDIR"/summary.txt" + fi + if test $mk_fail != 0 ; then return $mk_fail ; fi ) } +mk_mpi_size() +{ + # mk_mpi_size input_file output_file proc_Nb prefer_to_split_X + # + # make new SIZE.h (=output_file) from SIZE.h_mpi (=input_file) + # for an MPI build with no more than proc_Nb processors ; + # return the effective number of processors. + + inp=$1 + out=$2 + np=$3 + dirX=$4 + tmp=TTT.$$ + + px=`grep "^ & *nPx *=" $inp | sed "s/^ & *nPx *= *//" | sed 's/, *$//'` + py=`grep "^ & *nPy *=" $inp | sed "s/^ & *nPy *= *//" | sed 's/, *$//'` + sx=`grep "^ & *nSx *=" $inp | sed "s/^ & *nSx *= *//" | sed 's/, *$//'` + sy=`grep "^ & *nSy *=" $inp | sed "s/^ & *nSy *= *//" | sed 's/, *$//'` + + #- find the largest divisor of input_file proc Nb, but not larger than $np + pp=0 + i=1 + while [ $i -le $px ] ; do + if [ `expr $px % $i` -eq 0 ] ; then + j=1 + while [ $j -le $py ] ; do + if [ `expr $py % $j` -eq 0 ] ; then + ij=`expr $i \* $j` + if [ $ij -gt $pp ] ; then + flag=1 + elif [ $ij -eq $pp ] ; then + flag=$dirX + else + flag=0 + fi + if test $flag = 1 ; then + if [ $ij -le $np ] ; then + ix=$i ; jy=$j ; pp=$ij + #echo " ix,jy= $ix,$jy" + fi + fi + fi + j=`expr $j + 1` + done + fi + i=`expr $i + 1` + done + + #- create new SIZE.h type file: + sx=`expr $sx \* $px / $ix` + sy=`expr $sy \* $py / $jy` + if [ $verbose -gt 1 ]; then + echo " px,py,np= $px,$py,$np : New MPI size: px,py= $ix,$jy : sx,sy= $sx,$sy" + fi + sed "/^ \& *nPx *=/s/[0-9]*,/$ix,/" $inp > $tmp + sed "/^ \& *nPy *=/s/[0-9]*,/$jy,/" $tmp > $out + sed "/^ \& *nSx *=/s/[0-9]*,/$sx,/" $out > $tmp + sed "/^ \& *nSy *=/s/[0-9]*,/$sy,/" $tmp > $out + rm -f $tmp + return $pp +} + symlink_mpifiles() { # Put special links so that MPI specific files are used @@ -500,44 +592,44 @@ dir=$1 code_dir=$2 - BUILD_DIR=$dir/$3 - CODE_DIR=$dir/$code_dir + build_dir=$dir/$3 # These are files that should replace their counter-part when using -mpi - MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)` + MPI_FILES=`(cd $dir/$code_dir; find . -name "*_mpi" -print)` - # Is this an MPI run? - if test "x$MPI" = xt ; then - # YES: We symbolically link these files to the build - # dir so long as there is no real file in place - for ii in $MPI_FILES ; do - i=`echo $ii | sed 's:^\./::'` - name=`echo $i | sed 's:_mpi::' ` - cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1 - RETVAL=$? - if test "x$RETVAL" != x0 ; then - if ! test -f $BUILD_DIR/$i ; then - #echo Linking $name to $i - (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name) - fi - fi - done - else - # NO: We undo any _mpi symbolically linked files - for ii in $MPI_FILES ; do - i=`echo $ii | sed 's:^\./::'` - name=`echo $i | sed 's:_mpi::' ` - if test -L $BUILD_DIR/$name ; then - cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1 + for ii in $MPI_FILES ; do + i=`echo $ii | sed 's:^\./::'` + name=`echo $i | sed 's:_mpi::'` + file="../$code_dir/$i" + if test $name = 'SIZE.h' ; then file="SIZE.h.mpi" ; fi + + # Is this an MPI run? + if test "x$MPI" = x0 ; then + # NO: We undo any _mpi symbolically linked files + if test -L $build_dir/$name ; then + ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 ) RETVAL=$? if test "x$RETVAL" = x0 ; then - #echo Un-linking $name from $linktarg - rm -f $BUILD_DIR/$name + if [ $verbose -gt 1 ]; then + echo " Un-linking $name from ../$code_dir" ; fi + rm -f $build_dir/$name fi fi - done - fi - + else + # YES: We symbolically link these files to the build + # dir so long as there is no real file in place + ( cd $build_dir ; cmp $name $file > /dev/null 2>&1 ) + RETVAL=$? + if [ $verbose -gt 1 ]; then echo " cmp $name $file returns: $RETVAL" ; fi + if test "x$RETVAL" != x0 ; then + if test -h $build_dir/$name ; then rm -f $build_dir/$name ; fi + if test ! -r $build_dir/$name ; then + if [ $verbose -gt 1 ]; then echo " Linking $name to $file" ; fi + (cd $build_dir; ln -sf $file $name) + fi + fi + fi + done } linkdata() @@ -554,16 +646,16 @@ if test -r "../"$1"/"$xx ; then # found 1 _mpi sfx file in 1rst input dir and it is readable yy=`echo $xx | sed 's:\.mpi$::'` - if test "x$MPI" = "xt" ; then - # mpi test: remove symbolic link & link _mpi sfx file + if test "x$MPI" = "x0" ; then + # not mpi test: remove symbolic link + if test -h $yy ; then rm -f $yy ; fi + else + # mpi test: remove symbolic link & link .mpi sfx file if test -h $yy ; then rm -f $yy ; fi if test ! -r $yy ; then ln -sf "../"$1"/"$xx $yy ; printf " $xx" 1>&2 fi - else - # not mpi test: remove symbolic link - if test -h $yy ; then rm -f $yy ; fi fi fi done @@ -611,15 +703,24 @@ { # runmodel directory # - # runs "$COMMAND in "directory" + # runs $COMMAND in "directory" # (where "$COMMAND" is relative to "directory") ( cd $1 printf 'runmodel in %s ... ' $1 + if test "x$MPI" != x0 ; then + #- adjust the MPI run command with the right number of Procs + #echo '' ; echo " COMMAND='$COMMAND'" + COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"` + if test "x$MPI_MFILE" != x ; then + COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"` + fi + #echo " COMMAND='$COMMAND'" + fi if test -L $EXECUTABLE ; then if test -x "../"$builddir"/"$EXECUTABLE ; then - diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1 - outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; fi + cmp $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1 + outD=$? ; if test "x$outD" != x0 ; then rm -f $EXECUTABLE ; fi else rm -f $EXECUTABLE fi fi @@ -628,40 +729,46 @@ ln -sf "../"$builddir"/"$EXECUTABLE . fi if test ! -x $EXECUTABLE ; then - rm -f $RUNLOG ; touch $RUNLOG - if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi - echo " no executable:" $EXECUTABLE >> $RUNLOG - RETVAL=8 - ENDVAL=-1 + rm -f $RUNLOG ; touch $RUNLOG + if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi + echo " no executable:" $EXECUTABLE >> $RUNLOG + RETVAL=8 + ENDVAL=-1 else - if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then - # output do not exist or is older than executable: - rm -f $RUNLOG ; touch $RUNLOG - if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi - ( eval $COMMAND ) >> $RUNLOG 2>&1 - RETVAL=$? - else - RETVAL=0 - if test -f $RUNLOG ; then - if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi - echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1 - else - touch $RUNLOG - if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi - echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1 - echo " no previous $RUNLOG: assume NORMAL END" >> $RUNLOG 2>&1 - fi + if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then + # output do not exist or is older than executable: + rm -f $OUTPUTFILE $RUNLOG ; touch $RUNLOG + if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi + ( eval $COMMAND ) >> $RUNLOG 2>&1 + RETVAL=$? + ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'` + if [ $POSTCLEAN -eq 1 -a $ENDVAL -gt 0 ] ; then + find . -name "*.meta" -exec rm {} \; + find . -name "*.data" -exec rm {} \; + rm -rf mnc_test_* fi - ENDVAL=`cat $RUNLOG | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'` + else + RETVAL=0 + ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'` + touch $RUNLOG + if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi + echo "---------->> $OUTPUTFILE is up to date " >> $RUNLOG 2>&1 + fi fi rm -f run.log_tmp + #- in all cases where OutputFile exists, report SIZE (and AD time) + if test -f $OUTPUTFILE ; then + grep '(PID\.TID 0000\.0001) n.. =' $OUTPUTFILE \ + | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" + if test "x$ADM" = xt ; then + grep -A3 'Seconds in section "ALL' $OUTPUTFILE >> $CDIR"/summary.txt" + fi + fi + if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then echo successful printf '=> output from running in %s :\n' $1 1>&2 tail $RUNLOG | sed 's/^.*/> &/g' 1>&2 - # === Reduce the size of the testing emails! - #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE - if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi return 0 elif [ $RETVAL -ne 0 -a $ENDVAL -gt 0 ] ; then #-- for some weird cases (run is finihed but with error code) @@ -674,7 +781,6 @@ printf '=> output from running in %s :\n' $1 1>&2 tail $RUNLOG | sed 's/^.*/> &/g' 1>&2 cp $RUNLOG $CDIR"/"$RUNLOG - if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi return 1 fi ) @@ -776,27 +882,38 @@ check_eedata() { - # check_eedata size.h eedata + # check_eedata eedata size.h if [ $# -eq 2 ] ; then - if test -f $1 -a -f $2 ; then - sx=`grep "^ & *nSx *=" $1 | sed "s/^ & *nSx *=//" | sed 's/, *$//'` - nx=`grep "^ *nTx *=" $2 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"` + if test -f $1 -a -f $2 ; then + nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"` + sx=`grep "^ & *nSx *=" $2 | sed "s/^ & *nSx *=//" | sed 's/, *$//'` if test "x$nx" = x ; then rx=10 else rx=`expr $sx % $nx` fi - sy=`grep "^ & *nSy *=" $1 | sed "s/^ & *nSy *=//" | sed 's/, *$//'` - ny=`grep "^ *nTy *=" $2 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"` + ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"` + sy=`grep "^ & *nSy *=" $2 | sed "s/^ & *nSy *=//" | sed 's/, *$//'` if test "x$ny" = x ; then ry=20 else ry=`expr $sy % $ny` fi echo `expr $rx + $ry` - else + else echo '-1' - fi + fi + elif [ $# -eq 1 ] ; then + if test -f $1 ; then + nx=`grep "^ *nTx *=" $1 | tail -1 | sed 's/^ *nTx *= *//' | sed "s/, *$//"` + if test "x$nx" = x ; then nx=1 ; fi + ny=`grep "^ *nTy *=" $1 | tail -1 | sed 's/^ *nTy *= *//' | sed "s/, *$//"` + if test "x$ny" = x ; then ny=1 ; fi + #echo $nx $ny + echo $nx + else + echo '-1' + fi else echo '-2' fi @@ -809,12 +926,13 @@ # Default properties debug=0 verbose=1 -clean=0 +NBLINES_MKLOG=16000 IEEE=true if test "x$MITGCM_IEEE" != x ; then IEEE=$MITGCM_IEEE fi +OptLev=1 GSL=f CLEANUP=f @@ -824,7 +942,7 @@ NOGENMAKE=f NOCLEAN=f NODEPEND=f -POSTCLEAN=f +POSTCLEAN=0 BASH= OPTFILE=NONE @@ -843,11 +961,14 @@ CC=cc fi JOBS= -MPI=f +TARG= +MPI=0 +MPI_MFILE= MULTI_THREAD=f OUTDIR= DELDIR= USE_R4= +EXTRFLG= ADM= OADM= @@ -855,7 +976,7 @@ # list of pTracers to check for monitor output PTRACERS_NUM="1 2 3 4 5" -MATCH_CRIT=13 +MATCH_CRIT=10 printf "parsing options... " @@ -873,62 +994,38 @@ case $ac_option in - -help | --help | -h | --h) - usage ;; - - -optfile | --optfile | -of | --of) - ac_prev=OPTFILE ;; - -optfile=* | --optfile=* | -of=* | --of=*) - OPTFILE=$ac_optarg ;; - - -addr | --addr | -a | --a) - ac_prev=ADDRESSES ;; - -addr=* | --addr=* | -a=* | --a=*) - ADDRESSES=$ac_optarg ;; - -mpackdir | --mpackdir | -mpd | --mpd) - ac_prev=MPACKDIR ;; - -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*) - MPACKDIR=$ac_optarg ;; - - -tdir | --tdir | -t | --t) - ac_prev=TESTDIRS ;; - -tdir=* | --tdir=* | -t=* | --t=*) - TESTDIRS=$ac_optarg ;; - - -skipdir | --skipdir | -skd | --skd) - ac_prev=SKIPDIRS ;; - -skipdir=* | --skipdir=* | -skd=* | --skd=*) - SKIPDIRS=$ac_optarg ;; - - -bash | --bash | -b | --b) - ac_prev=BASH ;; - -bash=* | --bash=* | -b=* | --b=*) - BASH=$ac_optarg ;; - - -command | --command | -c | --c) - ac_prev=COMMAND ;; - -command=* | --command=* | -c=* | --c=*) - COMMAND=$ac_optarg ;; - - -makedepend | --makedepend | -md | --md) - ac_prev=MKDEPEND ;; - -makedepend=* | --makedepend=* | -md=* | --md=*) - MKDEPEND=$ac_optarg ;; - - -make | --make | -m | --m) - ac_prev=MAKE ;; - -make=* | --make=* | -m=* | --m=*) - MAKE=$ac_optarg ;; - - -odir | --odir) - ac_prev=OUTDIR ;; - -odir=* | --odir=*) - OUTDIR=$ac_optarg ;; - - -ptracers | --ptracers | -ptr | --ptr) - ac_prev=PTRACERS_NUM ;; - -ptracers=* | --ptracers=* | -ptr=* | --ptr=*) - PTRACERS_NUM=$ac_optarg ;; + -help | --help | -h | --h) usage ;; + + -optfile | --optfile | -of | --of) ac_prev=OPTFILE ;; + -optfile=* | --optfile=* | -of=* | --of=*) OPTFILE=$ac_optarg ;; + + -addr | --addr | -a | --a) ac_prev=ADDRESSES ;; + -addr=* | --addr=* | -a=* | --a=*) ADDRESSES=$ac_optarg ;; + -mpackdir | --mpackdir | -mpd | --mpd) ac_prev=MPACKDIR ;; + -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*) MPACKDIR=$ac_optarg ;; + + -tdir | --tdir | -t | --t) ac_prev=TESTDIRS ;; + -tdir=* | --tdir=* | -t=* | --t=*) TESTDIRS=$ac_optarg ;; + -skipdir | --skipdir | -skd | --skd) ac_prev=SKIPDIRS ;; + -skipdir=* | --skipdir=* | -skd=* | --skd=*) SKIPDIRS=$ac_optarg ;; + + -bash | --bash | -b | --b) ac_prev=BASH ;; + -bash=* | --bash=* | -b=* | --b=*) BASH=$ac_optarg ;; + + -command | --command | -c | --c) ac_prev=COMMAND ;; + -command=* | --command=* | -c=* | --c=*) COMMAND=$ac_optarg ;; + + -makedepend | --makedepend | -md | --md) ac_prev=MKDEPEND ;; + -makedepend=* | --makedepend=* | -md=* | --md=*) MKDEPEND=$ac_optarg ;; + + -make | --make | -m | --m) ac_prev=MAKE ;; + -make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;; + + -odir | --odir) ac_prev=OUTDIR ;; + -odir=* | --odir=*) OUTDIR=$ac_optarg ;; + + -ptracers | --ptracers | -ptr | --ptr) ac_prev=PTRACERS_NUM ;; + -ptracers=* | --ptracers=* | -ptr=* | --ptr=*) PTRACERS_NUM=$ac_optarg ;; -match | --match ) ac_prev=MATCH_CRIT ;; -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;; @@ -936,36 +1033,41 @@ -j | --j) ac_prev=JOBS ;; -j=* | --j=*) JOBS=$ac_optarg ;; - -clean | --clean) - CLEANUP=t ; DELDIR=t ;; + -ef | --ef) ac_prev=EXTRFLG ;; + -ef=* | --ef=*) EXTRFLG=$ac_optarg ;; - -norun | --norun | -nr | --nr) - NORUN=t ;; - -runonly | --runonly | -ro | --ro) - QUICK=t ; NOMAKE=t ;; - -quick | --quick | -q | --q) - QUICK=t ;; - -nogenmake | --nogenmake | -ng | --ng) - NOGENMAKE=t ;; - -noclean | --noclean | -nc | --nc) - NOCLEAN=t ;; - -nodepend | --nodepend | -nd | --nd) - NODEPEND=t ;; + -clean | --clean) CLEANUP=t ; DELDIR=t ;; - -postclean | --postclean | -pc | --pc) - POSTCLEAN=t ;; + -norun | --norun | -nr | --nr) NORUN=t ;; + -obj | --obj ) TARG='obj' ; NORUN=t ;; + -runonly | --runonly | -ro | --ro) QUICK=t ; NOMAKE=t ;; + -quick | --quick | -q | --q) QUICK=t ;; + -nogenmake | --nogenmake | -ng | --ng) NOGENMAKE=t ;; + -noclean | --noclean | -nc | --nc) NOCLEAN=t ;; + -nodepend | --nodepend | -nd | --nd) NODEPEND=t ;; + + -postclean | --postclean | -pc | --pc) POSTCLEAN=2 ;; + -deloutp | --deloutp | -do | --do) POSTCLEAN=1 ;; + + -mpi | --mpi) MPI=2 ;; + -MPI | --MPI) ac_prev=MPI ;; + -MPI=* | --MPI=*) MPI=$ac_optarg ;; - -mpi) MPI=t ;; + -mfile | --mfile | -mf | --mf) ac_prev=MPI_MFILE ;; + -mfile=* | --mfile=* | -mf=* | --mf=*) MPI_MFILE=$ac_optarg ;; -mth) MULTI_THREAD=t ;; -adm | -ad) ADM=t ;; - -oad) OADM=t; NODEPEND=t ;; - -ieee) IEEE=true ;; - -noieee) IEEE= ;; - -gsl) GSL=t ;; + -ieee) echo "Warning: ignore option '-ieee' (already the default)" + printf " ... " ;; + -noieee) echo "Warning: will use option '-fast' instead of '-noieee' (obsolete)" + printf " ... " ; OptLev=`expr $OptLev \* 2` ;; + -fast) OptLev=`expr $OptLev \* 2` ;; + -devel) OptLev=0 ;; + -gsl) GSL=t ;; -verbose) verbose=2 ;; -debug) debug=1 ;; @@ -979,15 +1081,10 @@ -papis) PAPIS=t;; -pcls) PCL=t;; - -*) - echo "Error: unrecognized option: "$ac_option - usage - ;; - - *) - echo "Error: unrecognized argument: "$ac_option - usage - ;; + -*) echo "Error: unrecognized option: "$ac_option + usage ;; + *) echo "Error: unrecognized argument: "$ac_option + usage ;; esac @@ -999,6 +1096,23 @@ NODEPEND=t fi +#- check length of MPI machine file: +if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then + if test -r $MPI_MFILE ; then + nl=`wc -l $MPI_MFILE | awk '{print $1}'` + if [ $nl -lt $MPI ] ; then + echo "Error: need at least $MPI nodes (currently only $nl) in MPI_MFILE=$MPI_FILE" + usage + fi + if [ $verbose -gt 1 ]; then + echo " MPI_MFILE=$MPI_MFILE : $nl procs for MPI=$MPI run" + fi + else + echo "Error: cannot access MPI_MFILE=$MPI_FILE" + usage + fi +fi + #- setting for forward or ADM testing if test "x$ADM" = xt ; then code_dir=code_ad @@ -1066,12 +1180,13 @@ OPTFILE=$MITGCM_OF fi +LOC_MFILE='tr_mpi_mfile' RUNLOG="run.tr_log" OUTPUTFILE=$ref_outp if test "x$COMMAND" = x ; then COMMAND="./$EXECUTABLE > $OUTPUTFILE" fi -if test "x$MPI" = xt ; then +if test "x$MPI" != x0 ; then OUTPUTFILE="STDOUT.0000" fi @@ -1088,7 +1203,7 @@ ii=`echo $EMPTY_RESULTS | awk '{print NF}'` EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'` else - DEF_CHECK_LIST='Grad Cost Grad' + DEF_CHECK_LIST='AdGrd Cost AdGrd FDGrd' EMPTY_RESULTS='.. ..' LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'` fi @@ -1164,8 +1279,8 @@ fi else echo >> $SUMMARY - echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY - echo " so the genmake default will be used." >> $SUMMARY + echo "No \"OPTFILE\" was specified ; genmake2 found and uses:" >> $SUMMARY + #-note: to be filled later after 1rst run fi echo echo >> $SUMMARY @@ -1175,8 +1290,8 @@ else line_0="default "$MATCH_CRIT ; fi line_0="$line_0 ----T----- ----S----- ----U----- ----V-----" line_1="G D M c m s m s m s m s" - line_2="E p a R g m m e . m m e . m m e . m m e ." - line_3="N n k u 2 i a a d i a a d i a a d i a a d" + line_2="e p a R g m m e . m m e . m m e . m m e ." + line_3="n n k u 2 i a a d i a a d i a a d i a a d" line_4="2 d e n d n x n . n x n . n x n . n x n ." for ii in $PTRACERS_NUM ; do line_0="$line_0 --PTR 0"$ii"--" @@ -1199,10 +1314,10 @@ else line_0="default "$MATCH_CRIT ; fi echo "$line_0" | tee -a $SUMMARY cat << EOF | tee -a $SUMMARY -G D M C G -E p a R o r -N n k u s a -2 d e n t d +G D M C A F +e p a R o d D +n n k u s G G +2 d e n t r r EOF fi @@ -1229,7 +1344,7 @@ ( cd $BUILD_DIR rm -f $EXECUTABLE *.bak rm -f genmake_state genmake_*optfile genmake.log - rm -f genmake.tr_log make.tr_log + rm -f SIZE.h.mpi genmake.tr_log make.tr_log ) if test -d $dir/$rundir/CVS ; then echo -n ' --- dir:' $dir/$rundir ': ' @@ -1254,7 +1369,7 @@ # Check for specific files for particular type of run - if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then + if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" != "x0" ; then echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" continue fi @@ -1262,12 +1377,49 @@ echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir" continue fi - if test "x$MPI" = "xt" -a "x$MULTI_THREAD" = "xt" ; then - retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/input/eedata.mth"` - if test $retv != 0 ; then - echo "input/eedata.mth tiling misfit -- skipping $dir" - continue - fi + + if test "x$MPI" != "x0" ; then + prefer_X=0 + if test "x$MULTI_THREAD" = "xt" ; then + retv=`check_eedata $dir"/input/eedata.mth"` + if test $retv = 1 ; then prefer_X=1 ; fi + fi + #- create new SIZE.h with no more than '$MPI' Procs + mk_mpi_size $CODE_DIR"/SIZE.h_mpi" $BUILD_DIR"/tr_size.mpi" $MPI $prefer_X + LOC_NPROC=$? + ( cd $BUILD_DIR + if test -r SIZE.h.mpi ; then + cmp tr_size.mpi SIZE.h.mpi > /dev/null 2>&1 ; RETVAL=$? + else RETVAL=1 + fi + if test "x$RETVAL" = x0 ; then + rm -f tr_size.mpi + else + rm -f SIZE.h.mpi ; mv tr_size.mpi SIZE.h.mpi + fi + ) + if test "x$MPI_MFILE" != x ; then + #- create new MPI machine-file with the right number of Procs + rm -f $LOC_MFILE + cat $MPI_MFILE | sort | uniq | head -$LOC_NPROC > $LOC_MFILE + nl=`wc -l $LOC_MFILE | awk '{print $1}'` + if [ $nl -lt $LOC_NPROC ] ; then + rm -f $LOC_MFILE + cat $MPI_MFILE | head -$LOC_NPROC > $LOC_MFILE + #sed -n "1,$LOC_NPROC p" $MPI_MFILE > $LOC_MFILE + fi + if [ $verbose -gt 1 ]; then + nl=`wc -l $LOC_MFILE | awk '{print $1}'` + echo " new LOC_MFILE=$LOC_MFILE : $nl procs for LOC_NPROC=$LOC_NPROC" + fi + fi + if test "x$MULTI_THREAD" = "xt" ; then + retv=`check_eedata $dir"/input/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"` + if test $retv != 0 ; then + echo "input/eedata.mth tiling misfit -- skipping $dir" + continue + fi + fi fi # Check whether there are "extra runs" for this testdir @@ -1283,15 +1435,15 @@ if test -f $outf -a -r $outf ; then if test "x$MULTI_THREAD" = "xt" ; then if test -r $dir"/"$exd"/eedata.mth" ; then - if test "x$MPI" = "xt" ; then - retv=`check_eedata $CODE_DIR"/SIZE.h_mpi" $dir"/"$exd"/eedata.mth"` + if test "x$MPI" = "x0" ; then + extra_runs="$extra_runs $name" + else + retv=`check_eedata $dir"/"$exd"/eedata.mth" $BUILD_DIR"/SIZE.h.mpi"` if test $retv = 0 ; then extra_runs="$extra_runs $name" else echo $exd"/eedata.mth tiling misfit -- skipping $dir" fi - else - extra_runs="$extra_runs $name" fi #else echo $dir"/"$exd"/eedata.mth: not found" fi @@ -1311,9 +1463,13 @@ unset genmake makedepend make run results=$EMPTY_RESULTS - # Create an output dir for each OPTFILE/tdir combination + # Create an output dir & summary.txt file for each tested experiment (tdir) locDIR=$DRESULTS"/"$dir mkdir $locDIR + #- report to this experiment local summary file --- + echo "DATE='$DATE' ; tdir='$dir'" > $locDIR"/summary.txt" + echo "MACH='$MACH'" >> $locDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" CDIR=`pwd`"/$locDIR" if test "x$NORUN" = xt ; then @@ -1339,26 +1495,20 @@ fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo 1>&2 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY - touch $locDIR"/summary.txt" echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" - echo "MACH='$MACH'" >> $locDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" - echo "DATE='$DATE'" >> $locDIR"/summary.txt" - echo "tdir='$dir'" >> $locDIR"/summary.txt" - if test "x$ADM" = xt ; then - head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt" - grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \ - >> $locDIR"/summary.txt" - fi for ex in $extra_runs ; do unset run results=$EMPTY_RESULTS # reference output file refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"` - # Create an output dir for each OPTFILE/tdir.ex combination + # Create an output dir & summary.txt file for each extra run (tdir.ex) locDIR=$DRESULTS"/"$dir"."$ex mkdir $locDIR + #- report to this experiment local summary file --- + echo "DATE='$DATE' ; tdir='$dir.$ex'" > $locDIR"/summary.txt" + #echo "MACH='$MACH'" >> $locDIR"/summary.txt" + #echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" CDIR=`pwd`"/$locDIR" test ! -e "$dir/$pfxdir.$ex" && mkdir "$dir/$pfxdir.$ex" run_clean $dir/$pfxdir.$ex @@ -1369,18 +1519,8 @@ fres="$fres.$ex" echo 1>&2 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY - touch $locDIR"/summary.txt" echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" - echo "MACH='$MACH'" >> $locDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $locDIR"/summary.txt" - echo "DATE='$DATE'" >> $locDIR"/summary.txt" - echo "tdir='$dir.$ex'" >> $locDIR"/summary.txt" - if test "x$ADM" = xt ; then - head -1 $dir/$builddir/taf_ad.log >> $locDIR"/summary.txt" - grep -A3 'Seconds in section "ALL' $dir/$pfxdir.$ex/$OUTPUTFILE \ - >> $locDIR"/summary.txt" - fi - if test "x$POSTCLEAN" = xt ; then + if test "x$POSTCLEAN" = x2 ; then run_clean $dir/$pfxdir.$ex fi done @@ -1390,14 +1530,37 @@ mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'` echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state - echo $mkOpt >> $DRESULTS/genmake_state + echo " $mkOpt" >> $DRESULTS/genmake_state + if test "x$OPTFILE" = xNONE ; then + eval $mkOpt + sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\ + $SUMMARY > tr_0.tmp_log + RETVAL=$? + if test "x$RETVAL" = x0 ; then rm -f $SUMMARY + cp tr_0.tmp_log $SUMMARY + else rm -f tr_0.tmp_log + fi + fi + gmkLog=$dir/$builddir/genmake.log + grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" = x0 ; then + echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state + sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \ + | grep -v '^... compiler version ' > tr_1.tmp_log + sed -n '1p' tr_1.tmp_log >> $DRESULTS/genmake_state + sed -n '2,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \ + >> $DRESULTS/genmake_state + rm -f tr_1.tmp_log + fi fi fi #postclean $dir/$builddir - if test "x$POSTCLEAN" = xt ; then + if test "x$POSTCLEAN" = x2 ; then makeclean $dir/$builddir \ && run_clean $dir/$rundir fi + if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then rm -f $LOC_MFILE ; fi echo "-------------------------------------------------------------------------------"