--- MITgcm/verification/testreport 2003/09/01 16:50:27 1.2 +++ MITgcm/verification/testreport 2003/11/29 00:58:43 1.22 @@ -1,6 +1,7 @@ -#!/bin/bash +#! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.2 2003/09/01 16:50:27 edhill Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.22 2003/11/29 00:58:43 edhill Exp $ +# $Name: $ # usage() @@ -10,9 +11,31 @@ echo echo "where possible OPTIONS are:" echo " (-help|-h) print usage" - echo " (-optfile=|-of=)STRING list of genmake2 \"optfiles\"" - echo " (-a|-addr)STRING list of email recipients" - echo " (-t|-tdir)STRING list of \"test\" dirs" + echo " (-mpi) use MPI input files" + echo " (-ieee|-noieee) if possible, use IEEE compiler flags" + echo " (DEF=\"noieee\")" + echo " (-optfile=|-of=)STRING list of optfiles to use" + echo " (-a|-addr) STRING list of email recipients" + echo " (DEF=\"edhill@mitgcm.org\")" + echo " (-t|-tdir) STRING list of test dirs to use" + echo " (DEF=\"\" which builds all)" + echo " (-b|-bash) STRING preferred location of a \"bash\" or" + echo " Bourne-compatible \"sh\" shell" + echo " (DEF=\"\" for \"bash\")" + echo " (-command) STRING command to run" + echo " (DEF=\"make output.txt\")" + echo " (-m|-make) STRING command to use for \"make\"" + echo " (DEF=\"make\")" + echo " (-clean) *ONLY* run \"make CLEAN\"" + echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\"" + echo " (-nogenmake|-ng) skip the genmake stage" + echo " (-noclean|-nc) skip the \"make clean\" stage" + echo " (-nodepend|-nd) skip the \"make depend\" stage" + echo + echo "and where STRING follows a whitespace-delimited format" + echo "such as:" + echo " -t 'exp0 exp2 exp3' " + echo " -addr='abc@123.com testing@home.org'" echo exit 1 } @@ -23,19 +46,26 @@ echo -n "building the mpack utility... " if test ! -x "$MPACKDIR/mpack" ; then if test ! -d $MPACKDIR ; then + echo echo "Error: can't find \"$MPACKDIR\"" echo " are you sure this program is being run in the correct " echo " (that is, \"MITGCM_ROOT\verification\") directory?" - exit 1 + echo + HAVE_MPACK=f fi echo -n "building mpack... " - ( cd $MPACKDIR && ./configure && make ) > build_mpack.out 2>&1 + ( cd $MPACKDIR && ./configure && $MAKE ) > build_mpack.out 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then echo echo "Error building the mpack tools at: $MPACK_DIR" - exit 1 + echo + HAVE_MPACK=f + else + HAVE_MPACK=t fi + else + HAVE_MPACK=t fi echo "OK" } @@ -60,7 +90,7 @@ echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2 fi if [ -r $1/$4/output.txt ]; then - grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt + grep "$2" $1/$4/output.txt | sed 's/.*=//' | cat -n > tmp1.txt lncnt=`wc -l tmp1.txt | awk '{print $1}' ` if [ $lncnt -lt 3 ]; then if [ $verbose -gt 0 ]; then @@ -75,7 +105,7 @@ if [ $debug -gt 0 ]; then echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2 fi - grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt + grep "$2" $1/results/output.txt | sed 's/.*=//' | cat -n > tmp2.txt lncnt=`wc -l tmp2.txt | awk '{print $1}' ` if [ $lncnt -lt 3 ]; then if [ $verbose -gt 0 ]; then @@ -90,8 +120,11 @@ if [ $debug -gt 0 ]; then echo testoutput_for_prop: compare_lines 1>&2 fi - compare_lines - digits_of_similarity=$? + if [ $verbose -gt 1 ]; then + cat tmp3.txt 1>&2 + fi + echo "-1" >> tmp3.txt + digits_of_similarity=`./tmp_cmpnum < tmp3.txt` if [ $digits_of_similarity -eq 99 ]; then if [ $verbose -gt 0 ]; then echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2 @@ -135,7 +168,6 @@ if [ $debug -gt 0 ]; then echo testoutput: cg2dres=$cg2dres 1>&2 fi - testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$? testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$? testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$? @@ -160,68 +192,84 @@ genmakemodel() { # genmakemodel directory - GENMAKE2="../../../tools/genmake2" - ( - cd $1; - printf 'genmake ... ' 1>&2 - # ../../../tools/genmake -ieee -mods=../code > make.log 2>&1 - $GENMAKE2 -ds --mods=../code "--optfile="$OPTFILE > make.log 2>&1 - RETVAL=$? - for i in gm_state gm_optfile gm_local Makefile ; do - if test -r $i ; then - cp $i $CDIR + if test "x$NOGENMAKE" = xt ; then + echo "genmake skipped!" + else + GENMAKE2="$BASH ../../../tools/genmake2" + ( + cd $1; + command="$GENMAKE2 -ds -m $MAKE --mods=../code" + if test "x$OPTFILE" != xNONE ; then + command="$command --optfile=$OPTFILE" fi - done - if test "x$RETVAL" != x0 ; then - tail make.log - echo "genmakemodel: genmake failed" 1>&2 - cp make.log $CDIR - return 1 - else - echo "succesful" 1>&2 - fi - ) + if test "x$IEEE" != x ; then + command="$command -ieee" + fi + printf 'genmake ... ' 1>&2 + $command > make.log 2>&1 + RETVAL=$? + cp Makefile $CDIR + if test "x$RETVAL" != x0 ; then + tail make.log + echo "genmakemodel: genmake failed" 1>&2 + cp genmake_* make.log $CDIR + return 1 + else + echo "successful" 1>&2 + fi + ) + fi } makeclean() { # makeclean directory - ( - cd $1; - rm -f output.txt - printf 'make CLEAN ... ' 2>&1 - if test -r Makefile ; then - make CLEAN >> make.log 2>&1 - RETVAL=$? - if test "x$RETVAL" != x0 ; then - tail make.log - echo "makeclean: \"make CLEAN\" failed" 1>&2 - cp make.log $CDIR"/make.log" - return 1 + if test "x$NOCLEAN" = xt ; then + echo "make CLEAN skipped!" + else + ( + cd $1; + if test -e output.txt ; then + rm -f output.txt fi - fi - echo succesful 1>&2 - exit 0 - ) + printf 'make CLEAN ... ' 2>&1 + if test -r Makefile ; then + $MAKE CLEAN >> make.log 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + tail make.log + echo "makeclean: \"make CLEAN\" failed" 1>&2 + cp make.log $CDIR"/make.log" + return 1 + fi + fi + echo successful 1>&2 + exit 0 + ) + fi } makedependmodel() { # makedependmodel directory - ( - cd $1; - printf 'make depend ... ' 1>&2 - make depend >> make.log 2>&1 - RETVAL=$? - if test "x$RETVAL" != x0 ; then - tail make.log - echo "makedependmodel: make depend failed" 1>&2 - cp make.log $CDIR"/make.log" - return 1 - else - echo succesful 1>&2 - fi - ) + if test "x$NODEPEND" = xt ; then + echo "make depend skipped!" + else + ( + cd $1; + printf 'make depend ... ' 1>&2 + $MAKE depend >> make.log 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + tail make.log + echo "makedependmodel: make depend failed" 1>&2 + cp make.log $CDIR"/make.log" + return 1 + else + echo successful 1>&2 + fi + ) + fi } makemodel() @@ -231,7 +279,7 @@ cd $1; if test -r Makefile ; then printf 'make ... ' 1>&2 - make >> make.log 2>&1 + $MAKE >> make.log 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.log @@ -239,7 +287,7 @@ cp make.log $CDIR"/make.log" return 1 else - echo succesful 1>&2 + echo successful 1>&2 fi fi ) @@ -250,31 +298,40 @@ # linkdata flag # # symbolically link data files to run directory - if [ $1 -ne 0 ]; then - ( cd $2 ; ln -sf ../input/* . ) + if test "x$1" = x1 ; then + ( + cd $2 + files=`( cd ../input ; ls -1 | grep -v CVS )` + for i in $files ; do + if test ! -d "../input/"$i ; then + ln -sf "../input/"$i $i + fi + done + ) fi } runmodel() { - # runmodel directory exe + # runmodel directory # - # runs the model "exe" in "directory" (exe is relative to directory) + # runs "$COMMAND" in "directory" + # (where "$COMMAND" is relative to "directory") ( cd $1 - if [ -x $2 ]; then - if [ $quick -eq 0 ]; then - rm -f output.txt - fi - printf 'runmodel: ' 1>&2 - make output.txt - RETVAL=$? - if test "x$RETVAL" = x0 ; then - cp output.txt $CDIR"/output.txt" - return 0 - else - return 1 - fi + printf 'runmodel ... ' 1>&2 + # make output.txt + $COMMAND >> run.log 2>&1 + RETVAL=$? + if test "x$RETVAL" = x0 ; then + echo successful 1>&2 + cp output.txt $CDIR"/output.txt" + return 0 + else + tail run.log + echo failed 1>&2 + cp run.log $CDIR"/run.log" + return 1 fi ) } @@ -284,38 +341,41 @@ # create codelet for comparing model output echo -n "creating the comparison code... " - cat > tmp_cmpnum.f < tmp_cmpnum.c < +#include +int main( int argc, char** argv ) { + int linnum,best; + double a,b,diff; + best = -16; + while( 1 ) { + scanf("%d", &linnum); + if (linnum == -1) break; + scanf("%lf", &a); scanf("%lf", &b); + diff = 0.5*(fabs(a)+fabs(b)); + if (diff > 1.e-12) { + diff=fabs(a-b)/diff; + if (diff > 0.0) { + linnum = (int)log10(diff); + best = (best > linnum) ? best : linnum; + } + else { + if (best == -16 && diff != 0) best = -22; + } + } + } + printf("%d\n", -best); + return 0; +} +EOF + cc -o tmp_cmpnum tmp_cmpnum.c -lm - f77 tmp_cmpnum.f - if [ -x ./a.out ]; then + if [ -x ./tmp_cmpnum ]; then echo "OK" return 0 else echo - echo "createcodelet: failed to compile codelet" | tee + echo "ERROR: failed to compile comparison code" exit 1 fi } @@ -347,14 +407,13 @@ { cat - << EOF $0 [-help] [-quick] [-verbose] dir1 [dir2] [...] - -help Show this help message - -quick Skip "genmake" and "make depend" if the Makefile exists + + -help|-h Show this help message -quiet Reduce the amount of output -verbose Produce copious amounts of output -debug Produce even more output which will mean nothing to most -force Do "make CLEAN" before compiling. This forces a complete rebuild. -clean Do "make CLEAN" after compiling and testing. - -noieee By default, $0 uses the -ieee option for genmake. This turns it off. -cleanup Aggresively removes all model output, executables and object files and then exits. Use with care. @@ -376,23 +435,37 @@ ############################################################################### -############################################################################### -############################################################################### # Default properties debug=0 verbose=1 -quick=0 clean=0 -ieee=1 expts='' +# ieee=1 + +IEEE= +if test "x$MITGCM_IEEE" != x ; then + IEEE=$MITGCM_IEEE +fi -OPTFILES= -ADDRESSES=edhill@mitgcm.org + +CLEANUP=f +QUICK=f +NOGENMAKE=f +NOCLEAN=f +NODEPEND=f + +BASH= +OPTFILE=NONE +ADDRESSES= TESTDIRS= MPACKDIR="../tools/mpack-1.6" +HAVE_MPACK= MPACK="$MPACKDIR/mpack" +COMMAND="make output.txt" +MAKE=make +MPI=f echo -n "parsing options... " @@ -414,9 +487,9 @@ usage ;; -optfile | --optfile | -of | --of) - ac_prev=OPTFILES ;; + ac_prev=OPTFILE ;; -optfile=* | --optfile=* | -of=* | --of=*) - OPTFILES=$ac_optarg ;; + OPTFILE=$ac_optarg ;; -addr | --addr | -a | --a) ac_prev=ADDRESSES ;; @@ -428,11 +501,40 @@ -tdir=* | --tdir=*) TESTDIRS=$ac_optarg ;; - -quick) quick=1 ;; + -bash | --bash | -b | --b) + ac_prev=BASH ;; + -bash=* | --bash=*) + BASH=$ac_optarg ;; + + -command | --command | -c | --c) + ac_prev=COMMAND ;; + -command=* | --command=*) + COMMAND=$ac_optarg ;; + + -make | --make | -m | --m) + ac_prev=MAKE ;; + -make=* | --make=*) + MAKE=$ac_optarg ;; + + -clean | --clean) + CLEANUP=t ;; + + -quick | --quick | -q | --q) + QUICK=t ;; + -nogenmake | --nogenmake | -ng | --ng) + NOGENMAKE=t ;; + -noclean | --noclean | -nc | --nc) + NOCLEAN=t ;; + -nodepend | --nodepend | -nd | --nd) + NODEPEND=t ;; + + -mpi) MPI=t ;; + + -ieee) IEEE=true ;; + -noieee) IEEE= ;; + -verbose) verbose=2 ;; -debug) debug=1 ;; - -clean) clean=1 ;; - -noieee) ieee=0 ;; -quiet) verbose=0 ;; -*) @@ -449,10 +551,20 @@ done +if test "x$QUICK" = xt ; then + NOGENMAKE=t + NOCLEAN=t + NODEPEND=t +fi + if test "x$TESTDIRS" = x ; then TESTDIRS=`scandirs` fi +if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then + OPTFILE=$MITGCM_OF +fi + echo "OK" # create the FORTRAN comparison code @@ -475,12 +587,42 @@ mkdir $DRESULTS RETVAL=$? if test "x$RETVAL" != x0 ; then - echo "Error: can't create results directory \"./$DRESULTS\"" + echo "ERROR: Can't create results directory \"./$DRESULTS\"" exit 1 fi SUMMARY="$DRESULTS/summary.txt" -date > $SUMMARY -cat << EOF >> $SUMMARY +echo -n "Start time: " >> $SUMMARY +start_date=`date` +echo $start_date > $SUMMARY + +of_path= +if test "x$OPTFILE" != xNONE ; then + if test -r $OPTFILE ; then + # get the path + path=${OPTFILE%/*} + if test "x$path" = x ; then + of_path=`pwd` + else + of_path=`( cd $path > /dev/null 2>&1 ; pwd )` + fi + file=${OPTFILE##*/} + OPTFILE=$of_path/$file + cp $OPTFILE $DRESULTS + echo >> $SUMMARY + echo " OPTFILE=$OPTFILE" >> $SUMMARY + else + echo | tee $SUMMARY + echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY + exit 1 + fi +else + echo >> $SUMMARY + echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY + echo " so the genmake default will be used." >> $SUMMARY +fi +echo +echo >> $SUMMARY +cat << EOF | tee -a $SUMMARY T S U V G D M c m s m s m s m s E p a R g m m e . m m e . m m e . m m e . @@ -489,98 +631,155 @@ EOF -NDIR=0 - -# For each optfile... -for OPTFILE in $OPTFILES ; do +# ...and each test directory... +for dir in $TESTDIRS ; do + + # Cleanup only! + if test "x$CLEANUP" = xt ; then + if test -r $dir/build/Makefile ; then + ( cd $dir/build ; make CLEAN ) + fi + if test -r $dir/input/Makefile ; then + ( cd $dir/input ; make CLEAN ) + fi + continue + fi - OPTFILE=`pwd`"/$OPTFILE" - if test ! -r $OPTFILE ; then - echo "Error: can't read optfile \"$OPTFILE\"" - exit 1 + # Verify that the testdir exists and contains previous + # results in the correct location--or skip this directory! + if test ! -r $dir"/results/output.txt" ; then + echo "can't read \"$dir/results/output.txt\" -- skipping $dir" + continue fi - echo - echo "OPTFILE=$OPTFILE" >> $SUMMARY - echo >> $SUMMARY - - # ...and each test directory... - for dir in $TESTDIRS ; do - - # Create an output dir for each OPTFILE/tdir combination - CDIR=$DRESULTS"/"$DRESULTS"_"$NDIR - mkdir $CDIR - CDIR=`pwd`"/$CDIR" - # ...configue, make, run, and compare the output. - echo "-------------------------------------------------------------------------------" - echo - echo "Experiment: $dir" - echo - unset genmake makedepend make run - results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --' - ( cd $dir/input; rm -f *.{o,f,c,F} *.[f,F]90 work* output.txt Make* make.log; ) - if [ -r $dir/build ]; then - seperatebuilddir=1 - builddir=build - rundir=build - ( cd $dir/build; ln -sf ../input/* . ) - else - seperatebuilddir=0 - builddir=input - rundir=input - fi + echo "-------------------------------------------------------------------------------" + echo + echo "Experiment: $dir" + echo + unset genmake makedepend make run + results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --' - # Verify that the testdir exists and contains previous - # results in the correct location--or skip this directory! - if test ! -r $dir"/results/output.txt" ; then - echo | tee $SUMMARY - echo "can't read \"$dir/results/output.txt\" -- skipping $dir" \ - | tee $SUMMARY - continue + builddir="input" + rundir="input" + use_seperate_build=0 + if test -d $dir/build -a -r $dir/build ; then + builddir="build" + rundir="build" + use_seperate_build=1 + linkdata $use_seperate_build $dir/$rundir + fi + + CODE_DIR=$dir/code + BUILD_DIR=$dir/$builddir + MPI_FILES="CPP_EEOPTIONS.h_mpi SIZE.h_mpi" + NOMPI_FILES="CPP_EEOPTIONS.h_nompi SIZE.h_nompi" + + # Is this an MPI run? + if test "x$MPI" = xt ; then + FILES=$MPI_FILES + endings="_mpi" + else + FILES=$NOMPI_FILES + endings="_nompi" + fi + + # Check to see that we have the files + have_files=t + for i in $FILES ; do + if test ! -r $CODE_DIR/$i ; then + echo "Warning: can't read file $CODE_DIR/$i" + have_files=f fi - + done + if test "x$have_files" != xt -a "x$MPI" = xt ; then + echo "Skipping $dir due to lack of input files (see above warning)" + continue + fi + + # If we have the $FILES and they differ, copy the $FILES to $BUILD_DIR + if test "x$have_files" = xt ; then + for i in $FILES ; do + sstr="s|$endings||" + name=`echo $i | sed -e $sstr ` + cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + cp $CODE_DIR/$i $BUILD_DIR/$name + fi + done + fi + + # Create an output dir for each OPTFILE/tdir combination + rel_CDIR=$DRESULTS"/"$dir + mkdir $rel_CDIR + CDIR=`pwd`"/$rel_CDIR" + + if test "x$CLEANUP" = xt ; then + makeclean $dir/$builddir + else genmakemodel $dir/$builddir && genmake=Y \ && makeclean $dir/$builddir \ && makedependmodel $dir/$builddir && makedepend=Y \ && makemodel $dir/$builddir && make=Y \ - && linkdata $seperatebuilddir $dir/$rundir \ - && runmodel $dir/$builddir mitgcmuv && run=Y \ + && linkdata $use_seperate_build $dir/$rundir \ + && runmodel $dir/$rundir && run=Y \ && results=`testoutput $dir $rundir` - echo - formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \ - ${run:-N} $results - echo - formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \ - ${run:-N} $results >> $SUMMARY - echo "fresults='" > $CDIR"/summary.txt" - formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} \ - ${run:-N} $results >> $CDIR"/summary.txt" - echo "'" >> $CDIR"/summary.txt" - echo "MACH='$MACH'" >> $CDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" - echo "DATE='$DATE'" >> $CDIR"/summary.txt" - echo "tdir='$dir'" >> $CDIR"/summary.txt" - - ( - cd $DRESULTS - tar -cf $NDIR".tar" $DRESULTS"_"$NDIR > /dev/null 2>&1 - gzip $NDIR".tar" - ) + fi + + echo + fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` + echo + echo "$fres" >> $SUMMARY + echo "fresults='$fres'" > $CDIR"/summary.txt" + echo "MACH='$MACH'" >> $CDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" + echo "DATE='$DATE'" >> $CDIR"/summary.txt" + echo "tdir='$dir'" >> $CDIR"/summary.txt" + + echo "-------------------------------------------------------------------------------" + +done - $MPACK -s MITgcm-test -m 1000000 $DRESULTS"/"$NDIR".tar.gz" $ADDRESSES +echo -n "Start time: " >> $SUMMARY +echo $start_date >> $SUMMARY +echo -n "End time: " >> $SUMMARY +date >> $SUMMARY + +# If addresses were supplied and mpack built successfully, then try +# to send email using mpack. +if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then + echo "No results email was sent." +else + if test "x$HAVE_MPACK" = xt ; then + tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \ + && gzip $DRESULTS".tar" \ + && $MPACK -s MITgcm-test -m 1500000 $DRESULTS".tar.gz" $ADDRESSES RETVAL=$? if test "x$RETVAL" != x0 ; then - echo "Warning: \"$MPACK\" failed -- please contact " + echo + echo "Warning: The tar, gzip, & mpack step failed. Please send email" + echo " to for help. You may copy the " + echo " summary of results from the directory \"$DRESULTS\"." + echo else - rm -f $DRESULTS"/"$NDIR".tar*" + echo + echo "An email containing results was sent to the following addresses:" + echo " \"$ADDRESSES\"" + echo fi + test -f $DRESULTS".tar" && rm -f $DRESULTS".tar" + test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz" + fi +fi - NDIR=$(( $NDIR + 1 )) - - done -done - -rm tmp_cmpnum.f a.out +# rm -f tmp_cmpnum.f a.out +rm -f tmp_cmpnum.c tmp_cmpnum -cat $SUMMARY +if test "x$CLEANUP" != xt ; then + cat $SUMMARY + if test -e tr.out ; then + mv tr.out tr.out.old + fi + cat $SUMMARY > tr.out +fi