--- MITgcm/verification/testreport 2003/08/27 22:12:31 1.1 +++ MITgcm/verification/testreport 2007/09/06 22:31:45 1.94 @@ -1,6 +1,7 @@ -#!/bin/bash +#! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.1 2003/08/27 22:12:31 edhill Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.94 2007/09/06 22:31:45 jmc Exp $ +# $Name: $ # usage() @@ -10,9 +11,51 @@ echo echo "where possible OPTIONS are:" echo " (-help|-h) print usage" - echo " (-optfile=|-of=)STRING list of genmake2 \"optfiles\"" - echo " (-a|-addr)STRING list of email recipients" - echo " (-t|-tdir)STRING list of \"test\" dirs" + echo " (-mth) run multi threaded (using eedata.mth)" + echo " (-mpi) compile and run using MPI" + echo " (-ieee|-noieee) if possible, use IEEE compiler flags" + echo " (DEF=\"-ieee\")" + echo " (-of=|-optfile=)STRING list of optfiles to use" + echo " (-a|-addr) STRING list of email recipients" + echo " (DEF=\"edhill@mitgcm.org\")" + echo " (-t|-tdir) STRING list of group and/or exp. dirs to test" + echo " (recognized groups: basic, tutorials)" + echo " (DEF=\"\" which test all)" + echo " (-skd|-skipdir) STRING list of exp. dirs to skip" + echo " (DEF=\"\" which test all)" + echo " (-b|-bash) STRING preferred location of a \"bash\" or" + echo " Bourne-compatible \"sh\" shell" + echo " (DEF=\"\" for \"bash\")" + echo " (-adm|-ad) perform an adjoint run" + echo " (-command) STRING command to run" + echo " (DEF=\"make output.txt\")" + echo " (-m|-make) STRING command to use for \"make\"" + echo " (DEF=\"make\")" + echo " (-odir) STRING used to build output directory name" + echo " (DEF=\"hostname\")" + echo " (-ptr|-ptracers) STRING specify which ptracers to test" + echo " (DEF=\"1 2 3 4 5\")" + echo " (-match) NUMBER Matching Criteria (number of digits)" + echo " (DEF=\"12\")" + echo " (-j) JOBS use \"make -j JOBS\" for parallel builds" + echo " (-clean) *ONLY* run \"make CLEAN\"" + echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\"" + echo " (-nogenmake|-ng) skip the genmake stage" + echo " (-noclean|-nc) skip the \"make clean\" stage" + echo " (-nodepend|-nd) skip the \"make depend\" stage" + echo " (-deldir|-dd) on success, delete the output directory" + echo " (-ts) provide timing information per timestep" + echo " (-papis) provide MFlop/s per timestep using PAPI" + echo " (-pcls) provide MFlop/s per timestep using PCL" + echo + echo "and where STRING can be a whitespace-delimited list" + echo "such as:" + echo + echo " -t 'exp0 exp2 exp3' " + echo " -addr='abc@123.com testing@home.org'" + echo + echo "provided that the expression is properly quoted within the current" + echo "shell (note the use of single quotes to protect white space)." echo exit 1 } @@ -20,81 +63,105 @@ # build the mpack utility build_mpack() { - echo -n "building the mpack utility... " + printf "building the mpack utility... " if test ! -x "$MPACKDIR/mpack" ; then if test ! -d $MPACKDIR ; then + echo echo "Error: can't find \"$MPACKDIR\"" echo " are you sure this program is being run in the correct " echo " (that is, \"MITGCM_ROOT\verification\") directory?" - exit 1 + echo + HAVE_MPACK=f + fi + printf "building mpack... " + if test "x$CC" = x ; then + export CC=cc fi - echo -n "building mpack... " - ( cd $MPACKDIR && ./configure && make ) > build_mpack.out 2>&1 + ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then echo echo "Error building the mpack tools at: $MPACK_DIR" - exit 1 + echo + HAVE_MPACK=f + else + rm -f tr_build_mpack.out + HAVE_MPACK=t fi + else + HAVE_MPACK=t fi echo "OK" } -compare_lines() -{ - # use codelet to compare lines - if [ $verbose -gt 1 ]; then - cat tmp3.txt 1>&2 - fi - return `./a.out < tmp3.txt` -} - -testoutput_for_prop() +testoutput_var() { - # testoutput_for_prop dir s1 label subdir + # testoutput_var dir s1 label subdir extension # - # compares files in $dir/$subdir/output.txt and $dir/results/output.txt - # using search strings s1 and text label + # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE + # with same output from reference file $dir/results/output.$extension + # using search strings s1 and text label if [ $debug -gt 0 ]; then - echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2 + echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2 fi - if [ -r $1/$4/output.txt ]; then - grep "$2" $1/$4/output.txt | sed 's/.*=//' | nl > tmp1.txt - lncnt=`wc -l tmp1.txt | awk '{print $1}' ` - if [ $lncnt -lt 3 ]; then + if [ -r $1/$4/$OUTPUTFILE ]; then + grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt + lncntA=`wc -l tmp1.txt | awk '{print $1}' ` + if [ $lncntA -lt 3 ]; then if [ $verbose -gt 0 ]; then echo Not enough lines of output when searching for "$2" 1>&2 fi return 99 fi else - echo testoutput_for_prop: output.txt from model run was not readable 1>&2 + echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2 return 99 fi if [ $debug -gt 0 ]; then - echo testoutput_for_prop: grep "$2" $1/results/output.txt 1>&2 + echo testoutput_var: grep "$2" $1/results/output.$5 1>&2 fi - grep "$2" $1/results/output.txt | sed 's/.*=//' | nl > tmp2.txt - lncnt=`wc -l tmp2.txt | awk '{print $1}' ` - if [ $lncnt -lt 3 ]; then + grep "$2" $1/results/output.$5 | sed 's/.*=//' | cat -n > tmp2.txt + lncntB=`wc -l tmp2.txt | awk '{print $1}' ` + if [ $lncntB -lt 3 ]; then if [ $verbose -gt 0 ]; then echo Not enough lines of output when searching for "$2" 1>&2 fi return 99 fi + if [ $lncntA -ne $lncntB ]; then + if [ $verbose -gt 0 ]; then + echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2 + fi + return 99 + fi + has_nan=`cat tmp1.txt | grep -i nan | wc -l` + if [ $has_nan -gt 0 ] ; then + echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2 + return 99 + fi + has_inf=`cat tmp1.txt | grep -i inf | wc -l` + if [ $has_inf -gt 0 ] ; then + echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2 + return 99 + fi if [ $debug -gt 0 ]; then - echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2 + echo testoutput_var: join tmp1.txt tmp2.txt 1>&2 fi join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt if [ $debug -gt 0 ]; then - echo testoutput_for_prop: compare_lines 1>&2 + echo testoutput_var: compare_lines 1>&2 fi - compare_lines - digits_of_similarity=$? + if [ $verbose -gt 1 ]; then + cat tmp3.txt 1>&2 + fi + echo "-1" >> tmp3.txt + # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number + cat tmp3.txt | sed -e 's|:||g' > tmp4.txt + digits_of_similarity=`./tr_cmpnum < tmp4.txt` if [ $digits_of_similarity -eq 99 ]; then if [ $verbose -gt 0 ]; then - echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2 + echo testoutput_var: No comparison was available for \"$2\" 1>&2 fi digits_of_similarity=99 else @@ -102,7 +169,7 @@ echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2 fi fi - rm tmp1.txt tmp2.txt tmp3.txt + rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt return $digits_of_similarity } @@ -122,106 +189,258 @@ done } -testoutput() +testoutput_ad() { - # testoutput diretory subdir - # - # test output in "directory" + grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $5}' > t05.txt + grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $5}' > t15.txt + grep $3 $1/results_ad/output_adm.txt | awk '{print NR " " $6}' > t06.txt + grep $3 $1/$2/$OUTPUTFILE | awk '{print NR " " $6}' > t16.txt + join t05.txt t15.txt > t5.txt + join t06.txt t16.txt > t6.txt + echo "-1" >> t5.txt + echo "-1" >> t6.txt + digits_5=`./tr_cmpnum < t5.txt` + digits_6=`./tr_cmpnum < t6.txt` + dashnum $digits_5 $digits_6 + rm -f t[01][56].txt t[56].txt +} - if [ $debug -gt 0 ]; then - echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2 +check_for_add_mon_output() +{ + # Check for additional types of monitor output + if test "x$1" = x ; then + return fi - testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2; cg2dres=$? - if [ $debug -gt 0 ]; then - echo testoutput: cg2dres=$cg2dres 1>&2 + + for ii in $PTRACERS_NUM ; do + eval "HAVE_PTR0"$ii"=f" + done + + ptr_add="trcstat_ptracerXX_min trcstat_ptracerXX_max" + ptr_add="$ptr_add trcstat_ptracerXX_mean trcstat_ptracerXX_sd" + for ii in $PTRACERS_NUM ; do + for jj in $ptr_add ; do + name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"` + tst=`grep $name $1 | wc -l | awk '{print $1}'` + if test ! "x$tst" = x0 ; then + eval "HAVE_PTR0"$ii"=t" + fi + done + # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' + done +} + +testoutput_run() +{ + # testoutput_run directory subdir extension + # + # test output from 1 run in "directory" + if test "x$ADM" = x ; then + # default list of output variables to be checked: + # 1rst : main variable used to decide if it pass or FAIL + # others : number of matching digits to be printed in summary.txt + listChk=$DEF_CHECK_LIST + nbDef=`echo $listChk | awk '{print NF-1+3*gsub("+","E")}'` + # load experiment-specific list from file "tr_checklist" (if it exist) + if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi + sVar=`echo $listChk | awk '{print $1}'` + # remove 1rst var and expand the list: + => min max mean s.d + listVar=`echo $listChk | awk '{ for(i=2;i<=NF;i++){if (sub("+","",$i)) \ + {printf " %s %s %s %s",$i"mn",$i"mx",$i"av",$i"sd"} else {printf " %s",$i} }}'` + if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi + for ii in 1 2 3 4 5 6 7 8 9 ; do + tst=`eval 'echo "$HAVE_PTR0'$ii'"'` + #echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2 + if test "x$tst" != xt ; then listVar=`echo "$listVar" | sed "s/ pt$ii..//g"` ; fi + done + tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){t+=($i==$1)}; print t }'` + if test $tst != 1 ; then + if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2 + else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi + echo "==> WARNING: in checked list:" $listVar 1>&2 + #- put it back once: + listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"` + fi + if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi + echo "listVar='$listVar'" > $CDIR"/summary.txt" + allargs="" + for xx in $listVar + do + case $xx in + 'PS') if [ $debug -gt 0 ] + then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi + testoutput_var $1 "cg2d_init_res" "cg2d init. residual" $2 $3; yy=$? + if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;; + 'Tmn') testoutput_var $1 "dynstat_theta_min" "theta minimum" $2 $3; yy=$? ;; + 'Tmx') testoutput_var $1 "dynstat_theta_max" "theta maximum" $2 $3; yy=$? ;; + 'Tav') testoutput_var $1 "dynstat_theta_mean" "theta mean" $2 $3; yy=$? ;; + 'Tsd') testoutput_var $1 "dynstat_theta_sd" "theta s.d." $2 $3; yy=$? ;; + 'Smn') testoutput_var $1 "dynstat_salt_min" "salt minimum" $2 $3; yy=$? ;; + 'Smx') testoutput_var $1 "dynstat_salt_max" "salt maximum" $2 $3; yy=$? ;; + 'Sav') testoutput_var $1 "dynstat_salt_mean" "salt mean" $2 $3; yy=$? ;; + 'Ssd') testoutput_var $1 "dynstat_salt_sd" "salt s.d." $2 $3; yy=$? ;; + 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;; + 'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;; + 'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;; + 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U s.d." $2 $3; yy=$? ;; + 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;; + 'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;; + 'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;; + 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V s.d." $2 $3; yy=$? ;; + 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;; + 'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;; + 'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;; + 'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_sd" $2 $3; yy=$? ;; + *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;; + esac + if test $xx = $sVar + then allargs="$allargs > $yy <" + else allargs="$allargs $yy" + fi + done + + nbVar=`echo $listVar | awk '{print NF}'` + if [ $nbVar -lt $nbDef ] ; then + #-- fill line (up to standard length) with dot: + adNul=`expr $nbDef - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'` + echo $allargs $adNul + else + echo $allargs + fi + + else + testoutput_ad $1 $2 "precision_grdchk_result" fi - - testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2; tmin=$? - testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2; tmax=$? - testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2; tmean=$? - testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2; tsd=$? - testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2; smin=$? - testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2; smax=$? - testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2; smean=$? - testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2; ssd=$? - testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2; umin=$? - testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2; umax=$? - testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2; umean=$? - testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2; usd=$? - testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2; vmin=$? - testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2; vmax=$? - testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2; vmean=$? - testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2; vsd=$? - - dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \ - $umin $umax $umean $usd $vmin $vmax $vmean $vsd } genmakemodel() { # genmakemodel directory - GENMAKE2="../../../tools/genmake2" - ( - cd $1; - printf 'genmake ... ' 1>&2 - # ../../../tools/genmake -ieee -mods=../code > make.log 2>&1 - $GENMAKE2 -ds --mods=../code "--optfile="$OPTFILE > make.log 2>&1 - RETVAL=$? - for i in gm_state gm_optfile gm_local Makefile ; do - if test -r $i ; then - cp $i $CDIR - fi - done - if test "x$RETVAL" != x0 ; then - tail make.log - echo "genmakemodel: genmake failed" 1>&2 - cp make.log $CDIR - return 1 + if test "x$NOGENMAKE" = xt ; then + echo "genmake skipped!" + else + if test "x$BASH" = x ; then + GENMAKE2="../../../tools/genmake2" else - echo "succesful" 1>&2 + GENMAKE2="$BASH ../../../tools/genmake2 -bash $BASH" fi - ) + ( + cd $1; + command="$GENMAKE2 -ds -m $MAKE" + if test "x$ADM" = x ; then + command="$command --mods=../code" + else + command="$command --mods=../code_ad" + fi + if test "x$OPTFILE" != xNONE ; then + command="$command --optfile=$OPTFILE" + fi + if test "x$IEEE" != x ; then + command="$command -ieee" + fi + if test "x$MPI" = xt ; then + command="$command -mpi" + fi + if test "x$TS" = xt ; then + command="$command -ts" + fi + if test "x$PAPIS" = xt ; then + command="$command -papis" + else + if test "x$PCLS" = xt ; then + command="$command -pcls" + fi + fi + printf 'genmake ... ' 1>&2 + $command > make.log 2>&1 + RETVAL=$? + # Reduce the size of the testing emails! + head -100 Makefile > $CDIR/Makefile_head + if test "x$RETVAL" != x0 ; then + tail make.log + echo "genmakemodel: genmake failed" 1>&2 + cp genmake_* make.log $CDIR + return 1 + else + echo "successful" 1>&2 + fi + ) + fi } makeclean() { # makeclean directory - ( - cd $1; - rm -f output.txt - printf 'make CLEAN ... ' 2>&1 - if test -r Makefile ; then - make CLEAN >> make.log 2>&1 - RETVAL=$? - if test "x$RETVAL" != x0 ; then - tail make.log - echo "makeclean: \"make CLEAN\" failed" 1>&2 - cp make.log $CDIR"/make.log" - return 1 + if test "x$NOCLEAN" = xt ; then + echo "make Clean skipped!" + else + ( + cd $1; + #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi + if test -r Makefile ; then + printf 'clean build-dir: make Clean ... ' 2>&1 + $MAKE Clean >> make.log 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + tail make.log + echo "makeclean: \"make Clean\" failed" 1>&2 + cp make.log $CDIR"/make.log" + return 1 + fi fi - fi - echo succesful 1>&2 - exit 0 - ) + echo successful 1>&2 + exit 0 + ) + fi +} + +run_clean() +{ + # run_clean directory + if test "x$NOCLEAN" = xt ; then + echo "run_clean skipped!" + else + ( + cd $1; + printf 'clean run-dir ... ' 2>&1 + # part of what is done after "make clean" when doing "make CLEAN" + find . -name "*.meta" -exec rm {} \; + find . -name "*.data" -exec rm {} \; + find . -name "fort.*" -exec rm {} \; + find . -type l -exec rm {} \; + rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime + rm -rf mnc_test_* + echo successful 1>&2 + exit 0 + ) + fi } makedependmodel() { # makedependmodel directory - ( - cd $1; - printf 'make depend ... ' 1>&2 - make depend >> make.log 2>&1 - RETVAL=$? - if test "x$RETVAL" != x0 ; then - tail make.log - echo "makedependmodel: make depend failed" 1>&2 - cp make.log $CDIR"/make.log" - return 1 - else - echo succesful 1>&2 - fi - ) + if test "x$NODEPEND" = xt ; then + echo "make depend skipped!" + else + ( + cd $1; + printf 'make depend ... ' 1>&2 + $MAKE depend >> make.log 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + tail make.log + echo "makedependmodel: make depend failed" 1>&2 + cp make.log $CDIR"/make.log" + return 1 + else + echo successful 1>&2 + fi + ) + fi } makemodel() @@ -231,7 +450,15 @@ cd $1; if test -r Makefile ; then printf 'make ... ' 1>&2 - make >> make.log 2>&1 + if test "x$ADM" = x ; then + if test "x$JOBS" = x ; then + $MAKE >> make.log 2>&1 + else + $MAKE -j $JOBS >> make.log 2>&1 + fi + else + $MAKE adall >> make.log 2>&1 + fi RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.log @@ -239,42 +466,160 @@ cp make.log $CDIR"/make.log" return 1 else - echo succesful 1>&2 + echo successful 1>&2 fi fi ) } +symlink_mpifiles() +{ + # Put special links so that MPI specific files are used + # This MUST be invoked between makeclean and makelinks because + # the Makefile will link to non-mpi files by default + + dir=$1 + code_dir=$2 + BUILD_DIR=$dir/$3 + CODE_DIR=$dir/$code_dir + + # These are files that should replace their counter-part when using -mpi + MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")` + + # Is this an MPI run? + if test "x$MPI" = xt ; then + # YES: We symbolically link these files to the build + # dir so long as there is no real file in place + for ii in $MPI_FILES ; do + i=`echo $ii | sed 's:^\./::'` + name=`echo $i | sed 's:_mpi::' ` + cmp $CODE_DIR/$i $BUILD_DIR/$name > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + if ! test -f $BUILD_DIR/$i ; then + #echo Linking $name to $i + (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name) + fi + fi + done + else + # NO: We undo any _mpi symbolically linked files + for ii in $MPI_FILES ; do + i=`echo $ii | sed 's:^\./::'` + name=`echo $i | sed 's:_mpi::' ` + if test -L $BUILD_DIR/$name ; then + cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" = x0 ; then + #echo Un-linking $name from $linktarg + rm -f $BUILD_DIR/$name + fi + fi + done + fi + +} + linkdata() { - # linkdata flag + # linkdata run_dir input_dir_1 input_dir_2 ... # # symbolically link data files to run directory - if [ $1 -ne 0 ]; then - ( cd $2 ; ln -sf ../input/* . ) + if test -d $1 ; then + ( + cd $1 ; shift + if test -r "../"$1"/eedata.mth" ; then + # found eedata.mth in 1rst input dir and it is readable + if test "x$MULTI_THREAD" = "xt" ; then + # multi-threaded test: remove symbolic link & link eedata.mth + if test -h eedata ; then rm -f eedata ; fi + if test ! -r eedata ; then + ln -sf "../"$1"/eedata.mth" eedata ; + printf 'eedata.mth ' 1>&2 + fi + else + # not multi-threaded test: remove eedata symbolic link + if test -h eedata ; then rm -f eedata ; fi + fi + fi + for ldir in $* ; do + if test -d "../"$ldir ; then + printf 'ldir='${ldir} 1>&2 + files=`( cd "../"$ldir ; ls -1 | grep -v CVS )` + for i in $files ; do + if test ! -d "../"$ldir/$i ; then + if test ! -r $i ; then + printf ' '$i 1>&2 + ln -sf "../"$ldir"/"$i $i + fi + fi + done + if test -x "../"$ldir"/"prepare_run ; then + "../"$ldir"/"prepare_run + fi + printf ' ; ' 1>&2 + fi + done + ) fi } runmodel() { - # runmodel directory exe + # runmodel directory # - # runs the model "exe" in "directory" (exe is relative to directory) + # runs "$COMMAND in "directory" + # (where "$COMMAND" is relative to "directory") ( cd $1 - if [ -x $2 ]; then - if [ $quick -eq 0 ]; then - rm -f output.txt - fi - printf 'runmodel: ' 1>&2 - make output.txt - RETVAL=$? - if test "x$RETVAL" = x0 ; then - cp output.txt $CDIR"/output.txt" - return 0 + printf 'runmodel in %s ...' $1 1>&2 + # make output.txt + echo + if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then + echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_00 + ln -sf "../"$builddir"/"$EXECUTABLE . + fi + if test ! -x $EXECUTABLE ; then + rm -f run.log ; touch run.log + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + echo " no executable:" $EXECUTABLE >> run.log + RETVAL=8 + ENDVAL=-1 + else + if test $OUTPUTFILE -ot $EXECUTABLE ; then + rm -f run.log ; touch run.log + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + ( eval $COMMAND ) >> run.log 2>&1 + RETVAL=$? else - return 1 + RETVAL=0 + if test -f run.log ; then + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1 + else + touch run.log + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1 + echo " no previous run.log: assume NORMAL END" >> run.log 2>&1 + fi fi + ENDVAL=`cat run.log | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'` + fi + rm -f run.log_00 + #if test "x$RETVAL" = x0 ; then + if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then + tail run.log + echo successful 1>&2 + # === Reduce the size of the testing emails! + #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE + if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi + return 0 + else + tail run.log + echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2 + cp run.log $CDIR"/run.log" + if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi + return 1 fi ) } @@ -283,39 +628,48 @@ { # create codelet for comparing model output - echo -n "creating the comparison code... " - cat > tmp_cmpnum.f < tr_cmpnum.c < +#include +int main( int argc, char** argv ) { + int linnum,cmplin,best,lncnt; + double a,b,abave,relerr; + best = -22; + lncnt = 0; + while( 1 & (lncnt+=1) < 999 ) { + scanf("%d", &linnum); + if (linnum == -1) break; + scanf("%lf", &a); scanf("%lf", &b); + abave = 0.5*(fabs(a)+fabs(b)); + if ( abave == abave ) { + if (abave > 0.0) { + relerr=fabs(a-b)/abave; + if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); } + else { cmplin = -16 ; } + best = (best > cmplin) ? best : cmplin; } + else { cmplin = -22 ; } + /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */ + } + else { + /* printf("%lf ; %lf ; %lf\n",abave,a,b); */ + break; } + } + if (lncnt == 999) best=-29; + if (linnum != -1) best=-99; + printf("%d\n", -best); + return 0; +} +EOF + $CC -o tr_cmpnum tr_cmpnum.c -lm - f77 tmp_cmpnum.f - if [ -x ./a.out ]; then + if [ -x ./tr_cmpnum ]; then echo "OK" return 0 else echo - echo "createcodelet: failed to compile codelet" | tee + echo "ERROR: failed to compile comparison code -- please specify" + echo " a C compiler using the CC environment variable." exit 1 fi } @@ -327,12 +681,20 @@ nm=$1 printf '%s %s %s %s' $2 $3 $4 $5 shift; shift; shift; shift; shift; - printf '%3s' $@ + listPrt=$@ + listRes=`echo $listPrt | sed 's/>//' | sed 's///' | sed 's/<.*//' | awk '{print $1}'` + printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < / / /' | sed 's/ < / /' - if [ $1 = '--' ]; then + if [ $xx = '..' ]; then + printf ' N/O ' + elif [ $xx = '--' ]; then + printf ' N/O ' + elif [ $xx = 99 ]; then printf ' N/O ' else - if [ $1 -gt 12 ]; then + if [ $xx -ge $MATCH_CRIT ]; then printf ' pass' else printf ' FAIL' @@ -343,58 +705,70 @@ } -show_help() -{ - cat - << EOF -$0 [-help] [-quick] [-verbose] dir1 [dir2] [...] - -help Show this help message - -quick Skip "genmake" and "make depend" if the Makefile exists - -quiet Reduce the amount of output - -verbose Produce copious amounts of output - -debug Produce even more output which will mean nothing to most - -force Do "make CLEAN" before compiling. This forces a complete rebuild. - -clean Do "make CLEAN" after compiling and testing. - -noieee By default, $0 uses the -ieee option for genmake. This turns it off. - -cleanup Aggresively removes all model output, executables and object files - and then exits. Use with care. - -Normal usage: - $0 * Configure, compile, run and analyze in all experiment directories -EOF -} - scandirs() { - if [ $# -eq 0 ]; then - for arg in * ; do - test -d $arg/input && echo $arg - done + if [ $# -eq 1 ]; then + for arg in * ; do + test -d $arg/$1 && echo $arg + done else - echo $* + echo $* fi } ############################################################################### -############################################################################### -############################################################################### # Default properties debug=0 verbose=1 -quick=0 clean=0 -ieee=1 expts='' +# ieee=1 + +IEEE=true +if test "x$MITGCM_IEEE" != x ; then + IEEE=$MITGCM_IEEE +fi + -OPTFILES= -ADDRESSES=edhill@mitgcm.org +CLEANUP=f +QUICK=f +NOGENMAKE=f +NOCLEAN=f +NODEPEND=f +POSTCLEAN=f + +BASH= +OPTFILE=NONE +ADDRESSES= TESTDIRS= +SKIPDIRS= MPACKDIR="../tools/mpack-1.6" +HAVE_MPACK= MPACK="$MPACKDIR/mpack" +COMMAND= +if test "x$MAKE" = x ; then + MAKE=make +fi +if test "x$CC" = x ; then + CC=cc +fi +JOBS= +MPI=f +MULTI_THREAD=f +OUTDIR= +DELDIR= + +ADM= + +# Additional monitor types +PTRACERS_NUM="1 2 3 4 5" -echo -n "parsing options... " +MATCH_CRIT=13 + +printf "parsing options... " ac_prev= for ac_option ; do @@ -413,10 +787,10 @@ -help | --help | -h | --h) usage ;; - -optfile | --optfile | -f | --f) - ac_prev=OPTFILES ;; - -optfile=* | --optfile=*) - OPTFILES=$ac_optarg ;; + -optfile | --optfile | -of | --of) + ac_prev=OPTFILE ;; + -optfile=* | --optfile=* | -of=* | --of=*) + OPTFILE=$ac_optarg ;; -addr | --addr | -a | --a) ac_prev=ADDRESSES ;; @@ -428,13 +802,78 @@ -tdir=* | --tdir=*) TESTDIRS=$ac_optarg ;; - -quick) quick=1 ;; + -skipdir | --skipdir | -skd | --skd) + ac_prev=SKIPDIRS ;; + -skipdir=* | --skipdir=*) + SKIPDIRS=$ac_optarg ;; + + -bash | --bash | -b | --b) + ac_prev=BASH ;; + -bash=* | --bash=*) + BASH=$ac_optarg ;; + + -command | --command | -c | --c) + ac_prev=COMMAND ;; + -command=* | --command=*) + COMMAND=$ac_optarg ;; + + -make | --make | -m | --m) + ac_prev=MAKE ;; + -make=* | --make=*) + MAKE=$ac_optarg ;; + + -odir | --odir) + ac_prev=OUTDIR ;; + -odir=* | --odir=*) + OUTDIR=$ac_optarg ;; + + -ptracers | --ptracers | -ptr | --ptr) + ac_prev=PTRACERS_NUM ;; + -ptracers=* | --ptracers=* | -ptr=* | --ptr=*) + PTRACERS_NUM=$ac_optarg ;; + + -match | --match ) ac_prev=MATCH_CRIT ;; + -match=* | --match=* ) MATCH_CRIT=$ac_optarg ;; + + -j) ac_prev=JOBS ;; + -j=*) JOBS=$ac_optarg ;; + + -clean | --clean) + CLEANUP=t ; DELDIR=t ;; + + -quick | --quick | -q | --q) + QUICK=t ;; + -nogenmake | --nogenmake | -ng | --ng) + NOGENMAKE=t ;; + -noclean | --noclean | -nc | --nc) + NOCLEAN=t ;; + -nodepend | --nodepend | -nd | --nd) + NODEPEND=t ;; + + -postclean | --postclean | -pc | --pc) + POSTCLEAN=t ;; + + -mpi) MPI=t ;; + + -mth) MULTI_THREAD=t ;; + + -adm | -ad) ADM=t ;; + + -ieee) IEEE=true ;; + -noieee) IEEE= ;; + -verbose) verbose=2 ;; -debug) debug=1 ;; - -clean) clean=1 ;; - -noieee) ieee=0 ;; -quiet) verbose=0 ;; + -deldir | -dd) DELDIR=t ;; + + -ts) TS=t;; + + -papis) PAPIS=t;; + + -pcls) PCL=t;; + -*) echo "Error: unrecognized option: "$ac_option usage @@ -449,22 +888,109 @@ done +if test "x$QUICK" = xt ; then + NOGENMAKE=t + NOCLEAN=t + NODEPEND=t +fi + if test "x$TESTDIRS" = x ; then - TESTDIRS=`scandirs` + if test "x$ADM" = xt ; then + LIST=`scandirs results_ad` + else + LIST=`scandirs results` + fi +else + #- expand group of experiments: + LIST=" " + for xx in $TESTDIRS + do + case $xx in + 'basic') LIST=${LIST}" aim.5l_cs hs94.128x64x5 ideal_2D_oce" + LIST=${LIST}" lab_sea tutorial_baroclinic_gyre" + LIST=${LIST}" tutorial_global_oce_latlon tutorial_plume_on_slope" + ;; + 'tutorials') + LIST=${LIST}" "`ls | grep 'tutorial_'` ;; + *) LIST=${LIST}" "$xx ;; + esac + done +fi +#echo 'LIST='${LIST}'<' +#- skip dirs, remove duplicate and non-directory: +TESTDIRS=" " +count=0 +for xx in $LIST +do + yy=`echo $SKIPDIRS | grep -c $xx` + if test $yy = 0 ; then + if test -d $xx ; then + yy=`echo $TESTDIRS | grep -c $xx` + if test $yy = 0 ; then TESTDIRS=${TESTDIRS}" "$xx ; fi + else count=1 ; + echo ""; echo -n " -- skip \"$xx\" (not a directory !)" + fi + else + if test $count = 1 ; then echo -n ", \"$xx\"" + else count=1 ; echo "" ; echo -n " skip: \"$xx\"" + fi + fi +done +if test $count = 1 ; then echo "" ; echo -n " ... " ; fi +#echo 'TESTDIRS='${TESTDIRS}'<' + +if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then + OPTFILE=$MITGCM_OF fi -echo "OK" +if test "x$ADM" = xt ; then + EXECUTABLE="mitgcmuv_ad" + OUTPUTFILE="output_adm.txt" +else + EXECUTABLE="mitgcmuv" + OUTPUTFILE="output.txt" +fi + +if test "x$COMMAND" = x ; then + COMMAND="./$EXECUTABLE > $OUTPUTFILE" +fi +if test "x$MPI" = xt ; then + OUTPUTFILE="STDOUT.0000" +fi + +#echo "OK" +echo "OK (COMMAND= $COMMAND )" + +# set the Default List of output variables to be checked: +# (use default or load experiment-specific list from file "tr_checklist") +# content : 1rst = main variable used to decide if it pass or FAIL +# others = number of matching digits to be printed in summary.txt +if test "x$ADM" != xt ; then + DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+' + EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..' + nbDef=`echo $DEF_CHECK_LIST | awk '{print NF-1+3*gsub("+","E")}'` + ii=`echo $EMPTY_RESULTS | awk '{print NF}'` + EMPTY_RESULTS=$EMPTY_RESULTS`expr $nbDef - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'` +fi # create the FORTRAN comparison code createcodelet # build the mpack utility -build_mpack +if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then + echo "skipping mpack build" +else + build_mpack +fi # Create a uniquely named directory to store results MACH=`hostname` +UNAMEA=`uname -a` DATE=`date +%Y%m%d` -BASE=$MACH"_"$DATE"_" +BASE="tr_"$MACH"_"$DATE"_" +if test "x$OUTDIR" != x ; then + BASE="tr_"$OUTDIR"_"$DATE"_" +fi DNUM=0 DRESULTS="$BASE$DNUM" while test -e $DRESULTS ; do @@ -474,91 +1000,306 @@ mkdir $DRESULTS RETVAL=$? if test "x$RETVAL" != x0 ; then - echo "Error: can't create results directory \"./$DRESULTS\"" + echo "ERROR: Can't create results directory \"./$DRESULTS\"" exit 1 fi SUMMARY="$DRESULTS/summary.txt" -date > $SUMMARY -cat << EOF >> $SUMMARY - T S U V -G D M c m s m s m s m s -E p a R g m m e . m m e . m m e . m m e . -N n k u 2 i a a d i a a d i a a d i a a d -2 d e n d n x n . n x n . n x n . n x n . +printf "Start time: " >> $SUMMARY +start_date=`date` +echo $start_date > $SUMMARY + +of_path= +if test "x$OPTFILE" != xNONE ; then + if test -r $OPTFILE ; then + # get the path + path=${OPTFILE%/*} + if test "x$path" = x ; then + of_path=`pwd` + else + of_path=`( cd $path > /dev/null 2>&1 ; pwd )` + fi + file=${OPTFILE##*/} + OPTFILE=$of_path/$file + cp $OPTFILE $DRESULTS + echo >> $SUMMARY + echo " OPTFILE=$OPTFILE" >> $SUMMARY + else + echo | tee $SUMMARY + echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY + exit 1 + fi +else + echo >> $SUMMARY + echo "No \"OPTFILE\" was explicitly specified by testreport," >> $SUMMARY + echo " so the genmake default will be used." >> $SUMMARY +fi +echo +echo >> $SUMMARY +if test "x$ADM" = x ; then + if [ $MATCH_CRIT -lt 10 ] ; + then line_0="default "$MATCH_CRIT ; + else line_0="default "$MATCH_CRIT ; fi + line_0="$line_0 ----T----- ----S----- ----U----- ----V-----" +# line_0=" ----T----- ----S----- ----U----- ----V-----" + line_1="G D M c m s m s m s m s" + line_2="E p a R g m m e . m m e . m m e . m m e ." + line_3="N n k u 2 i a a d i a a d i a a d i a a d" + line_4="2 d e n d n x n . n x n . n x n . n x n ." + for ii in $PTRACERS_NUM ; do + # tst=`eval 'echo $HAVE_PTR0'$ii` + # if test "x$tst" = xt ; then + line_0="$line_0 --PTR 0"$ii"--" + line_1="$line_1 m s" + line_2="$line_2 m m e ." + line_3="$line_3 i a a d" + line_4="$line_4 n x n ." + # fi + done + echo "$line_0" | tee -a $SUMMARY + echo "$line_1" | tee -a $SUMMARY + echo "$line_2" | tee -a $SUMMARY + echo "$line_3" | tee -a $SUMMARY + echo "$line_4" | tee -a $SUMMARY + echo " " | tee -a $SUMMARY +else + echo "ADJOINT=true" >> $SUMMARY + echo >> $SUMMARY + cat << EOF | tee -a $SUMMARY +G D M C G +E p a R o r +N n k u s a +2 d e n t d EOF +fi -NDIR=0 +# ...and each test directory... +for dir in $TESTDIRS ; do + + # Cleanup only! + if test "x$CLEANUP" = xt ; then + if test -r $dir/build/Makefile ; then + echo ' ------ clean dir:' $dir/build + ( cd $dir/build ; make CLEAN ) + fi + if test -d $dir/run/CVS ; then + echo ' ------ clean dir:' $dir/run + run_clean $dir/run + fi + ( + cd $dir + rm -rf tr_run.* + ) + continue + fi + + # Verify that the testdir exists and contains previous + # results in the correct location--or skip this directory! + fout= + if test "x$ADM" = x ; then + fout=$dir"/results/output.txt" + else + fout=$dir"/results_ad/output_adm.txt" + fi + if test ! -r $fout ; then + echo "can't read \"$fout\" -- skipping $dir" + continue + fi + if test "x$ADM" = x ; then + check_for_add_mon_output $fout + fi -# For each optfile... -for OPTFILE in $OPTFILES ; do + # Check for additional types of monitor output - OPTFILE=`pwd`"/$OPTFILE" - if test ! -r $OPTFILE ; then - echo "Error: can't read optfile \"$OPTFILE\"" - exit 1 + builddir="build" + if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi + rundir="run" + #rundir=$builddir + if test ! -d $dir/$rundir ; then + rundir=$builddir fi - echo - echo "OPTFILE=$OPTFILE" >> $SUMMARY - echo >> $SUMMARY - # ...and each test directory... - for dir in $TESTDIRS ; do - - # Create an output dir for each OPTFILE/tdir combination - CDIR=$DRESULTS"/"$NDIR - mkdir $CDIR - CDIR=`pwd`"/$CDIR" + if test "x$ADM" = x ; then + code_dir=code + CODE_DIR=$dir/code + input_dirs='input' + else + code_dir=code_ad + CODE_DIR=$dir/code_ad + input_dirs='input_ad input' + fi + BUILD_DIR=$dir/$builddir - # ...configue, make, run, and compare the output. - echo "-------------------------------------------------------------------------------" - echo - echo "Experiment: $dir" - echo - unset genmake makedepend make run - results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --' - ( cd $dir/input; rm -f *.{o,f,c,F} *.[f,F]90 work* output.txt Make* make.log; ) - if [ -r $dir/build ]; then - seperatebuilddir=1 - builddir=build - rundir=build - ( cd $dir/build; ln -sf ../input/* . ) - else - seperatebuilddir=0 - builddir=input - rundir=input - fi + if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then + echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" + continue + fi + if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then + echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir" + continue + fi + + # Check whether there are "extra runs" for this testdir + extra_runs= + if test "x$ADM" = x ; then + ex_run_dirs=`( cd $dir ; echo input.* )` + #echo "ex_run_dirs='$ex_run_dirs'" + for exd in $ex_run_dirs ; do + name=`echo $exd | sed -e 's/input.//g'` + outf="$dir/results/output.$name.txt" + if test -f $outf -a -r $outf ; then + if test "x$MULTI_THREAD" = "xt" ; then + if test -r $dir"/"$exd"/eedata.mth" ; then + extra_runs="$extra_runs $name" + #else echo $dir"/"$exd"/eedata.mth: not found" + fi + else + extra_runs="$extra_runs $name" + fi + fi + done + fi + + echo "-------------------------------------------------------------------------------" + echo + if test "x$extra_runs" = "x" ; then + echo "Experiment: $dir" + else + echo "Experiment: $dir ; extra_runs=$extra_runs" + fi + echo + unset genmake makedepend make run + results=$EMPTY_RESULTS + + # Create an output dir for each OPTFILE/tdir combination + rel_CDIR=$DRESULTS"/"$dir + mkdir $rel_CDIR + CDIR=`pwd`"/$rel_CDIR" + + if test "x$CLEANUP" = xt ; then + echo '====>>> this is to check that we never go through this part <<< ===' + makeclean $dir/$builddir \ + && run_clean $dir/$rundir + else genmakemodel $dir/$builddir && genmake=Y \ && makeclean $dir/$builddir \ + && run_clean $dir/$rundir \ + && symlink_mpifiles $dir $code_dir $builddir \ && makedependmodel $dir/$builddir && makedepend=Y \ && makemodel $dir/$builddir && make=Y \ - && linkdata $seperatebuilddir $dir/$rundir \ - && runmodel $dir/$builddir mitgcmuv && run=Y \ - && results=`testoutput $dir $rundir` + && linkdata $dir/$rundir $input_dirs \ + && runmodel $dir/$rundir && run=Y \ + && results=`testoutput_run $dir $rundir "txt"` + fi + + echo + if test "x$ADM" = x ; then + fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo - formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results + echo "$fres" >> $SUMMARY + touch $CDIR"/summary.txt" + echo "fresults='$fres'" >> $CDIR"/summary.txt" + echo "MACH='$MACH'" >> $CDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" + echo "DATE='$DATE'" >> $CDIR"/summary.txt" + echo "tdir='$dir'" >> $CDIR"/summary.txt" + + for ex in $extra_runs ; do + unset run + results=$EMPTY_RESULTS + # Create an output dir for each OPTFILE/tdir.ex combination + rel_CDIR=$DRESULTS"/"$dir"."$ex + mkdir $rel_CDIR + CDIR=`pwd`"/$rel_CDIR" + test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex" + run_clean $dir/tr_run.$ex + linkdata $dir/tr_run.$ex input.$ex input + runmodel $dir/tr_run.$ex && run=Y \ + && results=`testoutput_run $dir tr_run.$ex ${ex}".txt"` + fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}` + fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` + fres="$fres.$ex" + echo + echo "$fres" >> $SUMMARY + touch $CDIR"/summary.txt" + echo "fresults='$fres'" >> $CDIR"/summary.txt" + echo "MACH='$MACH'" >> $CDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" + echo "DATE='$DATE'" >> $CDIR"/summary.txt" + echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt" + if test "x$POSTCLEAN" = xt ; then + run_clean $dir/tr_run.$ex + fi + done + else + fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}` + fres=$fres"$results $dir" echo - formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results >> $SUMMARY + echo "$fres" >> $SUMMARY + touch $CDIR"/summary.txt" + echo "fresults='$fres'" >> $CDIR"/summary.txt" + echo "MACH='$MACH'" >> $CDIR"/summary.txt" + echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" + echo "DATE='$DATE'" >> $CDIR"/summary.txt" + echo "tdir='$dir'" >> $CDIR"/summary.txt" + grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \ + >> $CDIR"/summary.txt" + fi - ( cd $DRESULTS"/"$NDIR ; gzip * ) + #postclean $dir/$builddir + if test "x$POSTCLEAN" = xt ; then + makeclean $dir/$builddir \ + && run_clean $dir/$rundir + fi + + echo "-------------------------------------------------------------------------------" + +done - tar -cf $DRESULTS"/"$NDIR".tar" $DRESULTS"/"$NDIR - $MPACK -s MITgcm-test -m 1000000 $DRESULTS"/"$NDIR".tar" $ADDRESSES +printf "Start time: " >> $SUMMARY +echo $start_date >> $SUMMARY +printf "End time: " >> $SUMMARY +date >> $SUMMARY - NDIR=$(( $NDIR + 1 )) - - done -done +# If addresses were supplied and mpack built successfully, then try +# to send email using mpack. +if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then + echo "No results email was sent." +else + if test "x$HAVE_MPACK" = xt ; then + tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \ + && gzip $DRESULTS".tar" \ + && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES + RETVAL=$? + if test "x$RETVAL" != x0 ; then + echo + echo "Warning: The tar, gzip, & mpack step failed. Please send email" + echo " to for help. You may copy the " + echo " summary of results from the directory \"$DRESULTS\"." + echo + else + echo + echo "An email containing results was sent to the following addresses:" + echo " \"$ADDRESSES\"" + echo + fi + test -f $DRESULTS".tar" && rm -f $DRESULTS".tar" + test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz" + fi +fi -# rm tmp_cmpnum.f a.out +rm -f tr_cmpnum.c tr_cmpnum -echo ------------------------------------------------------------------------------- -echo -echo >> $SUMMARY -date >> $SUMMARY -cat $SUMMARY +if test "x$CLEANUP" != xt ; then + cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \.//' +# cat $SUMMARY | sed 's/ \.//g' + if test -e tr_out.txt ; then + mv tr_out.txt tr_out.txt.old + fi + cat $SUMMARY | sed 's/ \.//g' > tr_out.txt +fi -# tar -cf $DRESULTS".tar" $DRESULTS -# -# $MPACK -s MITgcm-test -m 1000000 $DRESULTS".tar" $ADDRESSES +if test "x$DELDIR" = xt ; then + rm -rf $DRESULTS +fi