--- MITgcm/verification/testreport 2012/08/15 23:19:21 1.175 +++ MITgcm/verification/testreport 2016/01/15 22:14:22 1.207 @@ -1,6 +1,6 @@ #! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.175 2012/08/15 23:19:21 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.207 2016/01/15 22:14:22 jmc Exp $ # $Name: $ # @@ -11,20 +11,23 @@ echo echo "where possible OPTIONS are:" echo " (-help|-h) print usage" + echo " ---- type of test : ----" + echo " (-tlm) perform a Tangent Linear run" + echo " (-adm|-ad) perform an adjoint run" + echo " (-oad) perform an OpenAD adjoint run" echo " (-mth) run multi-threaded (using eedata.mth)" echo " (-mpi) use MPI to compile and run on 2 processors" echo " (-MPI) NUMBER use MPI to compile and run on max NUMBER procs" echo " (-mfile|-mf) STRING MPI: file with list of possible machines to run on" + echo " (-command|-c) STRING command to run (e.g., if non-standard MPI setting)" + echo " DEF='mitgcmuv' or ='mpirun -np TR_NPROC mitgcmuv'" + echo " ---- testing options : ----" + echo " (-optfile|-of) STRING optfile to use" echo " (-fast) use optfile default for compiler flags (no '-ieee')" echo " DEF=off => use IEEE numerics option (if available)" echo " (-devel) use optfile developement flags (if available)" echo " (-gsl) compile with \"-gsl\" flag" echo " (-use_r4|-ur4) if allowed, use real*4 type for '_RS' variable" - echo " (-optfile|-of) STRING list of optfiles to use" - echo " (-addr|-a) STRING list of email recipients" - echo " (DEF=\"\" no email is sent)" - echo " (-mpackdir|-mpd) DIR location of the mpack utility" - echo " (DEF=\"../tools/mpack-1.6\")" echo " (-tdir|-t) STRING list of group and/or exp. dirs to test" echo " (recognized groups: basic, tutorials)" echo " (DEF=\"\" which test all)" @@ -32,24 +35,31 @@ echo " test THIS_EXP + all the following)" echo " (-skipdir|-skd) STRING list of exp. dirs to skip" echo " (DEF=\"\" which test all)" - echo " (-bash|-b) STRING preferred location of a \"bash\" or" - echo " Bourne-compatible \"sh\" shell" + echo " (-ts) provide timing information per timestep" + echo " (-papis) provide MFlop/s per timestep using PAPI" + echo " (-pcls) provide MFlop/s per timestep using PCL" + echo " ---- system options : ----" + echo " (-bash|-b) STRING preferred location of a \"bash\" or \"sh\" shell" echo " (DEF=\"\" for \"bash\")" - echo " (-tlm) perform a Tangent Linear run" - echo " (-adm|-ad) perform an adjoint run" - echo " (-oad) perform an OpenAD adjoint run" - echo " (-command|-c) STRING command to run" + echo " (-ef) STRING used as genmake2 \"-extra_flag\" argument" + echo " (-ncad) use genmake2 option \"-nocat4ad\" (-ncad)" + echo " (-small_f) make target small_f before making target all" echo " (-makedepend|-md) STRING command to use for \"makedepend\"" echo " (-make|-m) STRING command to use for \"make\"" echo " (DEF=\"make\")" - echo " (-odir) STRING used to build output directory name" - echo " (DEF=\"hostname\")" -# echo " (-ptracers|-ptr) STRING specify which ptracers to test" -# echo " (DEF=\"1 2 3 4 5\")" + echo " (-j) JOBS use \"make -j JOBS\" for parallel builds" + echo " ---- output options : ----" echo " (-match) NUMBER Matching Criteria (number of digits)" echo " (DEF=\"$MATCH_CRIT\")" - echo " (-j) JOBS use \"make -j JOBS\" for parallel builds" - echo " (-ef) STRING used as genmake2 \"-extra_flag\" argument" + echo " (-odir) STRING used to build output directory name" + echo " (DEF=\"hostname\")" + echo " (-addr|-a) STRING list of email recipients" + echo " (DEF=\"\" no email is sent)" + echo " (-send) STRING sending command (instead of using mpack)" + echo " (-savdir|-sd) STRING location to save output tar file to send (DEF='$SAVDIR')" + echo " (-mpackdir|-mpd) DIR location of the mpack utility" + echo " (DEF=\"../tools/mpack-1.6\")" + echo " -- do only some parts: --" echo " (-clean) *ONLY* run \"make CLEAN\" & clean run-dir" echo " (-norun|-nr) skip the \"runmodel\" stage (stop after make)" echo " (-obj) only produces objects (=norun & no executable)" @@ -61,9 +71,6 @@ echo " (-postclean|-pc) after each exp. test, clean build-dir & run-dir" echo " (-deloutp|-do) delete output files after successful run" echo " (-deldir|-dd) on success, delete the output directory" - echo " (-ts) provide timing information per timestep" - echo " (-papis) provide MFlop/s per timestep using PAPI" - echo " (-pcls) provide MFlop/s per timestep using PCL" echo echo "and where STRING can be a whitespace-delimited list" echo "such as:" @@ -125,13 +132,13 @@ echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2 fi if [ -r $1/$4/$OUTPUTFILE ]; then - grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt - lncntA=`wc -l tmp1.txt | awk '{print $1}' ` + grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | nl > ${TMP}a.txt + lncntA=`wc -l ${TMP}a.txt | awk '{print $1}' ` if [ $lncntA -lt 2 ]; then if [ $verbose -gt 0 ]; then echo Not enough lines of output when searching for "$2" 1>&2 fi - rm -f tmp1.txt ; return 99 + rm -f ${TMP}a.txt ; return 99 fi else echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2 @@ -140,44 +147,43 @@ if [ $debug -gt 0 ]; then echo testoutput_var: grep "$2" $1/$5 1>&2 fi - grep "$2" $1/$5 | sed 's/.*=//' | cat -n > tmp2.txt - lncntB=`wc -l tmp2.txt | awk '{print $1}' ` + grep "$2" $1/$5 | sed 's/.*=//' | nl > ${TMP}b.txt + lncntB=`wc -l ${TMP}b.txt | awk '{print $1}' ` if [ $lncntB -lt 2 ]; then if [ $verbose -gt 0 ]; then echo Not enough lines of output when searching for "$2" 1>&2 fi - rm -f tmp1.txt tmp2.txt ; return 99 + rm -f ${TMP}a.txt ${TMP}b.txt ; return 99 fi if [ $lncntA -ne $lncntB ]; then if [ $verbose -gt 0 ]; then echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2 fi - rm -f tmp1.txt tmp2.txt ; return 99 + rm -f ${TMP}a.txt ${TMP}b.txt ; return 99 fi - has_nan=`cat tmp1.txt | grep -i nan | wc -l` + has_nan=`cat ${TMP}a.txt | grep -i nan | wc -l` if [ $has_nan -gt 0 ] ; then echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2 - rm -f tmp1.txt tmp2.txt ; return 99 + rm -f ${TMP}a.txt ${TMP}b.txt ; return 99 fi - has_inf=`cat tmp1.txt | grep -i inf | wc -l` + has_inf=`cat ${TMP}a.txt | grep -i inf | wc -l` if [ $has_inf -gt 0 ] ; then echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2 - rm -f tmp1.txt tmp2.txt ; return 99 + rm -f ${TMP}a.txt ${TMP}b.txt ; return 99 fi if [ $debug -gt 0 ]; then - echo testoutput_var: join tmp1.txt tmp2.txt 1>&2 + echo testoutput_var: join ${TMP}a.txt ${TMP}b.txt 1>&2 fi - join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt + # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number + join ${TMP}a.txt ${TMP}b.txt | awk '{print $1 " " $2 " " $3}' | sed -e 's|:||g' > ${TMP}c.txt if [ $debug -gt 0 ]; then echo testoutput_var: compare_lines 1>&2 fi if [ $verbose -gt 1 ]; then - cat tmp3.txt 1>&2 + cat ${TMP}c.txt 1>&2 fi - echo "-1" >> tmp3.txt - # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number - cat tmp3.txt | sed -e 's|:||g' > tmp4.txt - digits_of_similarity=`./tr_cmpnum < tmp4.txt` + echo "-1" >> ${TMP}c.txt + digits_of_similarity=`./tr_cmpnum < ${TMP}c.txt` if [ $digits_of_similarity -eq 99 ]; then if [ $verbose -gt 0 ]; then echo testoutput_var: No comparison was available for \"$3\" 1>&2 @@ -188,7 +194,7 @@ echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2 fi fi - rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt + rm -f ${TMP}a.txt ${TMP}b.txt ${TMP}c.txt return $digits_of_similarity } @@ -322,9 +328,16 @@ 'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $refoutp ; yy=$? ;; 'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $refoutp ; yy=$? ;; 'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $refoutp ; yy=$? ;; - 'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $refoutp ; yy=$? ;; - 'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $refoutp ; yy=$? ;; - 'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $refoutp ; yy=$? ;; + 'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Glob-ave" $2 $refoutp ; yy=$? ;; + 'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South-av" $2 $refoutp ; yy=$? ;; + 'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North-av" $2 $refoutp ; yy=$? ;; + 'HthMxS') testoutput_var $1 "thSI_IceH_max_S" "thSIc H South-max" $2 $refoutp ; yy=$? ;; + 'HthMxN') testoutput_var $1 "thSI_IceH_max_N" "thSIc H North-max" $2 $refoutp ; yy=$? ;; + 'sbo_M') testoutput_var $1 "sbo_mass" "SBO mass" $2 $refoutp ; yy=$? ;; + 'sboFW') testoutput_var $1 "sbo_mass_fw" "SBO m-FW" $2 $refoutp ; yy=$? ;; + 'sboAc') testoutput_var $1 "sbo_zoamc" "SBO AM-C" $2 $refoutp ; yy=$? ;; + 'sboAp') testoutput_var $1 "sbo_zoamp" "SBO AM-P" $2 $refoutp ; yy=$? ;; + 'StrmIc') testoutput_var $1 "STREAMICE_FP_ERR" "StreamIce Solver" $2 $refoutp ; yy=$? ;; *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;; esac if test $xx = $sVar @@ -361,12 +374,10 @@ if test "x$MKDEPEND" != x ; then command="$command -makedepend=$MKDEPEND" fi - if test $KIND = 1 -o $KIND = 2 ; then - command="$command --mods=../code_ad" - elif test $KIND = 4 ; then - command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods '../code_oad ../../OpenAD/code_oad_all'" + if test $KIND = 4 ; then + command="$command -oad -mods=../$code_dir" else - command="$command -mods=../code" + command="$command -mods=../$code_dir" fi if test "x$OPTFILE" != xNONE ; then command="$command -optfile=$OPTFILE" @@ -393,6 +404,9 @@ if test "x$EXTRFLG" != x ; then command="$command -extra_flag $EXTRFLG" fi + if test "x$NOCATAD" = xt ; then + command="$command -nocat4ad" + fi if test "x$TS" = xt ; then command="$command -ts" fi @@ -423,7 +437,7 @@ makeclean() { # makeclean directory - if test "x$NODEPEND" = xf ; then rm -f $1/make.tr_log ; fi + if test "x$NOGENMAKE" = xf ; then rm -f $1/make.tr_log ; fi if test "x$NOCLEAN" = xt ; then echo "make Clean skipped!" else @@ -455,21 +469,23 @@ if test "x$NOCLEAN" = xt ; then echo "run_clean skipped!" else - ( - cd $1; - printf 'clean run-dir ... ' - # part of what is done after "make clean" when doing "make CLEAN" - find . -name "*.meta" -exec rm {} \; - find . -name "*.data" -exec rm {} \; - find . -name "fort.*" -exec rm {} \; - find . -type l -exec rm {} \; - #- should remove executable only if sym-link (alredy done above) - rm -f $RUNLOG *.txt STD* w2_tile_topology.????.log *diagnostics.log datetime - rm -rf mnc_test_* - rm -f *_MIT_CE_000.opt0000 costfunction*0000 - echo successful - exit 0 - ) + ( + cd $1; + printf 'clean run-dir ... ' + # part of what is done after "make clean" when doing "make CLEAN" + find . -name "*.meta" -exec rm {} \; + find . -name "*.data" -exec rm {} \; + find . -name "fort.*" -exec rm {} \; + find . -type l -exec rm {} \; + #- should remove executable only if sym-link (already done above) + rm -f $RUNLOG *.txt STDOUT.* STDERR.* *diagnostics.log *.[0-9][0-9][0-9][0-9].log + rm -f datetime costfinal divided.ctrl snapshot* output_adm.*.diva_* + rm -f *_MIT_CE_000.opt0000 costfunction*0000 + rm -f oad_cp.[0-9][0-9][0-9].????? + rm -rf mnc_test_* + echo successful + exit 0 + ) fi } @@ -512,12 +528,23 @@ else cd $1; if test -r Makefile ; then - printf 'make ... ' - if test "x$REPLMAKE" = x ; then - $MAKE $TARG >> make.tr_log 2>&1 - else - $REPLMAKE $TARG >> make.tr_log 2>&1 + if test "x$MKSMALLF" = xt ; then + printf 'make small_f ... ' + $MAKE_CMD small_f >> make.tr_log 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + tail make.tr_log + echo failed + cp genmake.log genmake.tr_log $CDIR + tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail" + rm -f $EXECUTABLE + mk_fail=1 + else + echo successful + fi fi + printf 'make ... ' + $MAKE_CMD $TARG >> make.tr_log 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.tr_log @@ -534,11 +561,19 @@ mk_fail=2 fi fi + if test $KIND = 1 -a -f taf_ftl.log ; then + head -1 taf_ftl.log >> $CDIR"/summary.txt" + fi if test $KIND = 2 -a -f taf_ad.log ; then - head -1 taf_ad.log >> $CDIR"/summary.txt" - nerr=`grep -c 'TAF *.* ERROR ' taf_ad.log` - nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' taf_ad.log` - echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \ + head -1 taf_ad.log >> $CDIR"/summary.txt" + nerr=`grep -c 'TAF *.* ERROR ' taf_ad.log` + nwar=`grep -c 'TAF RECOMPUTATION *.* WARNING ' taf_ad.log` + if test -f taf_output ; then + n2er=`grep -c 'TAF *.* ERROR ' taf_output` + n3er=`grep -c '\*ERROR\* ' taf_output` + nerr=`expr $nerr + $n2er + $n3er` + fi + echo " TAF reports $nerr Errors and $nwar Recomputation Warnings" \ >> $CDIR"/summary.txt" fi if test $mk_fail != 0 ; then return $mk_fail ; fi @@ -757,7 +792,7 @@ if test "x$MPI" != x0 ; then #- adjust the MPI run command with the right number of Procs #echo '' ; echo " COMMAND='$COMMAND'" - COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"` + COMMAND=`echo $COMMAND | sed "s/ TR_NPROC/ $LOC_NPROC/"` if test "x$MPI_MFILE" != x ; then COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"` fi @@ -785,6 +820,23 @@ # output do not exist or is older than executable: rm -f $OUTPUTFILE $RUNLOG ; touch $RUNLOG if test -f run.log_tmp ; then cat run.log_tmp >> $RUNLOG ; fi + #- Divided Adjoint Run: + # get the number of additional runs (add_DIVA_runs) from file "run_ADM_DIVA" + if test $KIND = 2 -a -f run_ADM_DIVA ; then + adm_diva_nb=`sed -n '/^ *add_DIVA_runs\>.*=/p' run_ADM_DIVA | sed 's/ //g'` + echo " Divided Adjoint Run: $adm_diva_nb" >> $RUNLOG + eval "let $adm_diva_nb" + if [ $add_DIVA_runs -ge 1 ] ; then + rm -f costf* divided.ctrl snapshot* + echo -n "(add_DIVA_runs=$add_DIVA_runs) ... " + for ii in `seq 1 $add_DIVA_runs` ; do + ( eval $COMMAND ) >> $RUNLOG 2>&1 + echo " additional DIVA run # $ii : done" >> $RUNLOG + mv -f $OUTPUTFILE ${OUTPUTFILE}.diva_${ii} + done + fi + fi + #- special DIVA processing ends here ( eval $COMMAND ) >> $RUNLOG 2>&1 RETVAL=$? ENDVAL=`tail $OUTPUTFILE | grep -c 'PROGRAM MAIN: Execution ended Normally'` @@ -802,13 +854,19 @@ fi fi rm -f run.log_tmp - #- in all cases where OutputFile exists, report SIZE (and AD time) + #- in all cases where OutputFile exists, report SIZE and time if test -f $OUTPUTFILE ; then grep '(PID\.TID 0000\.0001) n.. =' $OUTPUTFILE \ - | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" - if test $KIND = 2 ; then - grep -A3 'Seconds in section "ALL' $OUTPUTFILE >> $CDIR"/summary.txt" - fi + | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" +# grep -A3 'Seconds in section "ALL' $OUTPUTFILE \ +# | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" +# some implementations of grep cannot do contextual searches so we +# replace the above with a sed command + cat $OUTPUTFILE | sed -n '/Seconds in section "ALL/{N + N + N + p + }' | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" fi if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then @@ -910,6 +968,13 @@ fi fi printf ' %s' $nm + if test $KIND = 2 ; then + #-- apppend taf repport summary: + tafrep=`grep -c '^ TAF reports ' $CDIR/summary.txt` + if test $tafrep = 1 ; then + grep '^ TAF reports ' $CDIR/summary.txt | awk '{printf " (e=%i, w=%i)",$3,$6}' + fi + fi printf '\n' } @@ -995,12 +1060,14 @@ MPACKDIR="../tools/mpack-1.6" HAVE_MPACK= MPACK= +SENDCMD= +SAVDIR='.' COMMAND= MKDEPEND= if test "x$MAKE" = x ; then MAKE=make fi -REPLMAKE= +MAKE_CMD= if test "x$CC" = x ; then CC=cc fi @@ -1013,6 +1080,8 @@ DELDIR= USE_R4= EXTRFLG= +NOCATAD= +MKSMALLF= #- type of testing (KIND): # KIND=0 : forward (= default) ; KIND=1 : Tangent Linear with TAF ; @@ -1050,6 +1119,11 @@ -mpackdir | --mpackdir | -mpd | --mpd) ac_prev=MPACKDIR ;; -mpackdir=* | --mpackdir=* | -mpd=* | --mpd=*) MPACKDIR=$ac_optarg ;; + -send | --send ) ac_prev=SENDCMD ;; + -send=* | --send=* ) SENDCMD=$ac_optarg ;; + -savdir | --savdir | -sd | --sd ) ac_prev=SAVDIR ;; + -savdir=* | --savdir=* | -sd=* | --sd=* ) SAVDIR=$ac_optarg ;; + -tdir | --tdir | -t | --t) ac_prev=TESTDIRS ;; -tdir=* | --tdir=* | -t=* | --t=*) TESTDIRS=$ac_optarg ;; -skipdir | --skipdir | -skd | --skd) ac_prev=SKIPDIRS ;; @@ -1067,8 +1141,8 @@ -make | --make | -m | --m) ac_prev=MAKE ;; -make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;; - -repl_mk | --repl_mk ) ac_prev=REPLMAKE ;; - -repl_mk=* | --repl_mk=*) REPLMAKE=$ac_optarg ;; + -repl_mk | --repl_mk ) ac_prev=MAKE_CMD ;; + -repl_mk=* | --repl_mk=*) MAKE_CMD=$ac_optarg ;; -odir | --odir) ac_prev=OUTDIR ;; -odir=* | --odir=*) OUTDIR=$ac_optarg ;; @@ -1116,6 +1190,8 @@ -oad) if test $KIND = 0 ; then KIND=4 ; NODEPEND=t ; else echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage fi ;; + -ncad) NOCATAD=t ;; + -small_f) MKSMALLF=t ;; -ieee) echo "Warning: ignore option '-ieee' (already the default)" printf " ... " ;; @@ -1135,7 +1211,7 @@ -ts) TS=t;; -papis) PAPIS=t;; - -pcls) PCL=t;; + -pcls) PCLS=t;; -*) echo "Error: unrecognized option: "$ac_option usage ;; @@ -1151,6 +1227,7 @@ NOCLEAN=t NODEPEND=t fi +if test "x$MAKE_CMD" = x ; then MAKE_CMD=$MAKE ; fi #- check length of MPI machine file: if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then @@ -1189,12 +1266,12 @@ ref_outp="output_oadm.txt" EXECUTABLE="mitgcmuv_ad" else - if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi code_dir=code inputdir=input ref_outp="output.txt" EXECUTABLE="mitgcmuv" fi +if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi xx=`echo $TESTDIRS | awk '{print $1}'` if test "x$TESTDIRS" = x ; then @@ -1241,21 +1318,29 @@ if test $count = 1 ; then echo "" ; echo -n " ... " ; fi #echo 'TESTDIRS='${TESTDIRS}'<' -if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then - OPTFILE=$MITGCM_OF -fi - LOC_MFILE='tr_mpi_mfile' RUNLOG="run.tr_log" -OUTPUTFILE=$ref_outp -if test "x$COMMAND" = x ; then - COMMAND="./$EXECUTABLE > $OUTPUTFILE" -fi -if test "x$MPI" != x0 ; then - OUTPUTFILE="STDOUT.0000" +if test "x$MPI" = x0 ; then + OUTPUTFILE=$ref_outp + if test "x$COMMAND" = x ; then COMMAND="./$EXECUTABLE > $OUTPUTFILE" ; fi +else + OUTPUTFILE="STDOUT.0000" + if test "x$COMMAND" = x ; then COMMAND="mpirun -np TR_NPROC ./$EXECUTABLE" ; fi fi -echo "OK (COMMAND= $COMMAND )" +echo "OK (COMMAND='$COMMAND')" + +#TMP=./tr_$$ +#- try to put temporary files in system-local /tmp dir +TMP=/tmp/tr_${USER}_$$ +touch $TMP ; retVal=$? +if [ $retVal -eq 0 ] ; then + if test ! -r $TMP ; then TMP=./tr_$$ ; fi +else + TMP=./tr_$$ +fi +rm -f $TMP +if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi # set the Default List of output variables to be checked: # (use default or load experiment-specific list from file "tr_checklist") @@ -1278,26 +1363,30 @@ fi # create the FORTRAN comparison code -if test -x tr_cmpnum ; then +if test "x$CLEANUP" = xt -o -x tr_cmpnum ; then echo "skipping comparison code build" else createcodelet fi # build the mpack utility (if ADDRESSES = NONE, do it to test the build) -if test "x$ADDRESSES" = x ; then +if test "x$ADDRESSES" = x -o "x$SENDCMD" != x ; then echo "skipping mpack build" else build_mpack + if test "x$HAVE_MPACK" = xt ; then SENDCMD=$MPACK ; fi fi # Create a uniquely named directory to store results CMDLINE=$0 for xx in "$@" ; do nw=`echo $xx | wc -w` - if test $nw = '1' ; then CMDLINE="$CMDLINE $xx" - else CMDLINE="$CMDLINE '$xx'" ; fi + if test $nw = '1' ; then CMDLINE="$CMDLINE $xx" ; else + nb=`echo $xx | grep -c '='` + if test $nb = 0 ; then CMDLINE="$CMDLINE '$xx'" + else yy=`echo "$xx'" | sed "s/=/='/"` ; + CMDLINE="$CMDLINE $yy" ; fi + fi done -#for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done MACH=`hostname` UNAMEA=`uname -a` DATE=`date +%Y%m%d` @@ -1305,7 +1394,9 @@ if test "x$OUTDIR" != x ; then BASE="tr_"$OUTDIR"_"$DATE"_" else - short_name=`hostname | sed 's/\..*$//'` + #short_name=`hostname -s | tr '[:upper:]' '[:lower:]'` + # hostname -s is not universal (does work on AIX system) + short_name=`hostname | sed 's/\..*$//' | tr '[:upper:]' '[:lower:]'` BASE="tr_"$short_name"_"$DATE"_" fi DNUM=0 @@ -1342,8 +1433,8 @@ echo >> $SUMMARY echo " OPTFILE=$OPTFILE" >> $SUMMARY else - echo | tee $SUMMARY - echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee $SUMMARY + echo | tee -a $SUMMARY + echo "ERROR: can't read OPTFILE=\"$OPTFILE\"" | tee -a $SUMMARY exit 1 fi else @@ -1370,11 +1461,20 @@ else line_0=`printf '%s %2i' 'default ' $MATCH_CRIT` if test $KIND = 1 ; then - echo "TANGLIN=true" >> $SUMMARY + #echo "TANGLIN=true" >> $SUMMARY + echo "TangLin generated by TAF" >> $SUMMARY + elif test $KIND = 3 ; then + echo "TangLin generated by OpenAD" >> $SUMMARY + elif test $KIND = 2 ; then + #echo "ADJOINT=true" >> $SUMMARY + echo "Adjoint generated by TAF" >> $SUMMARY + else + echo "Adjoint generated by OpenAD" >> $SUMMARY + fi + if test $KIND = 1 -o $KIND = 3 ; then line_1="G D M C T F" line_2="e p a R o L D" else - echo "ADJOINT=true" >> $SUMMARY line_1="G D M C A F" line_2="e p a R o d D" fi @@ -1382,12 +1482,14 @@ line_4="2 d e n t r r" echo >> $SUMMARY fi +if test "x$CLEANUP" != xt ; then echo "$line_0" | tee -a $SUMMARY echo "$line_1" | tee -a $SUMMARY echo "$line_2" | tee -a $SUMMARY echo "$line_3" | tee -a $SUMMARY echo "$line_4" | tee -a $SUMMARY echo "" | tee -a $SUMMARY +fi echo "-------------------------------------------------------------------------------" # ...and each test directory... @@ -1412,6 +1514,7 @@ rm -f $EXECUTABLE *.bak rm -f genmake_state genmake_*optfile genmake.log rm -f SIZE.h.mpi genmake.tr_log make.tr_log + rm -rf mpi_headers ) if test -d $dir/$rundir/CVS ; then echo -n ' --- dir:' $dir/$rundir ': ' @@ -1496,14 +1599,13 @@ # Check whether there are "extra runs" for this testdir extra_runs= if test "x$NORUN" = xf ; then - ex_run_dirs=`( cd $dir ; echo $inputdir.* )` + ex_run_dirs=`( cd $dir ; ls -d $inputdir.* 2> /dev/null )` fi #echo "ex_run_dirs='$ex_run_dirs'" for exd in $ex_run_dirs ; do name=`echo $exd | sed -e "s/$inputdir\.//"` refExOut=`echo $ref_outp | sed "s/\./.${name}./"` outf="$dir/results/$refExOut" - #if test -f $outf -a -r $outf ; then if test -r $outf -o -r ${outf}.gz ; then if test "x$MULTI_THREAD" = "xt" ; then if test -r $dir"/"$exd"/eedata.mth" ; then @@ -1587,8 +1689,7 @@ linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir runmodel $dir/$pfxdir.$ex && run=Y \ && results=`testoutput_run $dir $pfxdir.$ex $refExOut` - fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` - fres="$fres.$ex" + fres=`formatresults $dir.$ex ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo 1>&2 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" @@ -1597,33 +1698,51 @@ fi done - if test -f $DRESULTS"/"genmake_state ; then : ; else + if test ! -f $DRESULTS"/"genmake_state ; then if test -f $dir/$builddir/Makefile ; then mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'` echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state - sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state +# sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state +# bsd-sed cannot do the above code + cat $dir/$builddir/Makefile | \ + sed -n '/^# executed by:/{N + p + }' >> $DRESULTS/genmake_state echo " $mkOpt" >> $DRESULTS/genmake_state if test "x$OPTFILE" = xNONE ; then eval $mkOpt - sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\ - $SUMMARY > tr_0.tmp_log +# sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\ +# $SUMMARY > tmp.tr_log +# bsd-sed requires a newline after "a\": + sed "/^No \"OPTFILE\" was specified ; genmake2/a\\ + OPTFILE=${OPTFILE}" $SUMMARY > tmp.tr_log RETVAL=$? - if test "x$RETVAL" = x0 ; then rm -f $SUMMARY - cp tr_0.tmp_log $SUMMARY - else rm -f tr_0.tmp_log + if test "x$RETVAL" = x0 ; then + cp -f tmp.tr_log $SUMMARY fi + rm -f tmp.tr_log fi gmkLog=$dir/$builddir/genmake.log - grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1 - RETVAL=$? - if test "x$RETVAL" = x0 ; then - echo "from '$gmkLog', extract compiler version:" >> $DRESULTS/genmake_state - sed -n '/Get compiler version/,/<-- compiler version/p' $gmkLog \ - | grep -v '^... compiler version ' > tr_1.tmp_log - sed -n '1p' tr_1.tmp_log >> $DRESULTS/genmake_state - sed -n '2,/^$/p' tr_1.tmp_log | sed '/^$/d' | sed 's/^./ &/' \ - >> $DRESULTS/genmake_state - rm -f tr_1.tmp_log + if test -r $gmkLog ; then + grep '^Get compiler version using:' $gmkLog > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" = x0 ; then + echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state + echo "extract compiler version:" >> $DRESULTS/genmake_state + sed -n '/Get compiler version/,/<-- compiler version/p' \ + $gmkLog | grep -v '^... compiler version ' > tmp.tr_log + sed -n '1p' tmp.tr_log >> $DRESULTS/genmake_state + sed -n '2,/^$/p' tmp.tr_log | sed '/^$/d' | sed 's/^./ &/' \ + >> $DRESULTS/genmake_state + rm -f tmp.tr_log + fi + fi + gmkLog=$dir/$builddir/genmake_state + if test -r $gmkLog ; then + echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state + echo "get genmake settings:" >> $DRESULTS/genmake_state + sed -n '/^HAVE_/p' $gmkLog | sed 's/^./ &/' \ + >> $DRESULTS/genmake_state fi fi fi @@ -1648,10 +1767,13 @@ if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then echo "No results email was sent." else - if test "x$HAVE_MPACK" = xt ; then - tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \ - && gzip $DRESULTS".tar" \ - && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES + if test "x$SENDCMD" != x ; then + if [ $verbose -gt 1 ]; then + echo " run: $SENDCMD -s MITgcm-test -m 3555000 ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESSES" + fi + tar -cf ${SAVDIR}/${DRESULTS}".tar" $DRESULTS > /dev/null 2>&1 \ + && gzip ${SAVDIR}/${DRESULTS}".tar" \ + && $SENDCMD -s MITgcm-test -m 3555000 ${SAVDIR}/${DRESULTS}".tar.gz" $ADDRESSES RETVAL=$? if test "x$RETVAL" != x0 ; then echo @@ -1664,8 +1786,8 @@ echo "An email containing results was sent to the following addresses:" echo " \"$ADDRESSES\"" echo - test -f $DRESULTS".tar" && rm -f $DRESULTS".tar" - test -f $DRESULTS".tar.gz" && rm -f $DRESULTS".tar.gz" + test -f ${SAVDIR}/${DRESULTS}".tar" && rm -f ${SAVDIR}/${DRESULTS}".tar" + test -f ${SAVDIR}/${DRESULTS}".tar.gz" && rm -f ${SAVDIR}/${DRESULTS}".tar.gz" fi fi fi @@ -1685,3 +1807,4 @@ if test "x$DELDIR" = xt ; then rm -rf $DRESULTS fi +echo "======== End of testreport execution ========"