--- MITgcm/verification/testreport 2013/08/22 19:34:15 1.194 +++ MITgcm/verification/testreport 2017/01/11 23:21:21 1.211 @@ -1,6 +1,6 @@ #! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.194 2013/08/22 19:34:15 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.211 2017/01/11 23:21:21 jmc Exp $ # $Name: $ # @@ -19,9 +19,10 @@ echo " (-mpi) use MPI to compile and run on 2 processors" echo " (-MPI) NUMBER use MPI to compile and run on max NUMBER procs" echo " (-mfile|-mf) STRING MPI: file with list of possible machines to run on" - echo " (-command|-c) STRING command to run (needed for MPI run)" + echo " (-command|-c) STRING command to run (e.g., if non-standard MPI setting)" + echo " DEF='mitgcmuv' or ='mpirun -np TR_NPROC mitgcmuv'" echo " ---- testing options : ----" - echo " (-optfile|-of) STRING list of optfiles to use" + echo " (-optfile|-of) STRING optfile to use" echo " (-fast) use optfile default for compiler flags (no '-ieee')" echo " DEF=off => use IEEE numerics option (if available)" echo " (-devel) use optfile developement flags (if available)" @@ -41,6 +42,8 @@ echo " (-bash|-b) STRING preferred location of a \"bash\" or \"sh\" shell" echo " (DEF=\"\" for \"bash\")" echo " (-ef) STRING used as genmake2 \"-extra_flag\" argument" + echo " (-ncad) use genmake2 option \"-nocat4ad\" (-ncad)" + echo " (-small_f) make target small_f before making target all" echo " (-makedepend|-md) STRING command to use for \"makedepend\"" echo " (-make|-m) STRING command to use for \"make\"" echo " (DEF=\"make\")" @@ -60,6 +63,8 @@ echo " (-clean) *ONLY* run \"make CLEAN\" & clean run-dir" echo " (-norun|-nr) skip the \"runmodel\" stage (stop after make)" echo " (-obj) only produces objects (=norun & no executable)" + echo " (-src) only produces small '*.f' src files (not even obj)" + echo " + with: '-adm/-tlm', also makes taf outp src code" echo " (-runonly|-ro) *ONLY* run stage (=\"-quick\" without make)" echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\"" echo " (-nogenmake|-ng) skip the genmake stage" @@ -129,7 +134,7 @@ echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2 fi if [ -r $1/$4/$OUTPUTFILE ]; then - grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > ${TMP}a.txt + grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | nl > ${TMP}a.txt lncntA=`wc -l ${TMP}a.txt | awk '{print $1}' ` if [ $lncntA -lt 2 ]; then if [ $verbose -gt 0 ]; then @@ -144,7 +149,7 @@ if [ $debug -gt 0 ]; then echo testoutput_var: grep "$2" $1/$5 1>&2 fi - grep "$2" $1/$5 | sed 's/.*=//' | cat -n > ${TMP}b.txt + grep "$2" $1/$5 | sed 's/.*=//' | nl > ${TMP}b.txt lncntB=`wc -l ${TMP}b.txt | awk '{print $1}' ` if [ $lncntB -lt 2 ]; then if [ $verbose -gt 0 ]; then @@ -267,7 +272,11 @@ allargs="" for xx in $listVar do - case $xx in + #ii=`echo $xx | sed 's/^pt[0-9][0-9]*..$/XXX/'` #- any ptr number + ii=`echo $xx | sed 's/^pt[1-9]..$/XXX/'` #- 1 to 9 ptr number + if test $ii != 'XXX' ; then + #- all except ptracer stats: + case $xx in 'PS') if [ $debug -gt 0 ] then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $refoutp ; yy=$? @@ -294,14 +303,6 @@ 'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $refoutp ; yy=$? ;; 'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $refoutp ; yy=$? ;; 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $refoutp ; yy=$? ;; - 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` - testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $refoutp ; yy=$? ;; - 'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` - testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $refoutp ; yy=$? ;; - 'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` - testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $refoutp ; yy=$? ;; - 'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` - testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $refoutp ; yy=$? ;; 'Qntmn') testoutput_var $1 "forcing_qnet_min" "Qnet minimum" $2 $refoutp ; yy=$? ;; 'Qntmx') testoutput_var $1 "forcing_qnet_max" "Qnet maximum" $2 $refoutp ; yy=$? ;; 'Qntav') testoutput_var $1 "forcing_qnet_mean" "Qnet mean" $2 $refoutp ; yy=$? ;; @@ -330,9 +331,24 @@ 'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North-av" $2 $refoutp ; yy=$? ;; 'HthMxS') testoutput_var $1 "thSI_IceH_max_S" "thSIc H South-max" $2 $refoutp ; yy=$? ;; 'HthMxN') testoutput_var $1 "thSI_IceH_max_N" "thSIc H North-max" $2 $refoutp ; yy=$? ;; + 'sbo_M') testoutput_var $1 "sbo_mass" "SBO mass" $2 $refoutp ; yy=$? ;; + 'sboFW') testoutput_var $1 "sbo_mass_fw" "SBO m-FW" $2 $refoutp ; yy=$? ;; + 'sboAc') testoutput_var $1 "sbo_zoamc" "SBO AM-C" $2 $refoutp ; yy=$? ;; + 'sboAp') testoutput_var $1 "sbo_zoamp" "SBO AM-P" $2 $refoutp ; yy=$? ;; 'StrmIc') testoutput_var $1 "STREAMICE_FP_ERR" "StreamIce Solver" $2 $refoutp ; yy=$? ;; *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;; - esac + esac + else + #- ptracers stats: + nn=`echo $xx | sed 's/pt//' | sed 's/..$//'` + ii=`echo $xx | sed 's/^pt[0-9]*//'` + case $ii in + 'mn') testoutput_var $1 "trcstat_ptracer0"$nn"_min" "p0"$nn"_min" $2 $refoutp ; yy=$? ;; + 'mx') testoutput_var $1 "trcstat_ptracer0"$nn"_max" "p0"$nn"_max" $2 $refoutp ; yy=$? ;; + 'av') testoutput_var $1 "trcstat_ptracer0"$nn"_mean" "p0"$nn"_mean" $2 $refoutp ; yy=$? ;; + 'sd') testoutput_var $1 "trcstat_ptracer0"$nn"_sd" "p0"$nn"_StDv" $2 $refoutp ; yy=$? ;; + esac + fi if test $xx = $sVar then allargs="$allargs > $yy <" else allargs="$allargs $yy" @@ -367,12 +383,10 @@ if test "x$MKDEPEND" != x ; then command="$command -makedepend=$MKDEPEND" fi - if test $KIND = 1 -o $KIND = 2 ; then - command="$command --mods=../code_ad" - elif test $KIND = 4 ; then - command="$command -adof ../../../tools/adjoint_options/adjoint_oad -mods ../code_oad" + if test $KIND = 4 ; then + command="$command -oad -mods=../$code_dir" else - command="$command -mods=../code" + command="$command -mods=../$code_dir" fi if test "x$OPTFILE" != xNONE ; then command="$command -optfile=$OPTFILE" @@ -399,6 +413,9 @@ if test "x$EXTRFLG" != x ; then command="$command -extra_flag $EXTRFLG" fi + if test "x$NOCATAD" = xt ; then + command="$command -nocat4ad" + fi if test "x$TS" = xt ; then command="$command -ts" fi @@ -520,12 +537,23 @@ else cd $1; if test -r Makefile ; then - printf 'make ... ' - if test "x$REPLMAKE" = x ; then - $MAKE $TARG >> make.tr_log 2>&1 - else - $REPLMAKE $TARG >> make.tr_log 2>&1 + if test "x$MKSMALLF" = xt ; then + printf 'make small_f ... ' + $MAKE_CMD small_f >> make.tr_log 2>&1 + RETVAL=$? + if test "x$RETVAL" != x0 ; then + tail make.tr_log + echo failed + cp genmake.log genmake.tr_log $CDIR + tail -$NBLINES_MKLOG make.tr_log > $CDIR"/make.tr_log_tail" + rm -f $EXECUTABLE + mk_fail=1 + else + echo successful + fi fi + printf "make $TARG ... " + $MAKE_CMD $TARG >> make.tr_log 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.tr_log @@ -773,7 +801,7 @@ if test "x$MPI" != x0 ; then #- adjust the MPI run command with the right number of Procs #echo '' ; echo " COMMAND='$COMMAND'" - COMMAND=`echo $COMMAND | sed "s/ TR_NPROC / $LOC_NPROC /"` + COMMAND=`echo $COMMAND | sed "s/ TR_NPROC/ $LOC_NPROC/"` if test "x$MPI_MFILE" != x ; then COMMAND=`echo $COMMAND | sed "s/ TR_MFILE / ..\/..\/$LOC_MFILE /"` fi @@ -839,8 +867,15 @@ if test -f $OUTPUTFILE ; then grep '(PID\.TID 0000\.0001) n.. =' $OUTPUTFILE \ | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" - grep -A3 'Seconds in section "ALL' $OUTPUTFILE \ - | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" +# grep -A3 'Seconds in section "ALL' $OUTPUTFILE \ +# | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" +# some implementations of grep cannot do contextual searches so we +# replace the above with a sed command + cat $OUTPUTFILE | sed -n '/Seconds in section "ALL/{N + N + N + p + }' | sed 's/(PID.TID 0000.0001) //' >> $CDIR"/summary.txt" fi if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then @@ -1041,7 +1076,7 @@ if test "x$MAKE" = x ; then MAKE=make fi -REPLMAKE= +MAKE_CMD= if test "x$CC" = x ; then CC=cc fi @@ -1054,6 +1089,8 @@ DELDIR= USE_R4= EXTRFLG= +NOCATAD= +MKSMALLF= #- type of testing (KIND): # KIND=0 : forward (= default) ; KIND=1 : Tangent Linear with TAF ; @@ -1113,8 +1150,8 @@ -make | --make | -m | --m) ac_prev=MAKE ;; -make=* | --make=* | -m=* | --m=*) MAKE=$ac_optarg ;; - -repl_mk | --repl_mk ) ac_prev=REPLMAKE ;; - -repl_mk=* | --repl_mk=*) REPLMAKE=$ac_optarg ;; + -repl_mk | --repl_mk ) ac_prev=MAKE_CMD ;; + -repl_mk=* | --repl_mk=*) MAKE_CMD=$ac_optarg ;; -odir | --odir) ac_prev=OUTDIR ;; -odir=* | --odir=*) OUTDIR=$ac_optarg ;; @@ -1135,6 +1172,7 @@ -norun | --norun | -nr | --nr) NORUN=t ;; -obj | --obj ) TARG='obj' ; NORUN=t ;; + -src | --src ) TARG='small_f' ; NORUN=t ;; -runonly | --runonly | -ro | --ro) QUICK=t ; NOMAKE=t ;; -quick | --quick | -q | --q) QUICK=t ;; -nogenmake | --nogenmake | -ng | --ng) NOGENMAKE=t ;; @@ -1162,6 +1200,8 @@ -oad) if test $KIND = 0 ; then KIND=4 ; NODEPEND=t ; else echo "Error: '-tlm', '-adm' and '-oad' are exclusive" ; usage fi ;; + -ncad) NOCATAD=t ;; + -small_f) MKSMALLF=t ;; -ieee) echo "Warning: ignore option '-ieee' (already the default)" printf " ... " ;; @@ -1181,7 +1221,7 @@ -ts) TS=t;; -papis) PAPIS=t;; - -pcls) PCL=t;; + -pcls) PCLS=t;; -*) echo "Error: unrecognized option: "$ac_option usage ;; @@ -1197,6 +1237,7 @@ NOCLEAN=t NODEPEND=t fi +if test "x$MAKE_CMD" = x ; then MAKE_CMD=$MAKE ; fi #- check length of MPI machine file: if test "x$MPI" != x0 -a "x$MPI_MFILE" != x ; then @@ -1217,13 +1258,15 @@ #- setting for forward or ADM testing if test $KIND = 1 ; then - TARG=ftlall + if test "x$TARG" = xsmall_f ; then TARG=ftltaf ; else TARG=ftlall ; fi code_dir=code_ad inputdir=input_ad ref_outp="output_tlm.txt" EXECUTABLE="mitgcmuv_ftl" elif test $KIND = 2 ; then - if test "x$TARG" = xobj ; then TARG=adobj ; else TARG=adall ; fi + if test "x$TARG" = x ; then TARG=adall ; fi + if test "x$TARG" = xobj ; then TARG=adobj ; fi + if test "x$TARG" = xsmall_f ; then TARG=adtaf ; fi code_dir=code_ad inputdir=input_ad ref_outp="output_adm.txt" @@ -1235,12 +1278,12 @@ ref_outp="output_oadm.txt" EXECUTABLE="mitgcmuv_ad" else - if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi code_dir=code inputdir=input ref_outp="output.txt" EXECUTABLE="mitgcmuv" fi +if test "x$JOBS" != x ; then TARG="-j $JOBS $TARG" ; fi xx=`echo $TESTDIRS | awk '{print $1}'` if test "x$TESTDIRS" = x ; then @@ -1287,33 +1330,29 @@ if test $count = 1 ; then echo "" ; echo -n " ... " ; fi #echo 'TESTDIRS='${TESTDIRS}'<' -if test "x$OPTFILE" = xNONE -a "x$MITGCM_OF" != x ; then - OPTFILE=$MITGCM_OF -fi - LOC_MFILE='tr_mpi_mfile' RUNLOG="run.tr_log" -OUTPUTFILE=$ref_outp -if test "x$COMMAND" = x ; then - COMMAND="./$EXECUTABLE > $OUTPUTFILE" -fi -if test "x$MPI" != x0 ; then - OUTPUTFILE="STDOUT.0000" +if test "x$MPI" = x0 ; then + OUTPUTFILE=$ref_outp + if test "x$COMMAND" = x ; then COMMAND="./$EXECUTABLE > $OUTPUTFILE" ; fi +else + OUTPUTFILE="STDOUT.0000" + if test "x$COMMAND" = x ; then COMMAND="mpirun -np TR_NPROC ./$EXECUTABLE" ; fi fi -echo "OK (COMMAND= $COMMAND )" +echo "OK (COMMAND='$COMMAND')" -TMP=./tr_$$ +#TMP=./tr_$$ #- try to put temporary files in system-local /tmp dir -#TMP=/tmp/tr_$$ -#touch $TMP ; retVal=$? -#if [ $retVal -eq 0 ] ; then -# if test ! -r $TMP ; then TMP=./tr_$$ ; fi -#else -# TMP=./tr_$$ -#fi -#rm -f $TMP -#if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi +TMP=/tmp/tr_${USER}_$$ +touch $TMP ; retVal=$? +if [ $retVal -eq 0 ] ; then + if test ! -r $TMP ; then TMP=./tr_$$ ; fi +else + TMP=./tr_$$ +fi +rm -f $TMP +if [ $verbose -gt 1 ]; then echo " temp files: $TMP" ; fi # set the Default List of output variables to be checked: # (use default or load experiment-specific list from file "tr_checklist") @@ -1360,7 +1399,6 @@ CMDLINE="$CMDLINE $yy" ; fi fi done -#for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done MACH=`hostname` UNAMEA=`uname -a` DATE=`date +%Y%m%d` @@ -1368,7 +1406,9 @@ if test "x$OUTDIR" != x ; then BASE="tr_"$OUTDIR"_"$DATE"_" else - short_name=`hostname | sed 's/\..*$//'` + #short_name=`hostname -s | tr '[:upper:]' '[:lower:]'` + # hostname -s is not universal (does work on AIX system) + short_name=`hostname | sed 's/\..*$//' | tr '[:upper:]' '[:lower:]'` BASE="tr_"$short_name"_"$DATE"_" fi DNUM=0 @@ -1625,6 +1665,7 @@ && symlink_mpifiles $dir $code_dir $builddir \ && makedependmodel $dir/$builddir && makedepend=Y \ && makemodel $dir/$builddir && make=Y + echo else genmakemodel $dir/$builddir && genmake=Y \ && makeclean $dir/$builddir \ @@ -1635,11 +1676,11 @@ && linkdata $dir/$rundir $inputdir \ && runmodel $dir/$rundir && run=Y \ && results=`testoutput_run $dir $rundir $ref_outp` + echo 1>&2 fi #echo "results='$results'" fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` - echo 1>&2 echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" @@ -1661,8 +1702,9 @@ linkdata $dir/$pfxdir.$ex $inputdir.$ex $inputdir runmodel $dir/$pfxdir.$ex && run=Y \ && results=`testoutput_run $dir $pfxdir.$ex $refExOut` - fres=`formatresults $dir.$ex ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo 1>&2 + + fres=`formatresults $dir.$ex ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $locDIR"/summary.txt" if test "x$POSTCLEAN" = x2 ; then @@ -1674,12 +1716,20 @@ if test -f $dir/$builddir/Makefile ; then mkOpt=`grep '^# OPTFILE=' $dir/$builddir/Makefile 2>/dev/null | head -1 | sed 's/^# //'` echo "from '$dir/$builddir/Makefile', extract:" > $DRESULTS/genmake_state - sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state +# sed -n '/^# executed by:/,+1 p' $dir/$builddir/Makefile >> $DRESULTS/genmake_state +# bsd-sed cannot do the above code + cat $dir/$builddir/Makefile | \ + sed -n '/^# executed by:/{N + p + }' >> $DRESULTS/genmake_state echo " $mkOpt" >> $DRESULTS/genmake_state if test "x$OPTFILE" = xNONE ; then eval $mkOpt - sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\ - $SUMMARY > tmp.tr_log +# sed "/^No \"OPTFILE\" was specified ; genmake2/a\ OPTFILE=${OPTFILE}"\ +# $SUMMARY > tmp.tr_log +# bsd-sed requires a newline after "a\": + sed "/^No \"OPTFILE\" was specified ; genmake2/a\\ + OPTFILE=${OPTFILE}" $SUMMARY > tmp.tr_log RETVAL=$? if test "x$RETVAL" = x0 ; then cp -f tmp.tr_log $SUMMARY @@ -1700,10 +1750,13 @@ >> $DRESULTS/genmake_state rm -f tmp.tr_log fi - echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state - echo "get NETCDF & LAPACK settings:" >> $DRESULTS/genmake_state - sed -n '/set HAVE_NETCDF=/p' $gmkLog >> $DRESULTS/genmake_state - sed -n '/set HAVE_LAPACK=/p' $gmkLog >> $DRESULTS/genmake_state + fi + gmkLog=$dir/$builddir/genmake_state + if test -r $gmkLog ; then + echo -n "from '$gmkLog', " >> $DRESULTS/genmake_state + echo "get genmake settings:" >> $DRESULTS/genmake_state + sed -n '/^HAVE_/p' $gmkLog | sed 's/^./ &/' \ + >> $DRESULTS/genmake_state fi fi fi