--- MITgcm/verification/testreport 2004/03/12 16:45:38 1.35 +++ MITgcm/verification/testreport 2008/08/19 18:29:12 1.117 @@ -1,6 +1,6 @@ #! /usr/bin/env bash # -# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.35 2004/03/12 16:45:38 edhill Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/verification/testreport,v 1.117 2008/08/19 18:29:12 jmc Exp $ # $Name: $ # @@ -11,14 +11,21 @@ echo echo "where possible OPTIONS are:" echo " (-help|-h) print usage" - echo " (-mpi) use MPI input files" + echo " (-mth) run multi threaded (using eedata.mth)" + echo " (-mpi) compile and run using MPI" echo " (-ieee|-noieee) if possible, use IEEE compiler flags" - echo " (DEF=\"noieee\")" - echo " (-optfile=|-of=)STRING list of optfiles to use" + echo " (DEF=\"-ieee\")" + echo " (-gsl) compile with \"-gsl\" flag" + echo " (-of=|-optfile=)STRING list of optfiles to use" echo " (-a|-addr) STRING list of email recipients" - echo " (DEF=\"edhill@mitgcm.org\")" - echo " (-t|-tdir) STRING list of test dirs to use" - echo " (DEF=\"\" which builds all)" + echo " (DEF=\"\" no email is sent)" + echo " (-mpd|-mpackdir) DIR location of the mpack utility" + echo " (DEF=\"../tools/mpack-1.6\")" + echo " (-t|-tdir) STRING list of group and/or exp. dirs to test" + echo " (recognized groups: basic, tutorials)" + echo " (DEF=\"\" which test all)" + echo " (-skd|-skipdir) STRING list of exp. dirs to skip" + echo " (DEF=\"\" which test all)" echo " (-b|-bash) STRING preferred location of a \"bash\" or" echo " Bourne-compatible \"sh\" shell" echo " (DEF=\"\" for \"bash\")" @@ -27,17 +34,32 @@ echo " (DEF=\"make output.txt\")" echo " (-m|-make) STRING command to use for \"make\"" echo " (DEF=\"make\")" + echo " (-odir) STRING used to build output directory name" + echo " (DEF=\"hostname\")" + echo " (-ptr|-ptracers) STRING specify which ptracers to test" + echo " (DEF=\"1 2 3 4 5\")" + echo " (-match) NUMBER Matching Criteria (number of digits)" + echo " (DEF=\"12\")" + echo " (-j) JOBS use \"make -j JOBS\" for parallel builds" echo " (-clean) *ONLY* run \"make CLEAN\"" echo " (-quick|-q) same as \"-nogenmake -noclean -nodepend\"" echo " (-nogenmake|-ng) skip the genmake stage" echo " (-noclean|-nc) skip the \"make clean\" stage" echo " (-nodepend|-nd) skip the \"make depend\" stage" + echo " (-deldir|-dd) on success, delete the output directory" + echo " (-ts) provide timing information per timestep" + echo " (-papis) provide MFlop/s per timestep using PAPI" + echo " (-pcls) provide MFlop/s per timestep using PCL" echo - echo "and where STRING follows a whitespace-delimited format" + echo "and where STRING can be a whitespace-delimited list" echo "such as:" + echo echo " -t 'exp0 exp2 exp3' " echo " -addr='abc@123.com testing@home.org'" echo + echo "provided that the expression is properly quoted within the current" + echo "shell (note the use of single quotes to protect white space)." + echo exit 1 } @@ -45,7 +67,8 @@ build_mpack() { printf "building the mpack utility... " - if test ! -x "$MPACKDIR/mpack" ; then + MPACK="$MPACKDIR/mpack" + if test ! -x $MPACK ; then if test ! -d $MPACKDIR ; then echo echo "Error: can't find \"$MPACKDIR\"" @@ -54,10 +77,10 @@ echo HAVE_MPACK=f fi - printf "building mpack... " if test "x$CC" = x ; then export CC=cc fi + printf "building mpack (using CC=$CC)... " ( cd $MPACKDIR && ./configure && $MAKE ) > tr_build_mpack.out 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then @@ -68,53 +91,71 @@ else rm -f tr_build_mpack.out HAVE_MPACK=t + echo "done" fi else HAVE_MPACK=t + echo "already exist" fi - echo "OK" } -testoutput_for_prop() +testoutput_var() { - # testoutput_for_prop dir s1 label subdir extension + # testoutput_var dir s1 label subdir reference_output # - # compares files in $dir/$subdir/output.txt and $dir/results/output.txt - # using search strings s1 and text label + # compares 1 variable output selected from file $dir/$subdir/$OUTPUTFILE + # with same output from reference file $dir/results/$reference_output + # using search strings s1 and text label if [ $debug -gt 0 ]; then - echo testoutput_for_prop: grep "$2" $1/$4/output.txt 1>&2 + echo testoutput_var: grep "$2" $1/$4/$OUTPUTFILE 1>&2 fi - if [ -r $1/$4/output.txt ]; then - grep "$2" $1/$4/output.txt | sed 's/.*=//' | cat -n > tmp1.txt - lncnt=`wc -l tmp1.txt | awk '{print $1}' ` - if [ $lncnt -lt 3 ]; then + if [ -r $1/$4/$OUTPUTFILE ]; then + grep "$2" $1/$4/$OUTPUTFILE | sed 's/.*=//' | cat -n > tmp1.txt + lncntA=`wc -l tmp1.txt | awk '{print $1}' ` + if [ $lncntA -lt 2 ]; then if [ $verbose -gt 0 ]; then echo Not enough lines of output when searching for "$2" 1>&2 fi return 99 fi else - echo testoutput_for_prop: output.txt from model run was not readable 1>&2 + echo testoutput_var: $OUTPUTFILE from model run was not readable 1>&2 return 99 fi if [ $debug -gt 0 ]; then - echo testoutput_for_prop: grep "$2" $1/results/output.txt$5 1>&2 + echo testoutput_var: grep "$2" $1/results/$5 1>&2 fi - grep "$2" $1/results/output.txt$5 | sed 's/.*=//' | cat -n > tmp2.txt - lncnt=`wc -l tmp2.txt | awk '{print $1}' ` - if [ $lncnt -lt 3 ]; then + grep "$2" $1/results/$5 | sed 's/.*=//' | cat -n > tmp2.txt + lncntB=`wc -l tmp2.txt | awk '{print $1}' ` + if [ $lncntB -lt 2 ]; then if [ $verbose -gt 0 ]; then echo Not enough lines of output when searching for "$2" 1>&2 fi return 99 fi + if [ $lncntA -ne $lncntB ]; then + if [ $verbose -gt 0 ]; then + echo Not same Nb of lines when searching for "$2" ":" $lncntA $lncntB 1>&2 + fi + return 99 + fi + has_nan=`cat tmp1.txt | grep -i nan | wc -l` + if [ $has_nan -gt 0 ] ; then + echo testoutput_var: $OUTPUTFILE contains $has_nan NaN values 1>&2 + return 99 + fi + has_inf=`cat tmp1.txt | grep -i inf | wc -l` + if [ $has_inf -gt 0 ] ; then + echo testoutput_var: $OUTPUTFILE contains $has_inf Inf values 1>&2 + return 99 + fi if [ $debug -gt 0 ]; then - echo testoutput_for_prop: join tmp1.txt tmp2.txt 1>&2 + echo testoutput_var: join tmp1.txt tmp2.txt 1>&2 fi join tmp1.txt tmp2.txt | awk '{print $1 " " $2 " " $3}' > tmp3.txt if [ $debug -gt 0 ]; then - echo testoutput_for_prop: compare_lines 1>&2 + echo testoutput_var: compare_lines 1>&2 fi if [ $verbose -gt 1 ]; then cat tmp3.txt 1>&2 @@ -122,15 +163,15 @@ echo "-1" >> tmp3.txt # On the SGI O3K (*not* the O2K), "cat -n" inserts a ":" after the line number cat tmp3.txt | sed -e 's|:||g' > tmp4.txt - digits_of_similarity=`./tmp_cmpnum < tmp4.txt` + digits_of_similarity=`./tr_cmpnum < tmp4.txt` if [ $digits_of_similarity -eq 99 ]; then if [ $verbose -gt 0 ]; then - echo testoutput_for_prop: No comparison was available for \"$2\" 1>&2 + echo testoutput_var: No comparison was available for \"$3\" 1>&2 fi digits_of_similarity=99 else if [ $verbose -gt 0 ]; then - echo There were $digits_of_similarity decimal places of similarity for \"$2\" 1>&2 + echo There were $digits_of_similarity decimal places of similarity for \"$3\" 1>&2 fi fi rm -f tmp1.txt tmp2.txt tmp3.txt tmp4.txt @@ -138,71 +179,125 @@ return $digits_of_similarity } -dashnum() -{ - # dashnum n1 n2 n3 ... - # - # print numbers using %3i format or "--" if number = 99 - - for num in $@ ; do - if [ $num = 99 ]; then - printf ' --' - else - printf '%3i' $num - fi - done -} - -testoutput_ad() -{ - grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $5}' > t05.txt - grep $3 $1/$2/output.txt_adm | awk '{print NR " " $5}' > t15.txt - grep $3 $1/results_ad/output.txt_adm | awk '{print NR " " $6}' > t06.txt - grep $3 $1/$2/output.txt_adm | awk '{print NR " " $6}' > t16.txt - join t05.txt t15.txt > t5.txt - join t06.txt t16.txt > t6.txt - echo "-1" >> t5.txt - echo "-1" >> t6.txt - digits_5=`./tmp_cmpnum < t5.txt` - digits_6=`./tmp_cmpnum < t6.txt` - dashnum $digits_5 $digits_6 - rm -f t[01][56].txt t[56].txt -} - -testoutput() +testoutput_run() { - # testoutput directory subdir extension + # testoutput_run directory subdir reference_output # - # test output in "directory" - if test "x$ADM" = x ; then - if [ $debug -gt 0 ]; then - echo testoutput: testoutput_for_prop $1 cg2d_init_res 1>&2 - fi - testoutput_for_prop $1 "cg2d_init_res" "cg2d init. residual" $2 $3; cg2dres=$? - if [ $debug -gt 0 ]; then - echo testoutput: cg2dres=$cg2dres 1>&2 - fi - testoutput_for_prop $1 "dynstat_theta_min" "theta minimum" $2 $3; tmin=$? - testoutput_for_prop $1 "dynstat_theta_max" "theta maximum" $2 $3; tmax=$? - testoutput_for_prop $1 "dynstat_theta_mean" "theta mean" $2 $3; tmean=$? - testoutput_for_prop $1 "dynstat_theta_sd" "theta s.d." $2 $3; tsd=$? - testoutput_for_prop $1 "dynstat_salt_min" "salt minimum" $2 $3; smin=$? - testoutput_for_prop $1 "dynstat_salt_max" "salt maximum" $2 $3; smax=$? - testoutput_for_prop $1 "dynstat_salt_mean" "salt mean" $2 $3; smean=$? - testoutput_for_prop $1 "dynstat_salt_sd" "salt s.d." $2 $3; ssd=$? - testoutput_for_prop $1 "dynstat_uvel_min" "U minimum" $2 $3; umin=$? - testoutput_for_prop $1 "dynstat_uvel_max" "U maximum" $2 $3; umax=$? - testoutput_for_prop $1 "dynstat_uvel_mean" "U mean" $2 $3; umean=$? - testoutput_for_prop $1 "dynstat_uvel_sd" "U s.d." $2 $3; usd=$? - testoutput_for_prop $1 "dynstat_vvel_min" "V minimum" $2 $3; vmin=$? - testoutput_for_prop $1 "dynstat_vvel_max" "V maximum" $2 $3; vmax=$? - testoutput_for_prop $1 "dynstat_vvel_mean" "V mean" $2 $3; vmean=$? - testoutput_for_prop $1 "dynstat_vvel_sd" "V s.d." $2 $3; vsd=$? - dashnum $cg2dres $tmin $tmax $tmean $tsd $smin $smax $smean $ssd \ - $umin $umax $umean $usd $vmin $vmax $vmean $vsd - else - testoutput_ad $1 $2 "precision_grdchk_result" - fi + # test output from 1 run in "directory" +# --> same processing for adjoint & forward test + # default list of output variables to be checked: + # 1rst : main variable used to decide if it pass or FAIL + # others : number of matching digits to be printed in summary.txt + listChk=$DEF_CHECK_LIST + # load experiment-specific list from file "tr_checklist" (if it exist) + if test -r $1/$2/tr_checklist ; then listChk=`cat $1/$2/tr_checklist` ; fi + sVar=`echo $listChk | awk '{print $1}'` + # remove 1rst var and expand the list: + => min max mean s.d + listVar=`echo $listChk | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' \ + | sed 's/+//g' | sed "s/^$sVar//"` + if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(I)='$listVar'" 1>&2 ; fi + # check for ptracer output in reference_output file : + outpref=$1/results/$3 + ptr_mon="trcstat_ptracerXX_min trcstat_ptracerXX_max" + ptr_mon="$ptr_mon trcstat_ptracerXX_mean trcstat_ptracerXX_sd" + for ii in $PTRACERS_NUM ; do + ptrfound=0 + for jj in $ptr_mon ; do + name=`eval "echo $jj | sed -e 's|XX|0"$ii"|g'"` + tst=`grep $name $outpref | wc -l | awk '{print $1}'` + if test ! "x$tst" = x0 ; then ptrfound=1 ; fi + done + if test $ptrfound = '1' ; then + eval "HAVE_PTR0"$ii"=t" + else + eval "HAVE_PTR0"$ii"=f" + if test "x$ADM" = x ; then + # remove this ptr from the list of output variable to check + # echo "-- ptr test=" $tst "number of var=" `echo $listVar | awk '{print NF}'` 1>&2 + listVar=`echo "$listVar" | sed "s/ pt$ii..//g"` + fi + fi + # eval 'echo "HAVE_PTR0'$ii' = $HAVE_PTR0'$ii'"' 1>&2 + done + tst=`echo $sVar $listVar | awk '{ for(i=2;i<=NF;i++){if($i==$1)t+=1}; print t }'` + if test $tst != 1 ; then + if test $tst = 0 ; then echo "==> WARNING: selected var >$sVar< not found" 1>&2 + else echo "==> WARNING: found selected var >$sVar< $tst times" 1>&2 ; fi + echo "==> WARNING: in checked list:" $listVar 1>&2 + #- put it back once: + listVar=" $sVar "`echo "$listVar " | sed "s/ $sVar / /g"` + fi + if [ $debug -gt 0 ]; then echo "testoutput_run: listVar(M)='$listVar'" 1>&2 ; fi + echo "listVar='$listVar'" > $CDIR"/summary.txt" + allargs="" + for xx in $listVar + do + case $xx in + 'PS') if [ $debug -gt 0 ] + then echo testoutput_run: testoutput_var $1 cg2d_init_res 1>&2 ; fi + testoutput_var $1 "cg2d_init_res" "Press. Solver (cg2d)" $2 $3; yy=$? + if [ $debug -gt 0 ] ; then echo testoutput_run: cg2dres=$yy 1>&2 ; fi ;; + 'Cost') testoutput_var $1 "ADM precision_derivative_cost" "ADM Cost" $2 $3; yy=$? ;; + 'Grad') testoutput_var $1 "ADM precision_derivative_grad" "ADM Grad" $2 $3; yy=$? ;; + 'Tmn') testoutput_var $1 "dynstat_theta_min" "Theta minimum" $2 $3; yy=$? ;; + 'Tmx') testoutput_var $1 "dynstat_theta_max" "Theta maximum" $2 $3; yy=$? ;; + 'Tav') testoutput_var $1 "dynstat_theta_mean" "Theta mean" $2 $3; yy=$? ;; + 'Tsd') testoutput_var $1 "dynstat_theta_sd" "Theta Std.Dev" $2 $3; yy=$? ;; + 'Smn') testoutput_var $1 "dynstat_salt_min" "Salt minimum" $2 $3; yy=$? ;; + 'Smx') testoutput_var $1 "dynstat_salt_max" "Salt maximum" $2 $3; yy=$? ;; + 'Sav') testoutput_var $1 "dynstat_salt_mean" "Salt mean" $2 $3; yy=$? ;; + 'Ssd') testoutput_var $1 "dynstat_salt_sd" "Salt Std.Dev" $2 $3; yy=$? ;; + 'Umn') testoutput_var $1 "dynstat_uvel_min" "U minimum" $2 $3; yy=$? ;; + 'Umx') testoutput_var $1 "dynstat_uvel_max" "U maximum" $2 $3; yy=$? ;; + 'Uav') testoutput_var $1 "dynstat_uvel_mean" "U mean" $2 $3; yy=$? ;; + 'Usd') testoutput_var $1 "dynstat_uvel_sd" "U Std.Dev" $2 $3; yy=$? ;; + 'Vmn') testoutput_var $1 "dynstat_vvel_min" "V minimum" $2 $3; yy=$? ;; + 'Vmx') testoutput_var $1 "dynstat_vvel_max" "V maximum" $2 $3; yy=$? ;; + 'Vav') testoutput_var $1 "dynstat_vvel_mean" "V mean" $2 $3; yy=$? ;; + 'Vsd') testoutput_var $1 "dynstat_vvel_sd" "V Std.Dev" $2 $3; yy=$? ;; + 'pt1mn'|'pt2mn'|'pt3mn'|'pt4mn'|'pt5mn') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_min" "p0"$ii"_min" $2 $3; yy=$? ;; + 'pt1mx'|'pt2mx'|'pt3mx'|'pt4mx'|'pt5mx') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_max" "p0"$ii"_max" $2 $3; yy=$? ;; + 'pt1av'|'pt2av'|'pt3av'|'pt4av'|'pt5av') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_mean" "p0"$ii"_mean" $2 $3; yy=$? ;; + 'pt1sd'|'pt2sd'|'pt3sd'|'pt4sd'|'pt5sd') ii=`echo $xx | sed 's/pt//' | sed 's/..$//'` + testoutput_var $1 "trcstat_ptracer0"$ii"_sd" "p0"$ii"_StDv" $2 $3; yy=$? ;; + 'Qntmn') testoutput_var $1 "extforcing_qnet_min" "Qnet minimum" $2 $3; yy=$? ;; + 'Qntmx') testoutput_var $1 "extforcing_qnet_max" "Qnet maximum" $2 $3; yy=$? ;; + 'Qntav') testoutput_var $1 "extforcing_qnet_mean" "Qnet mean" $2 $3; yy=$? ;; + 'Qntsd') testoutput_var $1 "extforcing_qnet_sd" "Qnet Std.Dev" $2 $3; yy=$? ;; + 'aSImn') testoutput_var $1 "seaice_area_min" "SIce Area min" $2 $3; yy=$? ;; + 'aSImx') testoutput_var $1 "seaice_area_max" "SIce Area max" $2 $3; yy=$? ;; + 'aSIav') testoutput_var $1 "seaice_area_mean" "SIce Area mean" $2 $3; yy=$? ;; + 'aSIsd') testoutput_var $1 "seaice_area_sd" "SIce Area StDv" $2 $3; yy=$? ;; + 'hSImn') testoutput_var $1 "seaice_heff_min" "SIce Heff min" $2 $3; yy=$? ;; + 'hSImx') testoutput_var $1 "seaice_heff_max" "SIce Hell max" $2 $3; yy=$? ;; + 'hSIav') testoutput_var $1 "seaice_heff_mean" "SIce Hell mean" $2 $3; yy=$? ;; + 'hSIsd') testoutput_var $1 "seaice_heff_sd" "SIce Hell StDv" $2 $3; yy=$? ;; + 'AthSiG') testoutput_var $1 "thSI_Ice_Area_G" "thSIc Area Global" $2 $3; yy=$? ;; + 'AthSiS') testoutput_var $1 "thSI_Ice_Area_S" "thSIc Area South" $2 $3; yy=$? ;; + 'AthSiN') testoutput_var $1 "thSI_Ice_Area_N" "thSIc Area North" $2 $3; yy=$? ;; + 'HthSiG') testoutput_var $1 "thSI_IceH_ave_G" "thSIc H Global" $2 $3; yy=$? ;; + 'HthSiS') testoutput_var $1 "thSI_IceH_ave_S" "thSIc H South" $2 $3; yy=$? ;; + 'HthSiN') testoutput_var $1 "thSI_IceH_ave_N" "thSIc H North" $2 $3; yy=$? ;; + *) yy=99; echo "WARNING: asking for var=$xx : not recognized !" 1>&2 ;; + esac + if test $xx = $sVar + then allargs="$allargs > $yy <" + else allargs="$allargs $yy" + fi + done + + nbVar=`echo $listVar | awk '{print NF}'` + if [ $nbVar -lt $LEN_CHECK_LIST ] ; then + #-- fill line (up to standard length) with dot: + adNul=`expr $LEN_CHECK_LIST - $nbVar | awk '{for(i=1;i<=$1;i++){print "."}}'` + echo $allargs $adNul + else + echo $allargs + fi +# <-- same processing for adjoint & forward test } genmakemodel() @@ -223,7 +318,6 @@ command="$command --mods=../code" else command="$command --mods=../code_ad" - command="$command -adof=../../../tools/adjoint_options/adjoint_staf" fi if test "x$OPTFILE" != xNONE ; then command="$command --optfile=$OPTFILE" @@ -231,10 +325,27 @@ if test "x$IEEE" != x ; then command="$command -ieee" fi + if test "x$GSL" = xt ; then + command="$command -gsl" + fi + if test "x$MPI" = xt ; then + command="$command -mpi" + fi + if test "x$TS" = xt ; then + command="$command -ts" + fi + if test "x$PAPIS" = xt ; then + command="$command -papis" + else + if test "x$PCLS" = xt ; then + command="$command -pcls" + fi + fi printf 'genmake ... ' 1>&2 $command > make.log 2>&1 RETVAL=$? - cp Makefile $CDIR + # Reduce the size of the testing emails! + head -100 Makefile > $CDIR/Makefile_head if test "x$RETVAL" != x0 ; then tail make.log echo "genmakemodel: genmake failed" 1>&2 @@ -251,20 +362,18 @@ { # makeclean directory if test "x$NOCLEAN" = xt ; then - echo "make CLEAN skipped!" + echo "make Clean skipped!" else ( cd $1; - if test -e output.txt ; then - rm -f output.txt - fi - printf 'make CLEAN ... ' 2>&1 + #if test -e $OUTPUTFILE ; then rm -f $OUTPUTFILE ; fi if test -r Makefile ; then - $MAKE CLEAN >> make.log 2>&1 + printf 'clean build-dir: make Clean ... ' 2>&1 + $MAKE Clean >> make.log 2>&1 RETVAL=$? if test "x$RETVAL" != x0 ; then tail make.log - echo "makeclean: \"make CLEAN\" failed" 1>&2 + echo "makeclean: \"make Clean\" failed" 1>&2 cp make.log $CDIR"/make.log" return 1 fi @@ -275,6 +384,29 @@ fi } +run_clean() +{ + # run_clean directory + if test "x$NOCLEAN" = xt ; then + echo "run_clean skipped!" + else + ( + cd $1; + printf 'clean run-dir ... ' 2>&1 + # part of what is done after "make clean" when doing "make CLEAN" + find . -name "*.meta" -exec rm {} \; + find . -name "*.data" -exec rm {} \; + find . -name "fort.*" -exec rm {} \; + find . -type l -exec rm {} \; + rm -f $EXECUTABLE *.txt STD* *diagnostics.log datetime + rm -rf mnc_test_* + rm -f *_MIT_CE_000.opt0000 costfunction*0000 + echo successful 1>&2 + exit 0 + ) + fi +} + makedependmodel() { # makedependmodel directory @@ -306,7 +438,11 @@ if test -r Makefile ; then printf 'make ... ' 1>&2 if test "x$ADM" = x ; then - $MAKE >> make.log 2>&1 + if test "x$JOBS" = x ; then + $MAKE >> make.log 2>&1 + else + $MAKE -j $JOBS >> make.log 2>&1 + fi else $MAKE adall >> make.log 2>&1 fi @@ -333,9 +469,9 @@ code_dir=$2 BUILD_DIR=$dir/$3 CODE_DIR=$dir/$code_dir - + # These are files that should replace their counter-part when using -mpi - MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi")` + MPI_FILES=`(cd $CODE_DIR; find . -name "*_mpi" -print)` # Is this an MPI run? if test "x$MPI" = xt ; then @@ -348,20 +484,21 @@ RETVAL=$? if test "x$RETVAL" != x0 ; then if ! test -f $BUILD_DIR/$i ; then - #echo Linking $name to $i + #echo Linking $name to $i (cd $BUILD_DIR; ln -sf ../$code_dir/$i $name) fi fi done else - # NO: We undo any _mpi symbolically linked files + # NO: We undo any _mpi symbolically linked files for ii in $MPI_FILES ; do i=`echo $ii | sed 's:^\./::'` name=`echo $i | sed 's:_mpi::' ` if test -L $BUILD_DIR/$name ; then - linktarg=`(cd $BUILD_DIR; readlink $name)` - if test $linktarg = "../$code_dir/$name"_mpi ; then - #echo Un-linking $name from $linktarg + cmp $BUILD_DIR/$name "../$code_dir/$name"_mpi > /dev/null 2>&1 + RETVAL=$? + if test "x$RETVAL" = x0 ; then + #echo Un-linking $name from $linktarg rm -f $BUILD_DIR/$name fi fi @@ -372,33 +509,46 @@ linkdata() { - # linkdata flag + # linkdata run_dir input_dir_1 input_dir_2 ... # # symbolically link data files to run directory - if test "x$1" = x1 ; then + if test -d $1 ; then ( - cd $2 - if test "x$ADM" = x ; then - files=`( cd ../input ; ls -1 | grep -v CVS )` - for i in $files ; do - if test ! -d "../input/"$i ; then - ln -sf "../input/"$i $i + cd $1 ; shift + if test -r "../"$1"/eedata.mth" ; then + # found eedata.mth in 1rst input dir and it is readable + if test "x$MULTI_THREAD" = "xt" ; then + # multi-threaded test: remove symbolic link & link eedata.mth + if test -h eedata ; then rm -f eedata ; fi + if test ! -r eedata ; then + ln -sf "../"$1"/eedata.mth" eedata ; + printf 'eedata.mth ' 1>&2 fi - done - else - files=`( cd ../input ; ls -1 *.bin | grep -v CVS )` - for i in $files ; do - if test ! -d "../input/"$i ; then - ln -sf "../input/"$i $i - fi - done - files=`( cd ../input_ad ; ls -1 | grep -v CVS )` - for i in $files ; do - if test ! -d "../input_ad/"$i ; then - ln -sf "../input_ad/"$i $i - fi - done + else + # not multi-threaded test: remove eedata symbolic link + if test -h eedata ; then rm -f eedata ; fi + fi fi + prevDir='NONE' + for ldir in $* ; do + if test -d "../"$ldir -a $ldir != $prevDir ; then + printf 'ldir='${ldir} 1>&2 + files=`( cd "../"$ldir ; ls -1 | grep -v CVS )` + for i in $files ; do + if test ! -d "../"$ldir/$i ; then + if test ! -r $i ; then + printf ' '$i 1>&2 + ln -sf "../"$ldir"/"$i $i + fi + fi + done + if test -x "../"$ldir"/"prepare_run ; then + "../"$ldir"/"prepare_run + fi + printf ' ; ' 1>&2 + fi + prevDir=$ldir + done ) fi } @@ -411,22 +561,58 @@ # (where "$COMMAND" is relative to "directory") ( cd $1 - printf 'runmodel ... ' 1>&2 + printf 'runmodel in %s ...' $1 1>&2 # make output.txt - $COMMAND >> run.log 2>&1 - RETVAL=$? - if test "x$RETVAL" = x0 ; then - echo successful 1>&2 - if test "x$ADM" = x ; then - cp output.txt $CDIR"/output.txt" + echo + if test -L $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then + diff -q $EXECUTABLE "../"$builddir"/"$EXECUTABLE > /dev/null 2>&1 + outD=$? ; if test $outD != 0 ; then rm -f $EXECUTABLE ; rm -f run.log ; fi + fi + if test ! -x $EXECUTABLE -a -x "../"$builddir"/"$EXECUTABLE ; then + echo " link" $EXECUTABLE "from dir ../"$builddir > run.log_00 + ln -sf "../"$builddir"/"$EXECUTABLE . + fi + if test ! -x $EXECUTABLE ; then + rm -f run.log ; touch run.log + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + echo " no executable:" $EXECUTABLE >> run.log + RETVAL=8 + ENDVAL=-1 + else + if test ! -f $OUTPUTFILE -o $OUTPUTFILE -ot $EXECUTABLE ; then + # output do not exist or is older than executable: + rm -f run.log ; touch run.log + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + ( eval $COMMAND ) >> run.log 2>&1 + RETVAL=$? else - cp output.txt_adm $CDIR"/output.txt_adm" + RETVAL=0 + if test -f run.log ; then + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1 + else + touch run.log + if test -f run.log_00 ; then cat run.log_00 >> run.log ; fi + echo "---------->> $OUTPUTFILE is up to date " >> run.log 2>&1 + echo " no previous run.log: assume NORMAL END" >> run.log 2>&1 + fi fi + ENDVAL=`cat run.log | grep -v 'ABNORMAL END' | grep -c 'NORMAL END'` + fi + rm -f run.log_00 + #if test "x$RETVAL" = x0 ; then + if [ $RETVAL -eq 0 -a $ENDVAL -gt 0 ] ; then + tail run.log + echo successful 1>&2 + # === Reduce the size of the testing emails! + #cp $OUTPUTFILE $CDIR"/"$OUTPUTFILE + if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi return 0 else tail run.log - echo failed 1>&2 + echo failed '(run:' $RETVAL ' end:' $ENDVAL ')' 1>&2 cp run.log $CDIR"/run.log" + if test -s STDERR.0000 ; then cp STDERR.0000 $CDIR"/STDERR.0000" ; fi return 1 fi ) @@ -436,44 +622,48 @@ { # create codelet for comparing model output - printf "creating the comparison code... " - cat > tmp_cmpnum.c < tr_cmpnum.c < #include int main( int argc, char** argv ) { - int linnum,best,lncnt; - double a,b,diff; - best = -16; + int linnum,cmplin,best,lncnt; + double a,b,abave,relerr; + best = -22; lncnt = 0; - while( 1 & (lncnt+=1) < 999 ) { + while( 1 & ( (lncnt+=1) < 999 ) ) { scanf("%d", &linnum); if (linnum == -1) break; scanf("%lf", &a); scanf("%lf", &b); - diff = 0.5*(fabs(a)+fabs(b)); - if (diff > 1.e-12) { - diff=fabs(a-b)/diff; - if (diff > 0.0) { - linnum = (int)log10(diff); - best = (best > linnum) ? best : linnum; - } - else { - if (best == -16 && diff != 0) best = -22; + abave = 0.5*(fabs(a)+fabs(b)); + if ( abave == abave ) { + if (abave > 0.0) { + relerr=fabs(a-b)/abave; + if (relerr > 0.0) { cmplin = (int)rint(log10(relerr)); } + else { cmplin = -16 ; } + best = (best > cmplin) ? best : cmplin; } + else { cmplin = -22 ; } + /* printf("%d ; %lf ; %lf\n",cmplin,a,b); */ } - } + else { + /* printf("%lf ; %lf ; %lf\n",abave,a,b); */ + break; } } if (lncnt == 999) best=-29; + if (linnum != -1) best=-99; printf("%d\n", -best); return 0; } EOF - cc -o tmp_cmpnum tmp_cmpnum.c -lm + $CC -o tr_cmpnum tr_cmpnum.c -lm - if [ -x ./tmp_cmpnum ]; then + if [ -x ./tr_cmpnum ]; then echo "OK" return 0 else echo - echo "ERROR: failed to compile comparison code" + echo "ERROR: failed to compile comparison code -- please specify" + echo " a C compiler using the CC environment variable." exit 1 fi } @@ -485,12 +675,21 @@ nm=$1 printf '%s %s %s %s' $2 $3 $4 $5 shift; shift; shift; shift; shift; - printf '%3s' $@ + listPrt=$@ + listRes=`echo $listPrt | sed 's/>//' | sed 's///' | sed 's/<.*//' | awk '{print $1}'` + printf '%3s' $listPrt +# line below does not work on hp-ux_ia64 : do those substitutions later on +# printf '%3s' $listPrt | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < / output.txt_adm 2>&1" -fi - +OUTPUTFILE=$ref_outp if test "x$COMMAND" = x ; then - COMMAND="make output.txt" + COMMAND="./$EXECUTABLE > $OUTPUTFILE" fi +if test "x$MPI" = xt ; then + OUTPUTFILE="STDOUT.0000" +fi + +echo "OK (COMMAND= $COMMAND )" -echo "OK" +# set the Default List of output variables to be checked: +# (use default or load experiment-specific list from file "tr_checklist") +# content : 1rst = main variable used to decide if it pass or FAIL +# others = number of matching digits to be printed in summary.txt +if test "x$ADM" = x ; then + DEF_CHECK_LIST='PS PS T+ S+ U+ V+ pt1+ pt2+ pt3+ pt4+ pt5+' + EMPTY_RESULTS='.. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. ..' + LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'` + ii=`echo $EMPTY_RESULTS | awk '{print NF}'` + EMPTY_RESULTS=$EMPTY_RESULTS`expr $LEN_CHECK_LIST - $ii | awk 'BEGIN{FS=":"}{for(i=1;i<=$1;i++){printf " ."}}'` +else + DEF_CHECK_LIST='Grad Cost Grad' + EMPTY_RESULTS='.. ..' + LEN_CHECK_LIST=`echo $DEF_CHECK_LIST | sed 's/ [a-zA-Z0-9]*+/&mn &mx &av &sd/g' | awk '{print NF-1}'` +fi # create the FORTRAN comparison code createcodelet -# build the mpack utility -if test "x$ADDRESSES" = xNONE -o "x$ADDRESSES" = x ; then +# build the mpack utility (if ADDRESSES = NONE, do it to test the build) +if test "x$ADDRESSES" = x ; then echo "skipping mpack build" else build_mpack fi # Create a uniquely named directory to store results +CMDLINE=$0 +for xx in "$@" ; do CMDLINE="$CMDLINE '$xx'" ; done MACH=`hostname` UNAMEA=`uname -a` DATE=`date +%Y%m%d` BASE="tr_"$MACH"_"$DATE"_" +if test "x$OUTDIR" != x ; then + BASE="tr_"$OUTDIR"_"$DATE"_" +fi DNUM=0 DRESULTS="$BASE$DNUM" while test -e $DRESULTS ; do @@ -705,9 +1009,10 @@ exit 1 fi SUMMARY="$DRESULTS/summary.txt" -printf "Start time: " >> $SUMMARY start_date=`date` echo $start_date > $SUMMARY +echo 'run:' $CMDLINE >> $SUMMARY +echo 'on :' $UNAMEA >> $SUMMARY of_path= if test "x$OPTFILE" != xNONE ; then @@ -737,17 +1042,35 @@ echo echo >> $SUMMARY if test "x$ADM" = x ; then - cat << EOF | tee -a $SUMMARY - T S U V -G D M c m s m s m s m s -E p a R g m m e . m m e . m m e . m m e . -N n k u 2 i a a d i a a d i a a d i a a d -2 d e n d n x n . n x n . n x n . n x n . - -EOF + if [ $MATCH_CRIT -lt 10 ] ; + then line_0="default "$MATCH_CRIT ; + else line_0="default "$MATCH_CRIT ; fi + line_0="$line_0 ----T----- ----S----- ----U----- ----V-----" +# line_0=" ----T----- ----S----- ----U----- ----V-----" + line_1="G D M c m s m s m s m s" + line_2="E p a R g m m e . m m e . m m e . m m e ." + line_3="N n k u 2 i a a d i a a d i a a d i a a d" + line_4="2 d e n d n x n . n x n . n x n . n x n ." + for ii in $PTRACERS_NUM ; do + line_0="$line_0 --PTR 0"$ii"--" + line_1="$line_1 m s" + line_2="$line_2 m m e ." + line_3="$line_3 i a a d" + line_4="$line_4 n x n ." + done + echo "$line_0" | tee -a $SUMMARY + echo "$line_1" | tee -a $SUMMARY + echo "$line_2" | tee -a $SUMMARY + echo "$line_3" | tee -a $SUMMARY + echo "$line_4" | tee -a $SUMMARY + echo " " | tee -a $SUMMARY else echo "ADJOINT=true" >> $SUMMARY echo >> $SUMMARY + if [ $MATCH_CRIT -lt 10 ] ; + then line_0="default "$MATCH_CRIT ; + else line_0="default "$MATCH_CRIT ; fi + echo "$line_0" | tee -a $SUMMARY cat << EOF | tee -a $SUMMARY G D M C G E p a R o r @@ -760,74 +1083,82 @@ # ...and each test directory... for dir in $TESTDIRS ; do + # set builddir & rundir: + builddir="build" + if test ! -d $dir/$builddir ; then mkdir $dir/$builddir ; fi + rundir="run" + if test ! -d $dir/$rundir ; then + rundir=$builddir + fi + CODE_DIR=$dir/$code_dir + BUILD_DIR=$dir/$builddir + # Cleanup only! if test "x$CLEANUP" = xt ; then - if test -r $dir/build/Makefile ; then - ( cd $dir/build ; make CLEAN ) + if test -r $BUILD_DIR/Makefile ; then + echo ' ------ clean dir:' $dir/$builddir + ( cd $BUILD_DIR ; make CLEAN ) fi - if test -r $dir/input/Makefile ; then - ( cd $dir/input ; make CLEAN ) + if test -d $dir/$rundir/CVS ; then + echo ' ------ clean dir:' $dir/$rundir + run_clean $dir/$rundir fi + ( + cd $dir + rm -rf tr_run.* + ) continue fi # Verify that the testdir exists and contains previous # results in the correct location--or skip this directory! - fout= - if test "x$ADM" = x ; then - fout=$dir"/results/output.txt" - else - fout=$dir"/results_ad/output.txt_adm" - fi + fout=$dir"/results/"$ref_outp if test ! -r $fout ; then echo "can't read \"$fout\" -- skipping $dir" continue fi - builddir="input" - rundir="input" - use_seperate_build=0 - if test -d $dir/build -a -r $dir/build ; then - builddir="build" - rundir="build" - use_seperate_build=1 - linkdata $use_seperate_build $dir/$rundir - fi - - # Check whether there are "extra runs" for this testdir - extra_runs= - if test "x$ADM" = x -a "x$use_seperate_build" = x1 ; then - ex_run_dirs=`( cd $dir ; echo input.* )` - echo "ex_run_dirs='$ex_run_dirs'" - for exd in $ex_run_dirs ; do - name=`echo $exd | sed -e 's/input.//g'` - outf="$dir/results/output.txt.$name" - if test -f $outf -a -r $outf ; then - extra_runs="$extra_runs $name" - fi - done - fi - - if test "x$ADM" = x ; then - code_dir=code - CODE_DIR=$dir/code - else - code_dir=code_ad - CODE_DIR=$dir/code_ad - fi - BUILD_DIR=$dir/$builddir + # Check for specific files for particular type of run if test ! -r $CODE_DIR"/SIZE.h_mpi" -a "x$MPI" = "xt" ; then echo "can't find \"$CODE_DIR/SIZE.h_mpi\" -- skipping $dir" continue fi + if test ! -r $dir"/input/eedata.mth" -a "x$MULTI_THREAD" = "xt" ; then + echo "can't find \"$dir/input/eedata.mth\" -- skipping $dir" + continue + fi + + # Check whether there are "extra runs" for this testdir + extra_runs= + ex_run_dirs=`( cd $dir ; echo $inputdir.* )` + #echo "ex_run_dirs='$ex_run_dirs'" + for exd in $ex_run_dirs ; do + name=`echo $exd | sed -e "s/$inputdir\.//"` + refExOut=`echo $ref_outp | sed "s/\./.${name}./"` + outf="$dir/results/$refExOut" + if test -f $outf -a -r $outf ; then + if test "x$MULTI_THREAD" = "xt" ; then + if test -r $dir"/"$exd"/eedata.mth" ; then + extra_runs="$extra_runs $name" + #else echo $dir"/"$exd"/eedata.mth: not found" + fi + else + extra_runs="$extra_runs $name" + fi + fi + done echo "-------------------------------------------------------------------------------" echo - echo "Experiment: $dir" + if test "x$extra_runs" = "x" ; then + echo "Experiment: $dir" + else + echo "Experiment: $dir ; extra_runs=$extra_runs" + fi echo unset genmake makedepend make run - results='-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --' + results=$EMPTY_RESULTS # Create an output dir for each OPTFILE/tdir combination rel_CDIR=$DRESULTS"/"$dir @@ -835,74 +1166,75 @@ CDIR=`pwd`"/$rel_CDIR" if test "x$CLEANUP" = xt ; then - makeclean $dir/$builddir + echo '====>>> this is to check that we never go through this part <<< ===' + makeclean $dir/$builddir \ + && run_clean $dir/$rundir else genmakemodel $dir/$builddir && genmake=Y \ && makeclean $dir/$builddir \ + && run_clean $dir/$rundir \ && symlink_mpifiles $dir $code_dir $builddir \ && makedependmodel $dir/$builddir && makedepend=Y \ && makemodel $dir/$builddir && make=Y \ - && linkdata $use_seperate_build $dir/$rundir \ + && linkdata $dir/$rundir $inputdir input \ && runmodel $dir/$rundir && run=Y \ - && results=`testoutput $dir $rundir` + && results=`testoutput_run $dir $rundir $ref_outp` fi echo - if test "x$ADM" = x ; then fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` echo - echo "$fres" >> $SUMMARY - echo "fresults='$fres'" > $CDIR"/summary.txt" + echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY + touch $CDIR"/summary.txt" + echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt" echo "MACH='$MACH'" >> $CDIR"/summary.txt" echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" echo "DATE='$DATE'" >> $CDIR"/summary.txt" echo "tdir='$dir'" >> $CDIR"/summary.txt" + if test "x$ADM" = xt ; then + head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt" + grep -A3 'Seconds in section "ALL' $dir/$rundir/$OUTPUTFILE \ + >> $CDIR"/summary.txt" + fi - OLD_COMMAND=$COMMAND - COMMAND="./mitgcmuv > output.txt" for ex in $extra_runs ; do - test -e "$dir/tr_run" && rm -rf "$dir/tr_run" - mkdir "$dir/tr_run" - ( cd "$dir/input" ; links=`ls -1 | grep -v CVS` ) - ( - cd "$dir/tr_run" - for i in $links; do - ln -s ../input/$i $i - done - ) - ( cd "$dir/input.$ex" ; links=`ls -1 | grep -v CVS` ) - ( - cd "$dir/tr_run" - for i in $links; do - test -e $i && rm -f $i - ln -s ../input.$ex/$i $i - done - ln -s ../$builddir/mitgcmuv mitgcmuv - ) - runmodel $dir/tr_run && run=Y \ - && results=`testoutput $dir $rundir "."$ex` - fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}` + unset run + results=$EMPTY_RESULTS + # reference output file + refExOut=`echo $ref_outp | sed "s/\./.${ex}./g"` + # Create an output dir for each OPTFILE/tdir.ex combination + rel_CDIR=$DRESULTS"/"$dir"."$ex + mkdir $rel_CDIR + CDIR=`pwd`"/$rel_CDIR" + test ! -e "$dir/tr_run.$ex" && mkdir "$dir/tr_run.$ex" + run_clean $dir/tr_run.$ex + linkdata $dir/tr_run.$ex $inputdir.$ex $inputdir input + runmodel $dir/tr_run.$ex && run=Y \ + && results=`testoutput_run $dir tr_run.$ex $refExOut` fres=`formatresults $dir ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N} $results` fres="$fres.$ex" echo - echo "$fres" >> $SUMMARY - echo "fresults='$fres'" > $CDIR"/summary.txt" + echo "$fres" | sed 's/ 99/ --/g' | sed 's/ > />/' | sed 's/ < /> $SUMMARY + touch $CDIR"/summary.txt" + echo "fresults='$fres'" | sed 's/ 99/ --/g' >> $CDIR"/summary.txt" echo "MACH='$MACH'" >> $CDIR"/summary.txt" echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" echo "DATE='$DATE'" >> $CDIR"/summary.txt" - echo "tdir='$dir'" >> $CDIR"/summary.txt" + echo "tdir='$dir.$ex'" >> $CDIR"/summary.txt" + if test "x$ADM" = xt ; then + head -1 $dir/$builddir/taf_ad.log >> $CDIR"/summary.txt" + grep -A3 'Seconds in section "ALL' $dir/tr_run.$ex/$OUTPUTFILE \ + >> $CDIR"/summary.txt" + fi + if test "x$POSTCLEAN" = xt ; then + run_clean $dir/tr_run.$ex + fi done - COMMAND=$OLD_COMMAND - else - fres=`printf '%s %s %s %s' ${genmake:-N} ${makedepend:-N} ${make:-N} ${run:-N}` - fres=$fres"$results $dir" - echo - echo "$fres" >> $SUMMARY - echo "fresults='$fres'" > $CDIR"/summary.txt" - echo "MACH='$MACH'" >> $CDIR"/summary.txt" - echo "UNAMEA='$UNAMEA'" >> $CDIR"/summary.txt" - echo "DATE='$DATE'" >> $CDIR"/summary.txt" - echo "tdir='$dir'" >> $CDIR"/summary.txt" + + #postclean $dir/$builddir + if test "x$POSTCLEAN" = xt ; then + makeclean $dir/$builddir \ + && run_clean $dir/$rundir fi echo "-------------------------------------------------------------------------------" @@ -910,7 +1242,7 @@ done printf "Start time: " >> $SUMMARY -echo $start_date >> $SUMMARY +echo "$start_date" >> $SUMMARY printf "End time: " >> $SUMMARY date >> $SUMMARY @@ -922,7 +1254,7 @@ if test "x$HAVE_MPACK" = xt ; then tar -cf $DRESULTS".tar" $DRESULTS > /dev/null 2>&1 \ && gzip $DRESULTS".tar" \ - && $MPACK -s MITgcm-test -m 1500000 $DRESULTS".tar.gz" $ADDRESSES + && $MPACK -s MITgcm-test -m 3555000 $DRESULTS".tar.gz" $ADDRESSES RETVAL=$? if test "x$RETVAL" != x0 ; then echo @@ -941,14 +1273,17 @@ fi fi -# rm -f tmp_cmpnum.f a.out -rm -f tmp_cmpnum.c tmp_cmpnum +rm -f tr_cmpnum.c tr_cmpnum if test "x$CLEANUP" != xt ; then - cat $SUMMARY + cat $SUMMARY | sed 's/ \. \. \. \. \. \. \. \. \. \. \. \. //' if test -e tr_out.txt ; then mv tr_out.txt tr_out.txt.old fi - cat $SUMMARY > tr_out.txt + cat $SUMMARY | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' > tr_out.txt +fi + +if test "x$DELDIR" = xt ; then + rm -rf $DRESULTS fi