#! /usr/bin/env bash

# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.59 2024/02/07 20:18:24 jmc Exp $
#
#  The purpose of this script is to compare
#  the latest output with the previous one (from the same platform with
#    same optfile)

usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-h|-help)            print usage"
    echo "  (-v|-verbose)         verbose mode"
    echo "  (-t|-test4update)     do nothing if previous output newer than"
    echo "                         emails-processing log-file '$PRC_MAILS'"
    echo "  (-l |-list )MACHINES  check platforms from this list"
    echo "                         [def=\"$MACHINES\"]"
    echo "  (-d |-day  )FIRSTDAY  select output from day=\"YYYYMMDD\""
    echo "                         (argument of \"date -d\" is allowed)"
    echo "                         [def=\"$FIRSTDAY\"]"
    echo "  (-u |-upto )LASTDAY   until last day=\"YYYYMMDD\" (=selected period)"
    echo "                         [def=\"FIRSTDAY\"]"
    echo "  (-b |-before)BEFORE   compare with output from before \"YYYYMMDD\""
    echo "                         (=previous period) [def=\"FIRSTDAY\"]"
    echo "  (-o |-outp )OUTPFIL   output file (-1 removed when exit)"
    echo "                         [def=\"$OUTPFIL\"]"
    echo "  (-n |-nldf )NBLDIFF   max number of diff lines to echo"
    echo "                         [def=\"$NBLDIFF\"]"
    echo "  (-a |-addr )ADDRESS   send summary to mail ADDRESS list"
    echo "                         [def=\"$ADDRESS\"]"
    echo
    exit 1
}

#  create list of results dir for one platform:
make_resdir_list()
{
      if test $monthDir == 1 ; then
	dir_list=`grep '.._'$mname $TMP.dir_all`
      else
	dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/??_*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )`
	#echo "dir_list='$dir_list'"
      fi
	for i in $dir_list ; do

	    dir=$INDIR"/"$i
	    #--- get the opfile used to run testreport
	    OPTFILE=
	    if test -r $dir/summary.txt ; then
		comm=`grep 'OPTFILE=' $dir/summary.txt`
		eval $comm
	    fi
	    if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then
		comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null`
		eval $comm
	    fi
	    if test "x$OPTFILE" = x ; then
		comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1`
		comm=${comm##*#}
		eval $comm
	    fi
	    if test "x$OPTFILE" = x ; then
		OPTFILE="not_explicitly_specified"
	    else
		OPTFILE=${OPTFILE##*/}
	    fi
	    #--- case select test run by 1 user: get user who did run testreport
	    if test "x$sUser" != x ; then
		tUser=
		if test -f $dir/genmake_state ; then
		   nn=`sed -n '/^# executed by:/=' $dir/genmake_state`
		   if test "x$nn" != x ; then
		     nn=`expr $nn + 1`
		     tUser=`sed -n "$nn s/^# *// p" $dir/genmake_state | sed 's/@.*$//'`
		   fi
		fi
		if test "x$tUser" = "x" ; then
		   mkfile=`ls $dir/*/Makefile* 2>/dev/null | head -1`
		   if test "x$mkfile" = 'x' ; then
		     echo "no Makefile => discard test from: $i" | tee -a $OUTPFIL
		     continue
		   fi
		   nn=`sed -n '/^# executed by:/=' $mkfile`
		   if test "x$nn" != x ; then
		     nn=`expr $nn + 1`
		     tUser=`sed -n "$nn s/^# *// p" $mkfile | sed 's/@.*$//'`
		   fi
		fi
		if test "x$tUser" = x ; then
		     echo "no User found => discard test from: $i" | tee -a $OUTPFIL
		     continue
		fi
		if test $tUser != $sUser ; then
		   if test $dBug = 't' ; then
			echo "  discard test from: $i" | tee -a $OUTPFIL
		   fi
		   continue
		fi
	    fi

	    ADJOINT=0
	    TANGLIN=0
	    OPENAD=0
	    TAPENAD=0
	    RESTART=0
	    Special=0
	    EXTRA=
	    FAST=0
	    DVLP=0
	    MPI=0
	    MTH=0
	    UR4=0
	    if test -r $dir/summary.txt ; then
		TAPENAD=`grep -c ' generated by Tapenade' $dir/summary.txt`
		ADJOINT=`grep -c -i '^ADJOINT' $dir/summary.txt`
		TANGLIN=`grep -c -i '^TANGLIN' $dir/summary.txt`
		if test "x$ADJOINT" = x1 ; then
		  OPENAD=`grep -c '^Adjoint .* OpenAD' $dir/summary.txt`
		fi
		RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt`
		#- special label for "darwin" test
		Special=`echo $i | grep -c '\-darwin[0-9]'`
		if test $Special != 0 ; then
		    Special=`echo $i | sed -e "s/.*$mname/$mname/" -e "s/_/ /g" -e "s/-/ /g"`
		    if test $dBug = 't' ; then echo -n " Special='$Special' ;" | tee -a $OUTPFIL ;  fi
		    Special=`echo $Special | awk '{print $2}' | sed 's/darwin//'`
		fi
		comm=`grep '^run: .*testreport.* ' $dir/summary.txt`
		EXTRA=`echo "$comm" | grep " -*-tdir\>" | sed -e "s/^.* -*-tdir\>//" -e "s/ -.*$//"`
		if test "x$EXTRA" = x ; then
		    EXTRA=`echo "$comm" | grep " -*-t\>" | sed -e "s/^.*-*-t\>//" -e "s/ -.*$//"`
		fi
		if test "x$EXTRA" = x ; then EXTRA=0 ; else
		    #if test $dBug = 't' ; then echo -n " EXTRA='$EXTRA' ;" | tee -a $OUTPFIL ;  fi
		    nn0=`echo $EXTRA | sed "s/ *' *//g" | wc -w`
		    nn1=`echo $EXTRA | sed "s/ *' *//g" | tr ' ' '\n' | grep -c "\<darwin_"`
		    nn2=`echo $EXTRA | sed "s/ *' *//g" | tr ' ' '\n' | grep -c "\<monod_"`
		    EXTRA=9
		    if [ $nn1 -ge 2 ] ; then EXTRA=1 ; fi
		    if [ $nn2 -ge 2 ] ; then EXTRA=2 ; fi
		    #echo " : nn0=$nn0 ; nn1=$nn1 ; nn2=$nn2"
		fi
		if test $dBug = 't' ; then if [ $Special -ne 0 -o $EXTRA -ne 0 ] ; then
		    echo "	Special='$Special' ; EXTRA='$EXTRA'" | tee -a $OUTPFIL
		fi ; fi
		FAST=`echo "$comm" | grep -c " -*-fast\>"`
		if test "x$FAST" = x0 ; then
		   FAST=`echo "$comm" | grep -c " '*-noieee'*"`
		fi
		DVLP=`echo "$comm" | grep -c " -*-devel\>"`
		MPI=`echo "$comm" | grep -c " -*-mpi\>"`
		if test "x$MPI" = x0 ; then
		   MPI=`echo "$comm" | grep -c " -*-MPI\>"`
		fi
		MTH=`echo "$comm" | grep -c " -*-mth\>"`
		UR4=`echo "$comm" | grep -c " -*-use_r4\>"`
		if test "x$UR4" = x0 ; then
		   UR4=`echo "$comm" | grep -c " -*-ur4\>"`
		fi
	    else EXTRA=0 ; fi
	    if test "x$ADJOINT" = x1 ; then
		kind="adm-TAF" ; order=0
		if test "x$TAPENAD" = x1 ; then
		   kind="adm-TAP" ; order=2
		elif test "x$OPENAD" = x1 ; then
		   kind="adm-OAD" ; order=4
		fi
	    elif test "x$TANGLIN" = x1 ; then
		kind="tlm-TAF" ; order=1
		if test "x$TAPENAD" = x1 ; then kind="tlm-TAP" ; order=3 ; fi
	    elif test "x$RESTART" = x0 ; then
		kind="forward" ; order=6
	    else
		kind="restart" ; order=7
	    fi
	    if test "x$UR4" = x1 ; then
		OPTFILE="${OPTFILE}.use_r4"
	    fi
	    if test "x$MPI" = x1 ; then
		yy=`echo $OPTFILE | grep -c '+mpi'`
		if test $yy = 0 ; then yy=`echo $OPTFILE | grep -c '+impi'` ; fi
		if test $yy = 0 ; then OPTFILE="${OPTFILE}+mpi"
				else order=`expr $order + 200` ; fi
	    fi
	    if test "x$MTH" = x1 ; then
		yy=`echo $OPTFILE | grep -c '+mth$'`
		if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi
	    fi
	    if test "x$FAST" = x1 ; then
		OPTFILE="${OPTFILE}.fast"
	    fi
	    if test "x$DVLP" = x1 ; then
		OPTFILE="${OPTFILE}.dvlp"
	    fi
	    if [ $Special -eq 0 ] ; then
		order=`expr $order + 10 \* $EXTRA`
	    else
		order=`expr $order + 10 \* $Special`
	    fi
	    order=`printf '%3.3i' $order`
	    day=`echo $i   | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'`
	    ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'`
	    day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'`

	    if [ $day -lt $BEFORE ] ; then
		echo "$OPTFILE$order $day $OPTFILE $kind $i" >> $TMP.plist
	    elif [ $day -ge $FIRSTDAY -a $day -le $LASTDAY ] ; then
		echo "$OPTFILE$order $day $OPTFILE $kind $i" >> $TMP.slist
	    else
		if test $dBug = 't' ; then
		   echo "discard day='$day' from: $i" | tee -a $OUTPFIL
		fi
	    fi

	done
	if test $dBug = 't' ; then
	  if test $monthDir == 1 ; then
	    echo "---- current list (n=$monthDir) :" | tee -a $OUTPFIL
		cat $TMP.slist | tee -a $OUTPFIL
	  fi
	  num1=`wc -l $TMP.slist | awk '{print $1}'`
	  if test $num1 -gt 0 ; then
	  echo   ".... previous list (n=$monthDir):" | tee -a $OUTPFIL
		cat $TMP.plist | tee -a $OUTPFIL | head -20
	  fi
	  echo "----" | tee -a $OUTPFIL
	fi
}

#--------------------------------------------------------------------------
# here starts the sequential part of the script:

#CURR_PER=`date +%Y`"_"`date +%m`
CURR_DAY=`date +%Y%m%d`
PRC_MAILS='prc_emails_'`date +%m%d`
# defaults
MACHINES='_All_'
FIRSTDAY=$CURR_DAY
LASTDAY="30000000"
BEFORE="30000000"
OUTPFIL=`basename $0`'.log'
NBLDIFF=5
ADDRESS='none'
MAILCMD="mail"
dBug=f
t4update=0
sTime=`date`
#- to get case insensitive "ls" (and order of tested experiments)
export LC_ALL="en_US.UTF-8"

#  Parse options
ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
	eval "$ac_prev=\$ac_option"
	ac_prev=
	continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
    case $ac_option in
	
	-help | --help | -h | --h) usage ;;
	-verbose | --verbose | -v | --v) dBug=t ;;
	-test4update | --test4update | -t | --t) t4update=1 ;;
	
	-list  | --list  | -l | --l) ac_prev=MACHINES ;;
	-list=* | --list=*) MACHINES=$ac_optarg ;;

	-day  | --day  | -d | --d) ac_prev=FIRSTDAY ;;
	-day=* | --day=*) FIRSTDAY=$ac_optarg ;;

	-upto | --upto | -u | --u) ac_prev=LASTDAY ;;
	-upto=* | --upto=*) LASTDAY=$ac_optarg ;;

	-before | --before | -b | --b) ac_prev=BEFORE ;;
	-before=* | --before=*) BEFORE=$ac_optarg ;;

	-outp | --outp | -o | --o) ac_prev=OUTPFIL ;;
	-outp=* | --outp=*) OUTPFIL=$ac_optarg ;;

	-nldf | --nldf | -n | --n) ac_prev=NBLDIFF ;;
	-nldf=* | --nldf=*) NBLDIFF=$ac_optarg ;;

	-addr | --addr | -a | --a) ac_prev=ADDRESS ;;
	-addr=* | --addr=*) ADDRESS=$ac_optarg ;;

	*) echo "Error: don't understand argument \"$ac_option\""
	   usage ;;
     esac

done

#TMP=./chkoutp_$$
#- try to put temporary files in system-local /tmp dir
TMP=/tmp/chkoutp_$$
touch $TMP ; retVal=$?
if [ $retVal -eq 0 ] ; then
  if test ! -r $TMP ; then TMP=./chkoutp_$$ ; fi
else
  TMP=./chkoutp_$$
fi
rm -f $TMP
if test $dBug = 't' ; then echo "temp files: $TMP" ; fi

#-- test FIRSTDAY content ; interpret as "date -d" arg. if not YYYYMMDD
ttt=`echo "y$FIRSTDAY" | sed 's/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]/z/'`
if test "x$ttt" != 'xyz' ; then
   FIRSTDAY=`date -d "$FIRSTDAY" +%Y%m%d`
fi
ttt=`echo "y$FIRSTDAY" | sed 's/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]/z/'`
if test "x$ttt" != 'xyz' ; then
  echo "Error: FIRSTDAY='$FIRSDAY' not recognized"
  exit 2
fi
if test $LASTDAY = "30000000" ; then LASTDAY=$FIRSTDAY ; fi
if test  $BEFORE = "30000000" ; then  BEFORE=$FIRSTDAY ; fi
CMDLINE=$0
for xx in "$@" ; do nw=`echo $xx | wc -w`
  if test $nw = '1' ; then CMDLINE="$CMDLINE $xx"
                      else CMDLINE="$CMDLINE '$xx'" ; fi
done

PERIOD=`echo $LASTDAY | sed 's/[0-9][0-9]$//'`
PP=`expr $PERIOD - 1`
PM=`echo $PP | sed 's/^[0-9][0-9][0-9][0-9]//'`
PY=`echo $PP | sed 's/[0-9][0-9]$//'`
if test $PM == '00' ; then PM='12' ; PY=`expr $PY - 1` ; fi
PREV_P="${PY}_${PM}";
PERIOD=`echo $PERIOD | sed 's/^[0-9][0-9][0-9][0-9]/&_/'`

#INDIR="/net/zany.mit.edu/data/ORWELL/export-9/mitgcm-testing/results/$PERIOD"
INDIR="/u/u0/httpd/html/testing/results/$PERIOD"
#INDIR="$HOME/mitgcm/test_web/results/$PERIOD"

#- a short summary of this checking :
OUTPSUM=`echo $LASTDAY | sed 's/^20../_/'`
OUTPSUM=`basename $0`"$OUTPSUM.txt"

if test $OUTPFIL = '-1' ; then
  OUTPFIL='TTT.'$$
fi
if test $t4update = 1 ; then
  if test -f $PRC_MAILS -a -f $OUTPSUM -a $FIRSTDAY = $CURR_DAY ; then
    #- if update-log-file is older than previous output summary, just return
    if test $PRC_MAILS -ot $OUTPSUM ; then
      echo ' '$CMDLINE | tee -a $OUTPSUM
      echo " t4update: skip "`basename $0`" ( $PRC_MAILS older than $OUTPSUM )" | tee -a $OUTPSUM
      echo " End  time:  "`date` | tee -a $OUTPSUM
      exit 0
    else
      if test -e $OUTPFIL ; then mv -f $OUTPFIL $OUTPFIL'_bak' ; fi
      echo " t4update: run "`basename $0`" ( $PRC_MAILS newer than $OUTPSUM )" | tee $OUTPFIL
    fi
  elif test -f $OUTPSUM -a $FIRSTDAY = $CURR_DAY ; then
      echo ' '$CMDLINE | tee -a $OUTPSUM
      echo " t4update: skip "`basename $0`" ( $PRC_MAILS missing )" | tee -a $OUTPSUM
      echo " End  time:  "`date` | tee -a $OUTPSUM
      exit 0
  else
      if test -e $OUTPFIL ; then mv -f $OUTPFIL $OUTPFIL'_bak' ; fi
      echo " t4update ignored (not current day or $OUTPSUM missing)" | tee $OUTPFIL
  fi
elif test -e $OUTPFIL ; then
  mv -f $OUTPFIL $OUTPFIL'_bak'
  touch $OUTPFIL
fi
if test -e $OUTPSUM ; then mv -f $OUTPSUM $OUTPSUM'_bak' ; fi
echo ' '$CMDLINE > $OUTPSUM

echo "CMDLINE='$CMDLINE'" >> $OUTPFIL
echo "PERIOD='$PERIOD' , PREV_P='$PREV_P'" >> $OUTPFIL
echo "INDIR='$INDIR'" >> $OUTPFIL
echo "Checking latest output from $FIRSTDAY until $LASTDAY" >> $OUTPFIL
echo "  versus latest from before $BEFORE" >> $OUTPFIL

if test $dBug = 't' ; then
  echo "PERIOD='$PERIOD' , PREV_P='$PREV_P'"
  echo "FIRSTDAY='$FIRSTDAY' , LASTDAY='$LASTDAY' , BEFORE='$BEFORE'"
fi
if test -d $INDIR ; then
  dirP=`echo $INDIR | sed "s/$PERIOD/$PREV_P/"`
  if test -d $dirP ; then
    listNum='1 2'
  else
    echo "$dirP is not a directory"
    listNum='1'
  fi
else
  echo "$INDIR is not a directory"
  exit 3
fi

if test $FIRSTDAY = $LASTDAY ; then
  echo -n "Checking latest output from $FIRSTDAY" | tee -a $OUTPSUM
else
  echo -n "Checking latest output from $FIRSTDAY until $LASTDAY" | tee -a $OUTPSUM
fi
if test $FIRSTDAY = $BEFORE ; then
  echo " versus latest prior to this period" | tee -a $OUTPSUM
else
  echo " versus latest from before $BEFORE" | tee -a $OUTPSUM
fi
echo "    Machine   Tot. Nb.   no    Nb with  Nb diff" >> $OUTPSUM
echo "     name     checked   comp    Diff    lines"  >> $OUTPSUM

( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > $TMP.dir_all

if test "x$MACHINES" = "x_All_" ; then
  MACHINES="villon batsi engaging1 engaging- svante glacier"
  MACHINES="$MACHINES albedo pleiades archer"

  MALL=`cat $TMP.dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq`
  for madd in $MALL ; do
    present=0
    for m in $MACHINES ; do
	mm=`echo $m | sed 's/\./ /g' | awk '{print $1}'`
	echo $madd | grep $mm > /dev/null 2>&1
	RETVAL=$?
	test $RETVAL = 0  &&  present=1
	continue
    done
    test $present = 0  &&  MACHINES="$MACHINES $madd"
  done
fi
#echo "MACHINES=$MACHINES" ; exit

for pname in $MACHINES ; do

    sname=`echo $pname | sed 's/-$//'`
    mname=`echo $pname | sed 's/\./ /g' | awk '{print $1}'`
    sUser=
    if test $mname != $pname ; then
      # to select results from user "sUser"
      sUser=`echo $pname | sed 's/\./ /g' | awk '{print $2}'`
    fi

    if test $dBug = 't' ; then echo "  $pname" ; fi
    echo " name='$pname'" >> $OUTPFIL
    ctot=0 ; cmis=0 ; cdif=0

    # put results from selected period in "TMP.slist",
    #     results from previous period in "TMP.plist"
    rm -f $TMP.plist $TMP.slist ; touch $TMP.plist $TMP.slist

    #- create list of results (slist & plist) from current month directory
    monthDir=1
    make_resdir_list

    #  Do we have any data?  If so, create the latest pointer.
    num=`wc -l $TMP.slist | awk '{print $1}'`
    if test $num -gt 0 ; then

	keys=`cat $TMP.slist | cut -d " " -f 1 | sort | uniq`

	for key in $keys ; do
	  ctot=`expr $ctot + 1`
	  sline=`grep "^$key " $TMP.slist | head -1`
	  sdir=`echo $sline | cut -d " " -f 5`
	  type=`echo $sline | cut -d " " -f 4`
	  optf=`echo $sline | cut -d " " -f 3`
	  num=`grep -c "^$key" $TMP.plist`
	  if test $num = 0 -a $monthDir = 1 ; then
	  #- add results from previous month directory to "plist".
	    monthDir=2
	    make_resdir_list
	    num=`grep -c "^$key" $TMP.plist`
	  fi
	  #- discard unsafe test:
	  dd=`echo $optf | grep -c 'gfortran+mth'`
	  #if test $mname = 'lagoon' -a $dd = 1 ; then num=-1 ; fi
	  #if test $mname = 'harbor' -a $dd = 1 ; then num=-1 ; fi
	  #if test $mname = 'aces' -a $optf = 'linux_ia32_open64' ; then num=-1 ; fi
	  #if test $mname = 'aces' -a $optf = 'linux_ia32_g95' ; then num=-1 ; fi
	  dd=`echo $optf | grep -c 'bigred.*noieee'`
	  #if test $mname = 'bigred' -a $dd = 1 ; then num=-1 ; fi
	  if test $num -lt 0 ; then
	    echo "  discard $sdir : $type , of='$optf'" | tee -a $OUTPFIL
	    cmis=`expr $cmis + 1`
	  elif test $num = '0' ; then
	    echo "  dir='$sdir', type='$type', of='$optf':" | tee -a $OUTPFIL
	    echo "  --> no previous test matching key='$key'" | tee -a $OUTPFIL
	    cmis=`expr $cmis + 1`
	  else
	    pline=`grep "^$key " $TMP.plist | head -1`
	    if test $dBug = 't' ; then
		echo "s='$sline'" | tee -a $OUTPFIL
		echo "p='$pline'" | tee -a $OUTPFIL
	    fi
	    pdir=`echo $pline | cut -d " " -f 5`
	    outs="$INDIR/$sdir/summary.txt"
	    xx=`( echo 5 ; sed -n '/^  OPTFILE=/=' $outs ) | tail -1`
	    sed "1,$xx d" $outs | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \
				| grep -v '^[A-S][a-t]* time:  ' > $TMP.fs
	    outp="$INDIR/$pdir/summary.txt"
	    xx=`( echo 5 ; sed -n '/^  OPTFILE=/=' $outp ) | tail -1`
	    sed "1,$xx d" $outp | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \
				| grep -v '^[A-S][a-t]* time:  ' > $TMP.fp
	    #-- check if list of exp are the same (and in the same order)
	    sed 's/  (e=.*, w=.*)$//' $TMP.fs > $TMP.fn
	    listDs=`grep '^[YN] [YN] [YN] [YN]' $TMP.fn | awk '{print $NF}'`
	    #mv -f $TMP.fn $TMP.fs
	    rm -f $TMP.fn
	    sed 's/  (e=.*, w=.*)$//' $TMP.fp > $TMP.fn
	    listDp=`grep '^[YN] [YN] [YN] [YN]' $TMP.fn | awk '{print $NF}'`
	    #mv -f $TMP.fn $TMP.fp
	    rm -f $TMP.fn
	    #echo "listDs='$listDs'" ;  echo "listDp='$listDp'"
	    if test "$listDs" != "$listDp" ; then
		if test $dBug = 't' ; then echo ' -> re-order list of exp' ; fi
		#-- re-order list of exp (according to local "sort" order)
		listDs=`echo "$listDs" | sort`
		for xx in $listDs ; do
		    sed -e "/ $xx\$/d" -e "/ $xx /d" $TMP.fs > $TMP.fn
		    sed -n -e "/ $xx\$/p" -e "/ $xx /p" $TMP.fs >> $TMP.fn
		    mv -f $TMP.fn $TMP.fs
		done
		listDp=`echo "$listDp" | sort`
		for xx in $listDp ; do
		    sed -e "/ $xx\$/d" -e "/ $xx /d" $TMP.fp > $TMP.fn
		    sed -n -e "/ $xx\$/p" -e "/ $xx /p" $TMP.fp >> $TMP.fn
		    mv -f $TMP.fn $TMP.fp
		done
	    #else
	    #   echo 'same list of exp => skip re-ordering'
	    fi
	    echo -n "=== diff $sdir $pdir :" | tee -a $OUTPFIL
	    diff $TMP.fs $TMP.fp > $TMP.dif
	    yy=$?
	    if test $yy != '0' ; then
		echo " $type , of='$optf'" | tee -a $OUTPFIL
		#grep '^run: ' $outp
		#grep '^run: ' $outs
		#-- score for each test:
		grep '^[YN] [YN] [YN] [YN]' $TMP.fs > tmploc 2>/dev/null
		t_tot=`cat tmploc | wc -l | sed -e 's| ||g'`
		t_pass=`grep '^Y Y Y Y' tmploc | grep 'pass ' | wc -l | sed -e 's| ||g'`
		score_s=`printf '%2i' $t_pass`":$t_tot"
		grep '^[YN] [YN] [YN] [YN]' $TMP.fp > tmploc 2>/dev/null
		t_tot=`cat tmploc | wc -l | sed -e 's| ||g'`
		t_pass=`grep '^Y Y Y Y' tmploc | grep 'pass ' | wc -l | sed -e 's| ||g'`
		score_p=`printf '%2i' $t_pass`":$t_tot"
		rm -f tmploc
		#-- count Nb of different lines
		ndf=`wc -l $TMP.dif | awk '{print $1}'`
		nn=`grep -c ' generated by TAF Version ' $TMP.dif`
		ndf=`expr $ndf - $nn`
		ndf=`expr $ndf / 2`
		nn=`grep -c '^---' $TMP.dif`
		ndf=`expr $ndf - $nn`
		if [ $ndf -le $NBLDIFF ] ; then
		  cat $TMP.dif
		else
		  echo " $ndf lines differ ( $score_s , $score_p )"
		fi
		echo "< "`head -1 $outs`" ( $score_s )" >> $OUTPFIL
		echo "> "`head -1 $outp`" ( $score_p )" >> $OUTPFIL
		cat $TMP.dif >> $OUTPFIL
		if [ $cdif -eq 0 ] ; then
		  clin=`printf '%3i (%5s,%5s)' $ndf $score_s $score_p`
		else
		  clin="$clin,"`printf '%3i (%5s,%5s)' $ndf $score_s $score_p`
		fi
		cdif=`expr $cdif + 1`
		echo '----------------------------------------' | tee -a $OUTPFIL
	    else
		echo " $type , of='$optf'" | tee -a $OUTPFIL
	    fi
	    rm -f $TMP.fs $TMP.fp $TMP.dif
	  fi
	done
	if [ $cmis -gt 0 ] ; then tmis='(-'$cmis')' ; else tmis='(--)' ; fi
	printf '%11s :  %3i      %4s   %3i  ' $sname $ctot $tmis $cdif >> $OUTPSUM
	if [ $cdif -gt 0 ] ; then
	  echo "   $clin" >> $OUTPSUM
	else echo "" >> $OUTPSUM ; fi
    fi

done

echo "Start time:  $sTime" | tee -a $OUTPFIL
echo "End time:    "`date` | tee -a $OUTPFIL | tee -a $OUTPSUM

rm -f $TMP.dir_all $TMP.slist $TMP.plist
if test $OUTPFIL = "TTT.$$" ; then rm -f $OUTPFIL ; fi
if test "x$ADDRESS" != 'xnone' ; then
  #- change default sender address ("From:") using "mail -r" option
  #  since, by default, baudelaire uses USER@localhost.localdomain
  #fullNam=`hostname -A | awk '{print $1}'`
  #MAILCMD="$MAILCMD -r ${USER}@${fullNam}"
  #---
  echo ".. send $OUTPSUM to '$ADDRESS' using '$MAILCMD'"
  $MAILCMD -s 'check_outp summary' $ADDRESS < $OUTPSUM
fi
echo ".. cat $OUTPSUM"
cat $OUTPSUM
exit 0