mitgcm.org/front_content/parse_emails

#! /usr/bin/env bash

# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.24 2010/10/31 02:25:05 jmc Exp $
#
#  The purpose of this script is to parse the emails produced by the
#  MITgcm/verificaton/testreport script and store the data in a
#  reasonable location.


usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-h|-help)           print usage"
    echo "  (-s|-silent)         silent mode"
    echo "  (-v|-verbose)        verbose mode"
    echo "  (-i |-ind )DIR       get mpack-created emails from DIR"
    echo "                         [def=\"$INDIR\"]"
    echo "  (-o |-outd )DIR      write the data to DIR"
    echo "                         [def=\"$BASEDIR/$monthDir\"]"
    echo "  (-t |-tempd )DIR     use temporary directory DIR"
    echo "                         [def=\"$TEMPDIR\"]"
    echo "  (-u |-unpack )EXE    use executable EXE to unpack e-mails"
    echo "                         [def=\"$MUNPACK\"]"
    echo "  (-a |-addr )ADDR     send e-mail to ADDR if Error"
    echo "                         [def='"$ADDRERR"']"
    echo
    exit 1
}

# defaults
HERE=`pwd`
INDIR="/u/u2/jmc/Mail/MITgcm-test"
BASEDIR="/u/u0/httpd/html/testing/results"
monthDir=`date +%Y`"_"`date +%m`
OUTDIR=
ADDRERR=
MUNPACK=$HERE/munpack
UnpTmpD="/var/tmp/m-prts-$USER"
TR_LIST='TTT.'$$
TEMPDIR=/tmp/prc_emails
STDOUT=$TEMPDIR/'outp.'$$
ERRMSG=/tmp/tmp.$$
PRT=1

#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
#  Parse options
ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`

    case $ac_option in

        -help | --help | -h | --h)
            usage ;;
        -s | --s | -silent | --silent)
            PRT=0 ;;
        -v | --v | -verbose | --verbose)
            PRT=2 ;;

        -ind | --ind | -i | --i)
            ac_prev=INDIR ;;
        --ind=* | -ind=* | --i=* | -i=*)
            INDIR=$ac_optarg ;;

        -outd | --outd | -o | --o)
            ac_prev=OUTDIR ;;
        --outd=* | -outd=* | --o=* | -o=*)
            OUTDIR=$ac_optarg ;;

        -tempd | --tempd | -t | --t)
            ac_prev=TEMPDIR ;;
        --tempd=* | -tempd=* | --t=* | -t=*)
            TEMPDIR=$ac_optarg ;;

        -u | --u | -unpack | --unpack)
            ac_prev=MUNPACK ;;
        -u=* | --u=* | -unpack=* | --unpack=*)
            MUNPACK=$ac_optarg ;;

        -a | --a | -addr | --addr)
            ac_prev=ADDRERR ;;
        -a=* | --a=* | -addr=* | --addr=*)
            ADDRERR=$ac_optarg ;;

        *)
            date
            echo "Error: don't understand argument \"$ac_option\""
            usage
            ;;

     esac

done

if test ! -x $MUNPACK ; then
        date
        echo "ERROR: \"$MUNPACK\" is not executable"
        exit 2
fi
if test "x$OUTDIR" = x ; then
  OUTDIR="$BASEDIR/$monthDir"
else
  monthDir=0
fi
if test ! -e $OUTDIR ; then
    mkdir $OUTDIR
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
        date
        echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
        exit 3
    else
        echo "Successfully created new dir: \"$OUTDIR\""
    fi
    chgrp gcmpack $OUTDIR
    chmod 775 $OUTDIR
fi

#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|

all_msg=`ls -1 $INDIR`
nb_files=`echo "$all_msg" | grep -c '^msg\.'`

if test $PRT = 2 ; then
  echo "Using OUTDIR=\"$OUTDIR\""
  echo "Using INDIR=\"$INDIR\""
  echo -n "Unpacking $nb_files emails ("`date`
  if test "x$ADDRERR" != x ; then
     echo -n ", err-msg: '$ADDRERR'"
  fi
  echo ")"
elif test $nb_files != 0 ; then
  echo -n "Unpacking $nb_files emails ("`date`
  if test "x$ADDRERR" != x ; then
     echo -n ", err-msg: '$ADDRERR'"
  fi
  echo ")"
  echo " from '$INDIR' to '$OUTDIR'"
fi

#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|

#-- process list of message to detect group of multi-part msgs:
rm -f $TR_LIST
touch $TR_LIST

flag=0
for xx in $all_msg
do
  in=`grep -c $xx $TR_LIST`
  if test $in = 0 ; then
    np=`grep -c 'Content-Type: message/partial' $INDIR/$xx`
    if test $np = 0 ; then
      echo $xx >> $TR_LIST
    else
      l=`sed -n '/Content-Type: message\/partial/=' $INDIR/$xx`
      lp=`expr $l + 1`
      id=`sed -n "$lp p" $INDIR/$xx`
      partM=`( cd $INDIR ; grep -c "$id" msg.* | grep -v ':0$' | sed 's/:1$//' )`
      echo $partM >> $TR_LIST
      if test "x$ADDRERR" != x ; then flag=1
        echo "multi-parts message:" $partM >> $ERRMSG
        ( cd $INDIR ; ls -l $partM ) >> $ERRMSG
      fi
    fi
  fi
done
if test $flag = 1 ; then
    mail -s 'parse_emails Multi-parts msg' $ADDRERR < $ERRMSG
    rm -f $ERRMSG
fi
#cat $TR_LIST ; echo '----------------------------------'

#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|

#-- process list of individual + group of multi-part messages
Nbl=`wc -l $TR_LIST | cut -d ' ' -f 1`
n=0
while [ $n -lt $Nbl ] ; do
    n=`expr $n + 1`
    errFlg=0 ; prcM=''
    grpM=`sed -n "$n p" $TR_LIST`
    nm=`sed -n "$n p" $TR_LIST | wc -w | cut -d ' ' -f 1`

#-  create local copy
    test -e $TEMPDIR  &&  rm -rf $TEMPDIR
    mkdir $TEMPDIR
    RETVAL=$?
    if test "x$RETVAL" = x0 ; then
      ( cd $INDIR ; cp $grpM $TEMPDIR )
      RETVAL=$?
    fi
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "error parsing email '$grpM' :" > $ERRMSG
        echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$grpM $TEMPDIR'" | tee -a $ERRMSG
        echo " returns error $RETVAL" | tee -a $ERRMSG
        mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
        rm -f $ERRMSG
      fi
      ( cd $INDIR ; mv -f $grpM ../fail2process )
      continue
    fi

    #-check that we have all the parts
    if [ $PRT -ge 1 -a $nm -gt 1 ] ; then
      echo " group (nm=$nm) of multi-parts msg: '$grpM'"
    fi
    for xx in $grpM ; do
      np=`grep 'Content-Type: message/partial' $TEMPDIR/$xx \
               | sed 's/^Content.*total=//' | sed 's/;$//'`
      if test "x$np" = x -a $nm = 1 ; then np=1 ; fi
      if test "x$np" != "x$nm" ; then
        if test $errFlg = 0 ; then errFlg=1
          echo "error parsing email '$grpM' :" > $ERRMSG
        fi
        echo " - Error: Number of parts='$np' in $xx but got '$nm' msg-files" \
             | tee -a $ERRMSG
      fi
    done
    if test $errFlg = 1 ; then
      if test "x$ADDRERR" != x ; then
        mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
      fi
      rm -f $ERRMSG
      ( cd $INDIR ; mv -f $grpM ../fail2process )
      continue
    fi

    if test $nm = 1 ; then
#-  unpack single part message:
      xx=$grpM
      prcM=$xx
      if test $PRT = 2 ; then echo " unpack single-part msg: '$xx'" ; fi
      ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
      RETVAL=$?
      if test "x$RETVAL" != x0 ; then errFlg=1
          echo "error parsing email '$xx' :" > $ERRMSG
          echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
          ls -l $INDIR"/"$xx >> $ERRMSG
      else
          trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
          if test ! -f $TEMPDIR/$trOutp ; then errFlg=1
            echo "error parsing email '$xx' :" > $ERRMSG
            echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
                 | tee -a $ERRMSG
          fi
      fi
      if test $errFlg = 1 ; then
          cat $STDOUT | tee -a $ERRMSG
          echo " <<<-----------------------"
          if test "x$ADDRERR" != x ; then
            mail -s 'parse_emails err_2s' $ADDRERR < $ERRMSG
          fi
          rm -f $ERRMSG
      fi
      rm -f $STDOUT

    else
#-  unpack group of multi-parts message:

    #-unpack each part, in the right order:
      j=0
      while [ $j -lt $nm ] ; do
        j=`expr $j + 1`
        #- get the j^th part msg
        xx=`( cd $TEMPDIR ; grep -c "Content-Type: message/partial; number=$j" $grpM \
            | grep -v ':0$' | sed 's/:1$//' )`
        if test $PRT = 2 ; then
          echo -n " $j : xx='$xx' : "
          grep 'Content-Type: message/partial' $TEMPDIR/$xx | sed 's/Content-Type: //'
        fi
        #- check
        if test ! -f $TEMPDIR/$xx ; then
            echo "error parsing email '$xx' :" > $ERRMSG
            echo " - Error: Missing msg file '$xx' in $TEMPDIR:" | tee -a $ERRMSG
            ls -l $TEMDIR | tee -a $ERRMSG
            errFlg=1 ; echo " <<<-----------------------"
            if test "x$ADDRERR" != x ; then
              mail -s 'parse_emails err_2g' $ADDRERR < $ERRMSG
            fi
            rm -f $ERRMSG
        else
          if test $j = 1 ; then prcM=$xx ; else prcM="$prcM $xx" ; fi
        #- do unpack
          ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
          RETVAL=$?
          if test "x$RETVAL" != x0 ; then errFlg=1
            echo "error parsing email '$xx' :" > $ERRMSG
            echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
            ls -l $INDIR"/"$xx | tee -a $ERRMSG
            cat $STDOUT | tee -a $ERRMSG
            errFlg=1 ; echo " <<<-----------------------"
            if test "x$ADDRERR" != x ; then
              mail -s 'parse_emails err_2u' $ADDRERR < $ERRMSG
            fi
            rm -f $ERRMSG
          fi
        fi
        if test $errFlg = 1 ; then j=`expr $nm + 1` ; fi

        if [ $j -lt $nm ] ; then
        #- if not last part:
          pfix=`cat $STDOUT | tail -1 | awk '{print $NF}'`
          if test -d $UnpTmpD/$pfix ; then
            lock=$UnpTmpD/$pfix/CT
            if test -e $lock ; then
              if test $PRT = 2 ; then echo "    remove lock: $lock" ; fi
              rm -f $lock
            else
              echo "error parsing email '$xx' :" > $ERRMSG
              echo " - Error: lock file '$lock' not found from $MUNPACK output:" \
                   | tee -a $ERRMSG
              cat $STDOUT | tee -a $ERRMSG
              errFlg=1 ; echo " <<<-----------------------"
            fi
          else
              echo "error parsing email '$xx' :" > $ERRMSG
              echo " - Error: found no dir '$UnpTmpD/$pfix' from $MUNPACK output:" \
                   | tee -a $ERRMSG
              cat $STDOUT | tee -a $ERRMSG
              errFlg=1 ; echo " <<<-----------------------"
          fi
        elif [ $j -eq $nm ] ; then
        #- if last part:
           trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
           if test ! -f $TEMPDIR/$trOutp ; then
              echo "error parsing email '$xx' :" > $ERRMSG
              echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
                   | tee -a $ERRMSG
              cat $STDOUT | tee -a $ERRMSG
              errFlg=1 ; echo " <<<-----------------------"
           fi
        fi
        if [ $errFlg -eq 1 -a $j -le $nm ] ; then j=$nm
           if test "x$ADDRERR" != x ; then
              mail -s 'parse_emails err_2m' $ADDRERR < $ERRMSG
           fi
           rm -f $ERRMSG
        fi
        rm -f $STDOUT
      done
    fi
    #--  in case of error: mv all $grpM msg to "fail2proc"
    if test $errFlg = 1 ; then
        ( cd $INDIR ; mv -f $grpM ../fail2process )
        continue
    fi
    if test $PRT = 2 ; then
          if test -f $TEMPDIR/$trOutp ; then ls -l $TEMPDIR/$trOutp ; fi
    fi

#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|

    #-- un-tar
    #( cd $TEMPDIR ; tar -xzvf $trOutp > $STDOUT )
    #   to remove small files "._mydir" that some MAC OS are adding
    #   (for each file or dir) to a tar-file, use option "--exclude=":
    ( cd $TEMPDIR ; tar -xzvf $trOutp --exclude="._*" > $STDOUT )
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      echo "parsing email error" > $ERRMSG
      echo " - Error:  tar -xzvf $trOutp returns:" $RETVAL | tee -a $ERRMSG
      ( cd $INDIR ; ls -l $prcM ) | tee -a $ERRMSG
      ls -l $TEMPDIR/$trOutp  | tee -a $ERRMSG
      if test "x$ADDRERR" != x ; then
        mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
      fi
      rm -f $ERRMSG
      ( cd $INDIR ; mv -f $grpM ../fail2process )
      continue
    fi
    tdir=`cat $STDOUT | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
    if test -d $TEMPDIR/$tdir ; then
      rm -f $STDOUT
    else
      echo "parsing email error" > $ERRMSG
      echo " - Error: fail to get dir output name 'tdir=$tdir'" | tee -a $ERRMSG
      echo "   (cont) from tar file '$TEMPDIR/$trOutp'" | tee -a $ERRMSG
      if test "x$ADDRERR" != x ; then
        mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
      fi
      rm -f $ERRMSG
      ( cd $INDIR ; mv -f $grpM ../fail2process )
      continue
    fi

    #-- select which Monthly Output Dir:
    locDir=$OUTDIR
    if test "x$monthDir" != x0 ; then
      dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
               | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
      mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
      if test "x$mn" != "x$monthDir" ; then
        if test "x$mn" = x ; then
          if [ $PRT -ge 1 ] ; then echo " cannot get month from '$tdir'" ; fi
        else
          locDir="$BASEDIR/$mn"
          if test ! -d $locDir ; then
            if [ $PRT -ge 1 ] ; then
              echo "NO DIR: '$locDir' => '$tdir' POSTPONED"
            fi
            if test "x$ADDRERR" != x ; then
              echo "parsing email error" > $ERRMSG
              echo "  no dir '$locDir' for outp. '$tdir'" >> $ERRMSG
              ( cd $INDIR ; ls -l $prcM ) >> $ERRMSG
              mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
              rm -f $ERRMSG
            fi
            continue
          fi
        fi
      fi
    fi

    #-- copy to $locDir and rename if necessary
    sdir=$tdir
    if test -e $locDir"/"$tdir ; then
        sdir=`echo $tdir | sed 's/_[0-9]*$//'`
        ad=0
        while test -e $locDir"/"$sdir"_"$ad ; do
            ad=$(( $ad + 1 ))
        done
        sdir=$sdir"_"$ad
    fi
    if [ $PRT -ge 1 ] ; then
      if test "x$locDir" = "x$OUTDIR"
      then echo " '$sdir' ($prcM)"
      else echo " '$sdir' ($prcM) => '$locDir'"
      fi
    fi
    mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > $ERRMSG
        echo " - Error: mv $TEMPDIR/$tdir $locDir/$sdir returns:" $RETVAL \
             | tee -a $ERRMSG
        echo -n "   in dir: $TEMPDIR : " ; ls -l $TEMPDIR  | tee -a $ERRMSG
        echo -n "   in dir: $OUTDIR : " ; ls -l $locDir | tee -a $ERRMSG
        mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
        rm -f $ERRMSG
      fi
      ( cd $INDIR ; mv -f $grpM ../fail2process )
      continue
    fi
    chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1

    #-- remove the original message files
    ( cd $INDIR ; rm -f $grpM )

done
rm -f $TR_LIST
if test $PRT = 2 ; then echo "  done" ; fi

1	edhill	1.1	#! /usr/bin/env bash
2
3	jmc	1.25	# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.24 2010/10/31 02:25:05 jmc Exp $
4	edhill	1.1	#
5			# The purpose of this script is to parse the emails produced by the
6			# MITgcm/verificaton/testreport script and store the data in a
7			# reasonable location.
8
9
10			usage()
11			{
12			echo
13			echo "Usage: $0 [OPTIONS]"
14	jmc	1.19	echo
15	edhill	1.1	echo "where possible OPTIONS are:"
16	jmc	1.10	echo " (-h\|-help) print usage"
17			echo " (-s\|-silent) silent mode"
18			echo " (-v\|-verbose) verbose mode"
19			echo " (-i \|-ind )DIR get mpack-created emails from DIR"
20	edhill	1.1	echo " [def=\"$INDIR\"]"
21	jmc	1.10	echo " (-o \|-outd )DIR write the data to DIR"
22	jmc	1.19	echo " [def=\"$BASEDIR/$monthDir\"]"
23	jmc	1.10	echo " (-t \|-tempd )DIR use temporary directory DIR"
24	edhill	1.1	echo " [def=\"$TEMPDIR\"]"
25	jmc	1.14	echo " (-u \|-unpack )EXE use executable EXE to unpack e-mails"
26			echo " [def=\"$MUNPACK\"]"
27	jmc	1.12	echo " (-a \|-addr )ADDR send e-mail to ADDR if Error"
28			echo " [def='"$ADDRERR"']"
29	jmc	1.19	echo
30	edhill	1.1	exit 1
31			}
32
33			# defaults
34	jmc	1.25	HERE=`pwd`
35	jmc	1.12	INDIR="/u/u2/jmc/Mail/MITgcm-test"
36	jmc	1.19	BASEDIR="/u/u0/httpd/html/testing/results"
37			monthDir=`date +%Y`"_"`date +%m`
38			OUTDIR=
39	jmc	1.25	ADDRERR=
40			MUNPACK=$HERE/munpack
41			UnpTmpD="/var/tmp/m-prts-$USER"
42			TR_LIST='TTT.'$$
43	jmc	1.22	TEMPDIR=/tmp/prc_emails
44	jmc	1.25	STDOUT=$TEMPDIR/'outp.'$$
45	jmc	1.22	ERRMSG=/tmp/tmp.$$
46	jmc	1.10	PRT=1
47	edhill	1.1
48	jmc	1.25	#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-\|--+----\|
49	edhill	1.1	# Parse options
50			ac_prev=
51			for ac_option ; do
52
53			# If the previous option needs an argument, assign it.
54			if test -n "$ac_prev"; then
55			eval "$ac_prev=\$ac_option"
56			ac_prev=
57			continue
58			fi
59
60			ac_optarg=`expr "x$ac_option" : 'x[^=]=\(.\)'`
61	jmc	1.19
62	edhill	1.1	case $ac_option in
63	jmc	1.25
64			-help \| --help \| -h \| --h)
65			usage ;;
66			-s \| --s \| -silent \| --silent)
67	jmc	1.10	PRT=0 ;;
68	jmc	1.25	-v \| --v \| -verbose \| --verbose)
69	jmc	1.10	PRT=2 ;;
70	jmc	1.25
71	edhill	1.1	-ind \| --ind \| -i \| --i)
72			ac_prev=INDIR ;;
73			--ind=* \| -ind=* \| --i=* \| -i=*)
74			INDIR=$ac_optarg ;;
75	jmc	1.19
76	edhill	1.1	-outd \| --outd \| -o \| --o)
77			ac_prev=OUTDIR ;;
78			--outd=* \| -outd=* \| --o=* \| -o=*)
79			OUTDIR=$ac_optarg ;;
80	jmc	1.19
81	edhill	1.1	-tempd \| --tempd \| -t \| --t)
82			ac_prev=TEMPDIR ;;
83			--tempd=* \| -tempd=* \| --t=* \| -t=*)
84			TEMPDIR=$ac_optarg ;;
85	jmc	1.14
86			-u \| --u \| -unpack \| --unpack)
87			ac_prev=MUNPACK ;;
88			-u=* \| --u=* \| -unpack=* \| --unpack=*)
89			MUNPACK=$ac_optarg ;;
90	jmc	1.19
91	jmc	1.14	-a \| --a \| -addr \| --addr)
92	jmc	1.12	ac_prev=ADDRERR ;;
93	jmc	1.14	-a=* \| --a=* \| -addr=* \| --addr=*)
94	jmc	1.12	ADDRERR=$ac_optarg ;;
95	jmc	1.14
96	edhill	1.1	*)
97	jmc	1.25	date
98			echo "Error: don't understand argument \"$ac_option\""
99			usage
100	edhill	1.1	;;
101	jmc	1.19
102	edhill	1.1	esac
103	jmc	1.19
104	edhill	1.1	done
105
106	jmc	1.14	if test ! -x $MUNPACK ; then
107	jmc	1.25	date
108			echo "ERROR: \"$MUNPACK\" is not executable"
109			exit 2
110	jmc	1.14	fi
111	jmc	1.19	if test "x$OUTDIR" = x ; then
112			OUTDIR="$BASEDIR/$monthDir"
113			else
114			monthDir=0
115			fi
116	edhill	1.1	if test ! -e $OUTDIR ; then
117			mkdir $OUTDIR
118			RETVAL=$?
119	jmc	1.15	if test "x$RETVAL" != x0 ; then
120	jmc	1.25	date
121			echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
122			exit 3
123	jmc	1.20	else
124	jmc	1.25	echo "Successfully created new dir: \"$OUTDIR\""
125	edhill	1.1	fi
126	jmc	1.8	chgrp gcmpack $OUTDIR
127			chmod 775 $OUTDIR
128	edhill	1.1	fi
129
130	jmc	1.25	#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-\|--+----\|
131	edhill	1.1
132	jmc	1.25	all_msg=`ls -1 $INDIR`
133			nb_files=`echo "$all_msg" \| grep -c '^msg\.'`
134
135			if test $PRT = 2 ; then
136	jmc	1.10	echo "Using OUTDIR=\"$OUTDIR\""
137			echo "Using INDIR=\"$INDIR\""
138	jmc	1.25	echo -n "Unpacking $nb_files emails ("`date`
139			if test "x$ADDRERR" != x ; then
140			echo -n ", err-msg: '$ADDRERR'"
141			fi
142			echo ")"
143	jmc	1.10	elif test $nb_files != 0 ; then
144	jmc	1.17	echo -n "Unpacking $nb_files emails ("`date`
145	jmc	1.12	if test "x$ADDRERR" != x ; then
146	jmc	1.17	echo -n ", err-msg: '$ADDRERR'"
147	jmc	1.12	fi
148	jmc	1.17	echo ")"
149			echo " from '$INDIR' to '$OUTDIR'"
150	jmc	1.10	fi
151	edhill	1.1
152	jmc	1.25	#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-\|--+----\|
153	edhill	1.1
154	jmc	1.25	#-- process list of message to detect group of multi-part msgs:
155			rm -f $TR_LIST
156			touch $TR_LIST
157
158			flag=0
159			for xx in $all_msg
160			do
161			in=`grep -c $xx $TR_LIST`
162			if test $in = 0 ; then
163			np=`grep -c 'Content-Type: message/partial' $INDIR/$xx`
164			if test $np = 0 ; then
165			echo $xx >> $TR_LIST
166			else
167			l=`sed -n '/Content-Type: message\/partial/=' $INDIR/$xx`
168			lp=`expr $l + 1`
169			id=`sed -n "$lp p" $INDIR/$xx`
170			partM=`( cd $INDIR ; grep -c "$id" msg.* \| grep -v ':0$' \| sed 's/:1$//' )`
171			echo $partM >> $TR_LIST
172			if test "x$ADDRERR" != x ; then flag=1
173			echo "multi-parts message:" $partM >> $ERRMSG
174			( cd $INDIR ; ls -l $partM ) >> $ERRMSG
175			fi
176			fi
177			fi
178			done
179			if test $flag = 1 ; then
180			mail -s 'parse_emails Multi-parts msg' $ADDRERR < $ERRMSG
181			rm -f $ERRMSG
182			fi
183			#cat $TR_LIST ; echo '----------------------------------'
184
185			#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-\|--+----\|
186
187			#-- process list of individual + group of multi-part messages
188			Nbl=`wc -l $TR_LIST \| cut -d ' ' -f 1`
189			n=0
190			while [ $n -lt $Nbl ] ; do
191			n=`expr $n + 1`
192			errFlg=0 ; prcM=''
193			grpM=`sed -n "$n p" $TR_LIST`
194			nm=`sed -n "$n p" $TR_LIST \| wc -w \| cut -d ' ' -f 1`
195
196			#- create local copy
197	edhill	1.2	test -e $TEMPDIR && rm -rf $TEMPDIR
198	edhill	1.1	mkdir $TEMPDIR
199	jmc	1.15	RETVAL=$?
200			if test "x$RETVAL" = x0 ; then
201	jmc	1.25	( cd $INDIR ; cp $grpM $TEMPDIR )
202	jmc	1.15	RETVAL=$?
203			fi
204			if test "x$RETVAL" != x0 ; then
205			if test "x$ADDRERR" != x ; then
206	jmc	1.25	echo "error parsing email '$grpM' :" > $ERRMSG
207			echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$grpM $TEMPDIR'" \| tee -a $ERRMSG
208			echo " returns error $RETVAL" \| tee -a $ERRMSG
209	jmc	1.22	mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
210			rm -f $ERRMSG
211	jmc	1.15	fi
212	jmc	1.25	( cd $INDIR ; mv -f $grpM ../fail2process )
213	jmc	1.15	continue
214			fi
215	edhill	1.1
216	jmc	1.25	#-check that we have all the parts
217			if [ $PRT -ge 1 -a $nm -gt 1 ] ; then
218			echo " group (nm=$nm) of multi-parts msg: '$grpM'"
219			fi
220			for xx in $grpM ; do
221			np=`grep 'Content-Type: message/partial' $TEMPDIR/$xx \
222			\| sed 's/^Content.*total=//' \| sed 's/;$//'`
223			if test "x$np" = x -a $nm = 1 ; then np=1 ; fi
224			if test "x$np" != "x$nm" ; then
225			if test $errFlg = 0 ; then errFlg=1
226			echo "error parsing email '$grpM' :" > $ERRMSG
227			fi
228			echo " - Error: Number of parts='$np' in $xx but got '$nm' msg-files" \
229			\| tee -a $ERRMSG
230			fi
231			done
232			if test $errFlg = 1 ; then
233	jmc	1.12	if test "x$ADDRERR" != x ; then
234	jmc	1.22	mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
235	jmc	1.12	fi
236	jmc	1.25	rm -f $ERRMSG
237			( cd $INDIR ; mv -f $grpM ../fail2process )
238	jmc	1.12	continue
239	edhill	1.1	fi
240
241	jmc	1.25	if test $nm = 1 ; then
242			#- unpack single part message:
243			xx=$grpM
244			prcM=$xx
245			if test $PRT = 2 ; then echo " unpack single-part msg: '$xx'" ; fi
246			( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
247			RETVAL=$?
248			if test "x$RETVAL" != x0 ; then errFlg=1
249			echo "error parsing email '$xx' :" > $ERRMSG
250			echo " - Error: $MUNPACK $xx returns: $RETVAL" \| tee -a $ERRMSG
251			ls -l $INDIR"/"$xx >> $ERRMSG
252			else
253			trOutp=`tail -1 $STDOUT \| cut -d ' ' -f 1`
254			if test ! -f $TEMPDIR/$trOutp ; then errFlg=1
255			echo "error parsing email '$xx' :" > $ERRMSG
256			echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
257			\| tee -a $ERRMSG
258			fi
259			fi
260			if test $errFlg = 1 ; then
261			cat $STDOUT \| tee -a $ERRMSG
262			echo " <<<-----------------------"
263			if test "x$ADDRERR" != x ; then
264			mail -s 'parse_emails err_2s' $ADDRERR < $ERRMSG
265			fi
266			rm -f $ERRMSG
267	jmc	1.12	fi
268	jmc	1.25	rm -f $STDOUT
269
270			else
271			#- unpack group of multi-parts message:
272
273			#-unpack each part, in the right order:
274			j=0
275			while [ $j -lt $nm ] ; do
276			j=`expr $j + 1`
277			#- get the j^th part msg
278			xx=`( cd $TEMPDIR ; grep -c "Content-Type: message/partial; number=$j" $grpM \
279			\| grep -v ':0$' \| sed 's/:1$//' )`
280			if test $PRT = 2 ; then
281			echo -n " $j : xx='$xx' : "
282			grep 'Content-Type: message/partial' $TEMPDIR/$xx \| sed 's/Content-Type: //'
283			fi
284			#- check
285			if test ! -f $TEMPDIR/$xx ; then
286			echo "error parsing email '$xx' :" > $ERRMSG
287			echo " - Error: Missing msg file '$xx' in $TEMPDIR:" \| tee -a $ERRMSG
288			ls -l $TEMDIR \| tee -a $ERRMSG
289			errFlg=1 ; echo " <<<-----------------------"
290			if test "x$ADDRERR" != x ; then
291			mail -s 'parse_emails err_2g' $ADDRERR < $ERRMSG
292			fi
293			rm -f $ERRMSG
294			else
295			if test $j = 1 ; then prcM=$xx ; else prcM="$prcM $xx" ; fi
296			#- do unpack
297			( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
298			RETVAL=$?
299			if test "x$RETVAL" != x0 ; then errFlg=1
300			echo "error parsing email '$xx' :" > $ERRMSG
301			echo " - Error: $MUNPACK $xx returns: $RETVAL" \| tee -a $ERRMSG
302			ls -l $INDIR"/"$xx \| tee -a $ERRMSG
303			cat $STDOUT \| tee -a $ERRMSG
304			errFlg=1 ; echo " <<<-----------------------"
305			if test "x$ADDRERR" != x ; then
306			mail -s 'parse_emails err_2u' $ADDRERR < $ERRMSG
307			fi
308			rm -f $ERRMSG
309			fi
310			fi
311			if test $errFlg = 1 ; then j=`expr $nm + 1` ; fi
312
313			if [ $j -lt $nm ] ; then
314			#- if not last part:
315			pfix=`cat $STDOUT \| tail -1 \| awk '{print $NF}'`
316			if test -d $UnpTmpD/$pfix ; then
317			lock=$UnpTmpD/$pfix/CT
318			if test -e $lock ; then
319			if test $PRT = 2 ; then echo " remove lock: $lock" ; fi
320			rm -f $lock
321			else
322			echo "error parsing email '$xx' :" > $ERRMSG
323			echo " - Error: lock file '$lock' not found from $MUNPACK output:" \
324			\| tee -a $ERRMSG
325			cat $STDOUT \| tee -a $ERRMSG
326			errFlg=1 ; echo " <<<-----------------------"
327			fi
328			else
329			echo "error parsing email '$xx' :" > $ERRMSG
330			echo " - Error: found no dir '$UnpTmpD/$pfix' from $MUNPACK output:" \
331			\| tee -a $ERRMSG
332			cat $STDOUT \| tee -a $ERRMSG
333			errFlg=1 ; echo " <<<-----------------------"
334			fi
335			elif [ $j -eq $nm ] ; then
336			#- if last part:
337			trOutp=`tail -1 $STDOUT \| cut -d ' ' -f 1`
338			if test ! -f $TEMPDIR/$trOutp ; then
339			echo "error parsing email '$xx' :" > $ERRMSG
340			echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
341			\| tee -a $ERRMSG
342			cat $STDOUT \| tee -a $ERRMSG
343			errFlg=1 ; echo " <<<-----------------------"
344			fi
345			fi
346			if [ $errFlg -eq 1 -a $j -le $nm ] ; then j=$nm
347			if test "x$ADDRERR" != x ; then
348			mail -s 'parse_emails err_2m' $ADDRERR < $ERRMSG
349			fi
350			rm -f $ERRMSG
351			fi
352			rm -f $STDOUT
353			done
354			fi
355			#-- in case of error: mv all $grpM msg to "fail2proc"
356			if test $errFlg = 1 ; then
357			( cd $INDIR ; mv -f $grpM ../fail2process )
358			continue
359			fi
360			if test $PRT = 2 ; then
361			if test -f $TEMPDIR/$trOutp ; then ls -l $TEMPDIR/$trOutp ; fi
362	edhill	1.1	fi
363
364	jmc	1.25	#---+----1----+----2----+----3----+----4----+----5----+----6----+----7-\|--+----\|
365
366	jmc	1.21	#-- un-tar
367	jmc	1.25	#( cd $TEMPDIR ; tar -xzvf $trOutp > $STDOUT )
368	jmc	1.21	# to remove small files "._mydir" that some MAC OS are adding
369			# (for each file or dir) to a tar-file, use option "--exclude=":
370	jmc	1.25	( cd $TEMPDIR ; tar -xzvf $trOutp --exclude="._*" > $STDOUT )
371	edhill	1.1	RETVAL=$?
372			if test "x$RETVAL" != x0 ; then
373	jmc	1.25	echo "parsing email error" > $ERRMSG
374			echo " - Error: tar -xzvf $trOutp returns:" $RETVAL \| tee -a $ERRMSG
375			( cd $INDIR ; ls -l $prcM ) \| tee -a $ERRMSG
376			ls -l $TEMPDIR/$trOutp \| tee -a $ERRMSG
377	jmc	1.12	if test "x$ADDRERR" != x ; then
378	jmc	1.22	mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
379	jmc	1.12	fi
380	jmc	1.25	rm -f $ERRMSG
381			( cd $INDIR ; mv -f $grpM ../fail2process )
382	jmc	1.12	continue
383	edhill	1.1	fi
384	jmc	1.25	tdir=`cat $STDOUT \| head -1 \| sed -e 's\|^./\|\|g' \| cut -d '/' -f 1`
385	jmc	1.21	if test -d $TEMPDIR/$tdir ; then
386	jmc	1.25	rm -f $STDOUT
387	jmc	1.21	else
388	jmc	1.25	echo "parsing email error" > $ERRMSG
389			echo " - Error: fail to get dir output name 'tdir=$tdir'" \| tee -a $ERRMSG
390			echo " (cont) from tar file '$TEMPDIR/$trOutp'" \| tee -a $ERRMSG
391	jmc	1.21	if test "x$ADDRERR" != x ; then
392	jmc	1.22	mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
393	jmc	1.21	fi
394	jmc	1.25	rm -f $ERRMSG
395			( cd $INDIR ; mv -f $grpM ../fail2process )
396	jmc	1.21	continue
397			fi
398	edhill	1.1
399	jmc	1.21	#-- select which Monthly Output Dir:
400	jmc	1.19	locDir=$OUTDIR
401			if test "x$monthDir" != x0 ; then
402			dd=`echo $tdir \| sed 's/_/ /g' \| awk '{ for(i=1;i<=NF;i++) print $i }'\
403			\| grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' \| tail -1`
404			mn=`echo $dd \| sed 's/..$//' \| sed 's/..$/_&/'`
405			if test "x$mn" != "x$monthDir" ; then
406	jmc	1.25	if test "x$mn" = x ; then
407			if [ $PRT -ge 1 ] ; then echo " cannot get month from '$tdir'" ; fi
408			else
409			locDir="$BASEDIR/$mn"
410			if test ! -d $locDir ; then
411			if [ $PRT -ge 1 ] ; then
412			echo "NO DIR: '$locDir' => '$tdir' POSTPONED"
413			fi
414			if test "x$ADDRERR" != x ; then
415			echo "parsing email error" > $ERRMSG
416			echo " no dir '$locDir' for outp. '$tdir'" >> $ERRMSG
417			( cd $INDIR ; ls -l $prcM ) >> $ERRMSG
418			mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
419			rm -f $ERRMSG
420			fi
421			continue
422	jmc	1.19	fi
423			fi
424			fi
425			fi
426
427	jmc	1.21	#-- copy to $locDir and rename if necessary
428	jmc	1.10	sdir=$tdir
429	jmc	1.19	if test -e $locDir"/"$tdir ; then
430	jmc	1.24	sdir=`echo $tdir \| sed 's/_[0-9]*$//'`
431	jmc	1.25	ad=0
432			while test -e $locDir"/"$sdir"_"$ad ; do
433			ad=$(( $ad + 1 ))
434			done
435	jmc	1.24	sdir=$sdir"_"$ad
436	edhill	1.1	fi
437	jmc	1.25	if [ $PRT -ge 1 ] ; then
438	jmc	1.19	if test "x$locDir" = "x$OUTDIR"
439	jmc	1.25	then echo " '$sdir' ($prcM)"
440			else echo " '$sdir' ($prcM) => '$locDir'"
441	jmc	1.19	fi
442			fi
443			mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
444	jmc	1.15	RETVAL=$?
445			if test "x$RETVAL" != x0 ; then
446			if test "x$ADDRERR" != x ; then
447	jmc	1.22	echo "parsing email error" > $ERRMSG
448	jmc	1.25	echo " - Error: mv $TEMPDIR/$tdir $locDir/$sdir returns:" $RETVAL \
449			\| tee -a $ERRMSG
450			echo -n " in dir: $TEMPDIR : " ; ls -l $TEMPDIR \| tee -a $ERRMSG
451			echo -n " in dir: $OUTDIR : " ; ls -l $locDir \| tee -a $ERRMSG
452	jmc	1.22	mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
453			rm -f $ERRMSG
454	jmc	1.15	fi
455	jmc	1.25	( cd $INDIR ; mv -f $grpM ../fail2process )
456	jmc	1.15	continue
457			fi
458	jmc	1.19	chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
459	edhill	1.1
460	jmc	1.25	#-- remove the original message files
461			( cd $INDIR ; rm -f $grpM )
462	edhill	1.1
463			done
464	jmc	1.25	rm -f $TR_LIST
465			if test $PRT = 2 ; then echo " done" ; fi
466	edhill	1.1