mitgcm.org/front_content/parse_emails

#! /usr/bin/env bash

# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.19 2009/02/02 19:16:45 jmc Exp $
#
#  The purpose of this script is to parse the emails produced by the
#  MITgcm/verificaton/testreport script and store the data in a
#  reasonable location.


usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-h|-help)           print usage"
    echo "  (-s|-silent)         silent mode"
    echo "  (-v|-verbose)        verbose mode"
    echo "  (-i |-ind )DIR       get mpack-created emails from DIR"
    echo "                         [def=\"$INDIR\"]"
    echo "  (-o |-outd )DIR      write the data to DIR"
    echo "                         [def=\"$BASEDIR/$monthDir\"]"
    echo "  (-t |-tempd )DIR     use temporary directory DIR"
    echo "                         [def=\"$TEMPDIR\"]"
    echo "  (-u |-unpack )EXE    use executable EXE to unpack e-mails"
    echo "                         [def=\"$MUNPACK\"]"
    echo "  (-a |-addr )ADDR     send e-mail to ADDR if Error"
    echo "                         [def='"$ADDRERR"']"
    echo
    exit 1
}

# defaults
INDIR="/u/u2/jmc/Mail/MITgcm-test"
BASEDIR="/u/u0/httpd/html/testing/results"
monthDir=`date +%Y`"_"`date +%m`
OUTDIR=
TEMPDIR=./ptmp
MUNPACK=munpack
ADDRERR=
PRT=1

#  Parse options
ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`

    case $ac_option in
	
	-help | --help | -h | --h)
	    usage ;;
	-s | --s | -silent | --silent)
            PRT=0 ;;
	-v | --v | -verbose | --verbose)
            PRT=2 ;;
	
        -ind | --ind | -i | --i)
            ac_prev=INDIR ;;
        --ind=* | -ind=* | --i=* | -i=*)
            INDIR=$ac_optarg ;;

        -outd | --outd | -o | --o)
            ac_prev=OUTDIR ;;
        --outd=* | -outd=* | --o=* | -o=*)
            OUTDIR=$ac_optarg ;;

        -tempd | --tempd | -t | --t)
            ac_prev=TEMPDIR ;;
        --tempd=* | -tempd=* | --t=* | -t=*)
            TEMPDIR=$ac_optarg ;;

        -u | --u | -unpack | --unpack)
            ac_prev=MUNPACK ;;
        -u=* | --u=* | -unpack=* | --unpack=*)
            MUNPACK=$ac_optarg ;;

        -a | --a | -addr | --addr)
            ac_prev=ADDRERR ;;
        -a=* | --a=* | -addr=* | --addr=*)
            ADDRERR=$ac_optarg ;;

        *)
	    # copy the file list to FL_#
	    date
	    echo "Error: don't understand argument \"$ac_option\""
	    usage
            ;;

     esac

done

if test ! -x $MUNPACK ; then
	date
	echo "ERROR: \"$MUNPACK\" is not executable"
	exit 2
fi
if test "x$OUTDIR" = x ; then
  OUTDIR="$BASEDIR/$monthDir"
else
  monthDir=0
fi
if test ! -e $OUTDIR ; then
    mkdir $OUTDIR
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
	date
	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
	exit 3
    else
	echo "Successfully created new dir: \"$OUTDIR\""
    fi
    chgrp gcmpack $OUTDIR
    chmod 775 $OUTDIR
fi

all_files=`ls -1 $INDIR`
nb_files=`echo "$all_files" | grep -c '^msg\.'`

if test $PRT = 1 ; then
  echo "Using OUTDIR=\"$OUTDIR\""
  echo "Using INDIR=\"$INDIR\""
  echo -n "Unpacking the emails ..."
elif test $nb_files != 0 ; then
  echo -n "Unpacking $nb_files emails ("`date`
  if test "x$ADDRERR" != x ; then
     echo -n ", err-msg: '$ADDRERR'"
  fi
  echo ")"
  echo " from '$INDIR' to '$OUTDIR'"
fi

for file in $all_files ; do

    #  create local copy
    test -e $TEMPDIR  &&  rm -rf $TEMPDIR
    mkdir $TEMPDIR
    RETVAL=$?
    if test "x$RETVAL" = x0 ; then
      cp $INDIR"/"$file $TEMPDIR
      RETVAL=$?
    fi
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo " processing file: '$INDIR/$file'" >> tmp.$$
        echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> tmp.$$
        echo " returns error $RETVAL" >> tmp.$$
        mail -s 'parse_emails err_0' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  ignore multi-part messages
    grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
    RETVAL=$?
    if test "x$RETVAL" = x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  munpack
    mun=`( cd $TEMPDIR ; $MUNPACK $file | cut -d ' ' -f 1 | head -1 )`
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  un-tar
    ( cd $TEMPDIR ; tar -xzvf $mun > out )
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        ls -l $mun >> tmp.$$
        mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi
    tdir=`cat $TEMPDIR"/out" | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
    rm -f $TEMPDIR"/out"

    #  select which Monthly Output Dir:
    locDir=$OUTDIR
    if test "x$monthDir" != x0 ; then
      dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
               | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
      mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
      if test "x$mn" != "x$monthDir" ; then
       if test "x$mn" = x ; then
        if test $PRT = 2 ; then echo " cannot get month from '$tdir'" ; fi
       else
       #  could comment out this line:
       #if test $PRT = 2 ; then echo " chg month: '$mn' for '$tdir'" ; fi
        locDir="$BASEDIR/$mn"
        if test ! -d $locDir ; then
          if test $PRT = 2 ; then echo "NO DIR: '$locDir' => '$tdir' POSTPONED" ; fi
          if test "x$ADDRERR" != x ; then
            echo "parsing email error" > tmp.$$
            echo "no dir '$locDir' for outp. '$tdir'" > tmp.$$
            ls -l $INDIR"/"$file >> tmp.$$
            mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
            rm -f tmp.$$
          fi
          continue
        fi
       fi
      fi
    fi

    #  copy to $locDir and rename if necessary
    sdir=$tdir
    if test -e $locDir"/"$tdir ; then
	ad=0
	while test -e $locDir"/"$tdir"_"$ad ; do
	    ad=$(( $ad + 1 ))
	done
        sdir=$tdir"_"$ad
    fi
    if test $PRT = 2 ; then
      if test "x$locDir" = "x$OUTDIR"
      then echo " '$sdir'"
      else echo " '$sdir' => '$locDir'"
      fi
    fi
    mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "mv $TEMPDIR/$tdir $locDir/$sdir returns error:" $RETVAL >> tmp.$$
        echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR  >> tmp.$$
        echo -n "in dir: $OUTDIR : " ; ls -l $locDir  >> tmp.$$
        mail -s 'parse_emails err_5' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi
    chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1

    #  remove the original file
    rm -f $INDIR"/"$file

done
if test $PRT = 1 ; then echo "  done" ; fi

1	edhill	1.1	#! /usr/bin/env bash
2
3	jmc	1.20	# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.19 2009/02/02 19:16:45 jmc Exp $
4	edhill	1.1	#
5			# The purpose of this script is to parse the emails produced by the
6			# MITgcm/verificaton/testreport script and store the data in a
7			# reasonable location.
8
9
10			usage()
11			{
12			echo
13			echo "Usage: $0 [OPTIONS]"
14	jmc	1.19	echo
15	edhill	1.1	echo "where possible OPTIONS are:"
16	jmc	1.10	echo " (-h\|-help) print usage"
17			echo " (-s\|-silent) silent mode"
18			echo " (-v\|-verbose) verbose mode"
19			echo " (-i \|-ind )DIR get mpack-created emails from DIR"
20	edhill	1.1	echo " [def=\"$INDIR\"]"
21	jmc	1.10	echo " (-o \|-outd )DIR write the data to DIR"
22	jmc	1.19	echo " [def=\"$BASEDIR/$monthDir\"]"
23	jmc	1.10	echo " (-t \|-tempd )DIR use temporary directory DIR"
24	edhill	1.1	echo " [def=\"$TEMPDIR\"]"
25	jmc	1.14	echo " (-u \|-unpack )EXE use executable EXE to unpack e-mails"
26			echo " [def=\"$MUNPACK\"]"
27	jmc	1.12	echo " (-a \|-addr )ADDR send e-mail to ADDR if Error"
28			echo " [def='"$ADDRERR"']"
29	jmc	1.19	echo
30	edhill	1.1	exit 1
31			}
32
33			# defaults
34	jmc	1.12	INDIR="/u/u2/jmc/Mail/MITgcm-test"
35	jmc	1.19	BASEDIR="/u/u0/httpd/html/testing/results"
36			monthDir=`date +%Y`"_"`date +%m`
37			OUTDIR=
38	edhill	1.1	TEMPDIR=./ptmp
39	jmc	1.14	MUNPACK=munpack
40	jmc	1.12	ADDRERR=
41	jmc	1.10	PRT=1
42	edhill	1.1
43			# Parse options
44			ac_prev=
45			for ac_option ; do
46
47			# If the previous option needs an argument, assign it.
48			if test -n "$ac_prev"; then
49			eval "$ac_prev=\$ac_option"
50			ac_prev=
51			continue
52			fi
53
54			ac_optarg=`expr "x$ac_option" : 'x[^=]=\(.\)'`
55	jmc	1.19
56	edhill	1.1	case $ac_option in
57
58			-help \| --help \| -h \| --h)
59			usage ;;
60	jmc	1.10	-s \| --s \| -silent \| --silent)
61			PRT=0 ;;
62			-v \| --v \| -verbose \| --verbose)
63			PRT=2 ;;
64	edhill	1.1
65			-ind \| --ind \| -i \| --i)
66			ac_prev=INDIR ;;
67			--ind=* \| -ind=* \| --i=* \| -i=*)
68			INDIR=$ac_optarg ;;
69	jmc	1.19
70	edhill	1.1	-outd \| --outd \| -o \| --o)
71			ac_prev=OUTDIR ;;
72			--outd=* \| -outd=* \| --o=* \| -o=*)
73			OUTDIR=$ac_optarg ;;
74	jmc	1.19
75	edhill	1.1	-tempd \| --tempd \| -t \| --t)
76			ac_prev=TEMPDIR ;;
77			--tempd=* \| -tempd=* \| --t=* \| -t=*)
78			TEMPDIR=$ac_optarg ;;
79	jmc	1.14
80			-u \| --u \| -unpack \| --unpack)
81			ac_prev=MUNPACK ;;
82			-u=* \| --u=* \| -unpack=* \| --unpack=*)
83			MUNPACK=$ac_optarg ;;
84	jmc	1.19
85	jmc	1.14	-a \| --a \| -addr \| --addr)
86	jmc	1.12	ac_prev=ADDRERR ;;
87	jmc	1.14	-a=* \| --a=* \| -addr=* \| --addr=*)
88	jmc	1.12	ADDRERR=$ac_optarg ;;
89	jmc	1.14
90	edhill	1.1	*)
91			# copy the file list to FL_#
92	jmc	1.18	date
93	edhill	1.1	echo "Error: don't understand argument \"$ac_option\""
94			usage
95			;;
96	jmc	1.19
97	edhill	1.1	esac
98	jmc	1.19
99	edhill	1.1	done
100
101	jmc	1.14	if test ! -x $MUNPACK ; then
102	jmc	1.18	date
103	jmc	1.14	echo "ERROR: \"$MUNPACK\" is not executable"
104			exit 2
105			fi
106	jmc	1.19	if test "x$OUTDIR" = x ; then
107			OUTDIR="$BASEDIR/$monthDir"
108			else
109			monthDir=0
110			fi
111	edhill	1.1	if test ! -e $OUTDIR ; then
112			mkdir $OUTDIR
113			RETVAL=$?
114	jmc	1.15	if test "x$RETVAL" != x0 ; then
115	jmc	1.18	date
116	edhill	1.1	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
117	jmc	1.14	exit 3
118	jmc	1.20	else
119			echo "Successfully created new dir: \"$OUTDIR\""
120	edhill	1.1	fi
121	jmc	1.8	chgrp gcmpack $OUTDIR
122			chmod 775 $OUTDIR
123	edhill	1.1	fi
124
125	jmc	1.10	all_files=`ls -1 $INDIR`
126	jmc	1.11	nb_files=`echo "$all_files" \| grep -c '^msg\.'`
127	edhill	1.1
128	jmc	1.10	if test $PRT = 1 ; then
129			echo "Using OUTDIR=\"$OUTDIR\""
130			echo "Using INDIR=\"$INDIR\""
131			echo -n "Unpacking the emails ..."
132			elif test $nb_files != 0 ; then
133	jmc	1.17	echo -n "Unpacking $nb_files emails ("`date`
134	jmc	1.12	if test "x$ADDRERR" != x ; then
135	jmc	1.17	echo -n ", err-msg: '$ADDRERR'"
136	jmc	1.12	fi
137	jmc	1.17	echo ")"
138			echo " from '$INDIR' to '$OUTDIR'"
139	jmc	1.10	fi
140	edhill	1.1
141			for file in $all_files ; do
142
143			# create local copy
144	edhill	1.2	test -e $TEMPDIR && rm -rf $TEMPDIR
145	edhill	1.1	mkdir $TEMPDIR
146	jmc	1.15	RETVAL=$?
147			if test "x$RETVAL" = x0 ; then
148			cp $INDIR"/"$file $TEMPDIR
149			RETVAL=$?
150			fi
151			if test "x$RETVAL" != x0 ; then
152			if test "x$ADDRERR" != x ; then
153			echo "parsing email error" > tmp.$$
154			echo " processing file: '$INDIR/$file'" >> tmp.$$
155			echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> tmp.$$
156			echo " returns error $RETVAL" >> tmp.$$
157			mail -s 'parse_emails err_0' $ADDRERR < tmp.$$
158			rm -f tmp.$$
159			fi
160			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
161			continue
162			fi
163	edhill	1.1
164			# ignore multi-part messages
165			grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
166			RETVAL=$?
167			if test "x$RETVAL" = x0 ; then
168	jmc	1.12	if test "x$ADDRERR" != x ; then
169			echo "parsing email error" > tmp.$$
170			echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
171			ls -l $INDIR"/"$file >> tmp.$$
172			mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
173			rm -f tmp.$$
174			fi
175			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
176			continue
177	edhill	1.1	fi
178
179			# munpack
180			mun=`( cd $TEMPDIR ; $MUNPACK $file \| cut -d ' ' -f 1 \| head -1 )`
181			RETVAL=$?
182			if test "x$RETVAL" != x0 ; then
183	jmc	1.12	if test "x$ADDRERR" != x ; then
184			echo "parsing email error" > tmp.$$
185			echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
186			ls -l $INDIR"/"$file >> tmp.$$
187			mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
188			rm -f tmp.$$
189			fi
190			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
191			continue
192	edhill	1.1	fi
193
194			# un-tar
195			( cd $TEMPDIR ; tar -xzvf $mun > out )
196			RETVAL=$?
197			if test "x$RETVAL" != x0 ; then
198	jmc	1.12	if test "x$ADDRERR" != x ; then
199			echo "parsing email error" > tmp.$$
200			echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
201			ls -l $INDIR"/"$file >> tmp.$$
202			ls -l $mun >> tmp.$$
203			mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
204			rm -f tmp.$$
205			fi
206			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
207			continue
208	edhill	1.1	fi
209	edhill	1.2	tdir=`cat $TEMPDIR"/out" \| head -1 \| sed -e 's\|^./\|\|g' \| cut -d '/' -f 1`
210	edhill	1.1	rm -f $TEMPDIR"/out"
211
212	jmc	1.19	# select which Monthly Output Dir:
213			locDir=$OUTDIR
214			if test "x$monthDir" != x0 ; then
215			dd=`echo $tdir \| sed 's/_/ /g' \| awk '{ for(i=1;i<=NF;i++) print $i }'\
216			\| grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' \| tail -1`
217			mn=`echo $dd \| sed 's/..$//' \| sed 's/..$/_&/'`
218			if test "x$mn" != "x$monthDir" ; then
219			if test "x$mn" = x ; then
220			if test $PRT = 2 ; then echo " cannot get month from '$tdir'" ; fi
221			else
222			# could comment out this line:
223			#if test $PRT = 2 ; then echo " chg month: '$mn' for '$tdir'" ; fi
224			locDir="$BASEDIR/$mn"
225			if test ! -d $locDir ; then
226			if test $PRT = 2 ; then echo "NO DIR: '$locDir' => '$tdir' POSTPONED" ; fi
227			if test "x$ADDRERR" != x ; then
228			echo "parsing email error" > tmp.$$
229			echo "no dir '$locDir' for outp. '$tdir'" > tmp.$$
230			ls -l $INDIR"/"$file >> tmp.$$
231			mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
232			rm -f tmp.$$
233			fi
234			continue
235			fi
236			fi
237			fi
238			fi
239
240			# copy to $locDir and rename if necessary
241	jmc	1.10	sdir=$tdir
242	jmc	1.19	if test -e $locDir"/"$tdir ; then
243	edhill	1.1	ad=0
244	jmc	1.19	while test -e $locDir"/"$tdir"_"$ad ; do
245	edhill	1.1	ad=$(( $ad + 1 ))
246			done
247	jmc	1.10	sdir=$tdir"_"$ad
248	edhill	1.1	fi
249	jmc	1.19	if test $PRT = 2 ; then
250			if test "x$locDir" = "x$OUTDIR"
251			then echo " '$sdir'"
252			else echo " '$sdir' => '$locDir'"
253			fi
254			fi
255			mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
256	jmc	1.15	RETVAL=$?
257			if test "x$RETVAL" != x0 ; then
258			if test "x$ADDRERR" != x ; then
259			echo "parsing email error" > tmp.$$
260	jmc	1.19	echo "mv $TEMPDIR/$tdir $locDir/$sdir returns error:" $RETVAL >> tmp.$$
261	jmc	1.15	echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR >> tmp.$$
262	jmc	1.19	echo -n "in dir: $OUTDIR : " ; ls -l $locDir >> tmp.$$
263			mail -s 'parse_emails err_5' $ADDRERR < tmp.$$
264	jmc	1.15	rm -f tmp.$$
265			fi
266			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
267			continue
268			fi
269	jmc	1.19	chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
270	edhill	1.1
271			# remove the original file
272			rm -f $INDIR"/"$file
273
274			done
275	jmc	1.10	if test $PRT = 1 ; then echo " done" ; fi
276	edhill	1.1