mitgcm.org/front_content/parse_emails

#! /usr/bin/env bash

# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.17 2008/03/19 00:55:28 jmc Exp $
#
#  The purpose of this script is to parse the emails produced by the
#  MITgcm/verificaton/testreport script and store the data in a
#  reasonable location.


usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo 
    echo "where possible OPTIONS are:"
    echo "  (-h|-help)           print usage"
    echo "  (-s|-silent)         silent mode"
    echo "  (-v|-verbose)        verbose mode"
    echo "  (-i |-ind )DIR       get mpack-created emails from DIR"
    echo "                         [def=\"$INDIR\"]"
    echo "  (-o |-outd )DIR      write the data to DIR"
    echo "                         [def=\"$OUTDIR\"]"
    echo "  (-t |-tempd )DIR     use temporary directory DIR"
    echo "                         [def=\"$TEMPDIR\"]"
    echo "  (-u |-unpack )EXE    use executable EXE to unpack e-mails"
    echo "                         [def=\"$MUNPACK\"]"
    echo "  (-a |-addr )ADDR     send e-mail to ADDR if Error"
    echo "                         [def='"$ADDRERR"']"
    echo 
    exit 1
}

# defaults
INDIR="/u/u2/jmc/Mail/MITgcm-test"
OUTDIR="/u/u0/httpd/html/testing/results/"`date +%Y`"_"`date +%m`
TEMPDIR=./ptmp
MUNPACK=munpack
ADDRERR=
PRT=1

#  Parse options
ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
    
    case $ac_option in
	
	-help | --help | -h | --h)
	    usage ;;
	-s | --s | -silent | --silent)
            PRT=0 ;;
	-v | --v | -verbose | --verbose)
            PRT=2 ;;
	
        -ind | --ind | -i | --i)
            ac_prev=INDIR ;;
        --ind=* | -ind=* | --i=* | -i=*)
            INDIR=$ac_optarg ;;
        
        -outd | --outd | -o | --o)
            ac_prev=OUTDIR ;;
        --outd=* | -outd=* | --o=* | -o=*)
            OUTDIR=$ac_optarg ;;
        
        -tempd | --tempd | -t | --t)
            ac_prev=TEMPDIR ;;
        --tempd=* | -tempd=* | --t=* | -t=*)
            TEMPDIR=$ac_optarg ;;

        -u | --u | -unpack | --unpack)
            ac_prev=MUNPACK ;;
        -u=* | --u=* | -unpack=* | --unpack=*)
            MUNPACK=$ac_optarg ;;
        
        -a | --a | -addr | --addr)
            ac_prev=ADDRERR ;;
        -a=* | --a=* | -addr=* | --addr=*)
            ADDRERR=$ac_optarg ;;

        *)
	    # copy the file list to FL_#
	    date
	    echo "Error: don't understand argument \"$ac_option\""
	    usage
            ;;
        
     esac
     
done

if test ! -x $MUNPACK ; then
	date
	echo "ERROR: \"$MUNPACK\" is not executable"
	exit 2
fi
if test ! -e $OUTDIR ; then
    mkdir $OUTDIR
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
	date
	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
	exit 3
    fi
    chgrp gcmpack $OUTDIR
    chmod 775 $OUTDIR
fi

all_files=`ls -1 $INDIR`
nb_files=`echo "$all_files" | grep -c '^msg\.'`

if test $PRT = 1 ; then
  echo "Using OUTDIR=\"$OUTDIR\""
  echo "Using INDIR=\"$INDIR\""
  echo -n "Unpacking the emails ..."
elif test $nb_files != 0 ; then
  echo -n "Unpacking $nb_files emails ("`date`
  if test "x$ADDRERR" != x ; then
     echo -n ", err-msg: '$ADDRERR'"
  fi
  echo ")"
  echo " from '$INDIR' to '$OUTDIR'"
fi

for file in $all_files ; do

    #  create local copy
    test -e $TEMPDIR  &&  rm -rf $TEMPDIR
    mkdir $TEMPDIR
    RETVAL=$?
    if test "x$RETVAL" = x0 ; then
      cp $INDIR"/"$file $TEMPDIR
      RETVAL=$?
    fi
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo " processing file: '$INDIR/$file'" >> tmp.$$
        echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> tmp.$$
        echo " returns error $RETVAL" >> tmp.$$
        mail -s 'parse_emails err_0' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  ignore multi-part messages
    grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
    RETVAL=$?
    if test "x$RETVAL" = x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  munpack
    mun=`( cd $TEMPDIR ; $MUNPACK $file | cut -d ' ' -f 1 | head -1 )`
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  un-tar
    ( cd $TEMPDIR ; tar -xzvf $mun > out )
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        ls -l $mun >> tmp.$$
        mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi
    tdir=`cat $TEMPDIR"/out" | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
    rm -f $TEMPDIR"/out"

    #  copy to $OUTDIR and rename if necessary
    sdir=$tdir
    if test -e $OUTDIR"/"$tdir ; then
	ad=0
	while test -e $OUTDIR"/"$tdir"_"$ad ; do
	    ad=$(( $ad + 1 ))
	done
        sdir=$tdir"_"$ad
    fi
    if test $PRT = 2 ; then echo " '$sdir'" ; fi
    mv $TEMPDIR"/"$tdir $OUTDIR"/"$sdir > /dev/null 2>&1
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "mv $TEMPDIR/$tdir $OUTDIR/$sdir returns error:" $RETVAL >> tmp.$$
        echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR  >> tmp.$$
        echo -n "in dir: $OUTDIR : " ; ls -l $OUTDIR  >> tmp.$$
        mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi
    chmod -R a+rx $OUTDIR"/"$sdir > /dev/null 2>&1
#   gzip $OUTDIR"/"$sdir"/output.txt"

    #  remove the original file
    rm -f $INDIR"/"$file

done
if test $PRT = 1 ; then echo "  done" ; fi

1	edhill	1.1	#! /usr/bin/env bash
2
3	jmc	1.18	# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.17 2008/03/19 00:55:28 jmc Exp $
4	edhill	1.1	#
5			# The purpose of this script is to parse the emails produced by the
6			# MITgcm/verificaton/testreport script and store the data in a
7			# reasonable location.
8
9
10			usage()
11			{
12			echo
13			echo "Usage: $0 [OPTIONS]"
14			echo
15			echo "where possible OPTIONS are:"
16	jmc	1.10	echo " (-h\|-help) print usage"
17			echo " (-s\|-silent) silent mode"
18			echo " (-v\|-verbose) verbose mode"
19			echo " (-i \|-ind )DIR get mpack-created emails from DIR"
20	edhill	1.1	echo " [def=\"$INDIR\"]"
21	jmc	1.10	echo " (-o \|-outd )DIR write the data to DIR"
22	edhill	1.1	echo " [def=\"$OUTDIR\"]"
23	jmc	1.10	echo " (-t \|-tempd )DIR use temporary directory DIR"
24	edhill	1.1	echo " [def=\"$TEMPDIR\"]"
25	jmc	1.14	echo " (-u \|-unpack )EXE use executable EXE to unpack e-mails"
26			echo " [def=\"$MUNPACK\"]"
27	jmc	1.12	echo " (-a \|-addr )ADDR send e-mail to ADDR if Error"
28			echo " [def='"$ADDRERR"']"
29	edhill	1.1	echo
30			exit 1
31			}
32
33			# defaults
34	jmc	1.12	INDIR="/u/u2/jmc/Mail/MITgcm-test"
35			OUTDIR="/u/u0/httpd/html/testing/results/"`date +%Y`"_"`date +%m`
36	edhill	1.1	TEMPDIR=./ptmp
37	jmc	1.14	MUNPACK=munpack
38	jmc	1.12	ADDRERR=
39	jmc	1.10	PRT=1
40	edhill	1.1
41			# Parse options
42			ac_prev=
43			for ac_option ; do
44
45			# If the previous option needs an argument, assign it.
46			if test -n "$ac_prev"; then
47			eval "$ac_prev=\$ac_option"
48			ac_prev=
49			continue
50			fi
51
52			ac_optarg=`expr "x$ac_option" : 'x[^=]=\(.\)'`
53
54			case $ac_option in
55
56			-help \| --help \| -h \| --h)
57			usage ;;
58	jmc	1.10	-s \| --s \| -silent \| --silent)
59			PRT=0 ;;
60			-v \| --v \| -verbose \| --verbose)
61			PRT=2 ;;
62	edhill	1.1
63			-ind \| --ind \| -i \| --i)
64			ac_prev=INDIR ;;
65			--ind=* \| -ind=* \| --i=* \| -i=*)
66			INDIR=$ac_optarg ;;
67
68			-outd \| --outd \| -o \| --o)
69			ac_prev=OUTDIR ;;
70			--outd=* \| -outd=* \| --o=* \| -o=*)
71			OUTDIR=$ac_optarg ;;
72
73			-tempd \| --tempd \| -t \| --t)
74			ac_prev=TEMPDIR ;;
75			--tempd=* \| -tempd=* \| --t=* \| -t=*)
76			TEMPDIR=$ac_optarg ;;
77	jmc	1.14
78			-u \| --u \| -unpack \| --unpack)
79			ac_prev=MUNPACK ;;
80			-u=* \| --u=* \| -unpack=* \| --unpack=*)
81			MUNPACK=$ac_optarg ;;
82
83			-a \| --a \| -addr \| --addr)
84	jmc	1.12	ac_prev=ADDRERR ;;
85	jmc	1.14	-a=* \| --a=* \| -addr=* \| --addr=*)
86	jmc	1.12	ADDRERR=$ac_optarg ;;
87	jmc	1.14
88	edhill	1.1	*)
89			# copy the file list to FL_#
90	jmc	1.18	date
91	edhill	1.1	echo "Error: don't understand argument \"$ac_option\""
92			usage
93			;;
94
95			esac
96
97			done
98
99	jmc	1.14	if test ! -x $MUNPACK ; then
100	jmc	1.18	date
101	jmc	1.14	echo "ERROR: \"$MUNPACK\" is not executable"
102			exit 2
103			fi
104	edhill	1.1	if test ! -e $OUTDIR ; then
105			mkdir $OUTDIR
106			RETVAL=$?
107	jmc	1.15	if test "x$RETVAL" != x0 ; then
108	jmc	1.18	date
109	edhill	1.1	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
110	jmc	1.14	exit 3
111	edhill	1.1	fi
112	jmc	1.8	chgrp gcmpack $OUTDIR
113			chmod 775 $OUTDIR
114	edhill	1.1	fi
115
116	jmc	1.10	all_files=`ls -1 $INDIR`
117	jmc	1.11	nb_files=`echo "$all_files" \| grep -c '^msg\.'`
118	edhill	1.1
119	jmc	1.10	if test $PRT = 1 ; then
120			echo "Using OUTDIR=\"$OUTDIR\""
121			echo "Using INDIR=\"$INDIR\""
122			echo -n "Unpacking the emails ..."
123			elif test $nb_files != 0 ; then
124	jmc	1.17	echo -n "Unpacking $nb_files emails ("`date`
125	jmc	1.12	if test "x$ADDRERR" != x ; then
126	jmc	1.17	echo -n ", err-msg: '$ADDRERR'"
127	jmc	1.12	fi
128	jmc	1.17	echo ")"
129			echo " from '$INDIR' to '$OUTDIR'"
130	jmc	1.10	fi
131	edhill	1.1
132			for file in $all_files ; do
133
134			# create local copy
135	edhill	1.2	test -e $TEMPDIR && rm -rf $TEMPDIR
136	edhill	1.1	mkdir $TEMPDIR
137	jmc	1.15	RETVAL=$?
138			if test "x$RETVAL" = x0 ; then
139			cp $INDIR"/"$file $TEMPDIR
140			RETVAL=$?
141			fi
142			if test "x$RETVAL" != x0 ; then
143			if test "x$ADDRERR" != x ; then
144			echo "parsing email error" > tmp.$$
145			echo " processing file: '$INDIR/$file'" >> tmp.$$
146			echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> tmp.$$
147			echo " returns error $RETVAL" >> tmp.$$
148			mail -s 'parse_emails err_0' $ADDRERR < tmp.$$
149			rm -f tmp.$$
150			fi
151			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
152			continue
153			fi
154	edhill	1.1
155			# ignore multi-part messages
156			grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
157			RETVAL=$?
158			if test "x$RETVAL" = x0 ; then
159	jmc	1.12	if test "x$ADDRERR" != x ; then
160			echo "parsing email error" > tmp.$$
161			echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
162			ls -l $INDIR"/"$file >> tmp.$$
163			mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
164			rm -f tmp.$$
165			fi
166			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
167			continue
168	edhill	1.1	fi
169
170			# munpack
171			mun=`( cd $TEMPDIR ; $MUNPACK $file \| cut -d ' ' -f 1 \| head -1 )`
172			RETVAL=$?
173			if test "x$RETVAL" != x0 ; then
174	jmc	1.12	if test "x$ADDRERR" != x ; then
175			echo "parsing email error" > tmp.$$
176			echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
177			ls -l $INDIR"/"$file >> tmp.$$
178			mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
179			rm -f tmp.$$
180			fi
181			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
182			continue
183	edhill	1.1	fi
184
185			# un-tar
186			( cd $TEMPDIR ; tar -xzvf $mun > out )
187			RETVAL=$?
188			if test "x$RETVAL" != x0 ; then
189	jmc	1.12	if test "x$ADDRERR" != x ; then
190			echo "parsing email error" > tmp.$$
191			echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
192			ls -l $INDIR"/"$file >> tmp.$$
193			ls -l $mun >> tmp.$$
194			mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
195			rm -f tmp.$$
196			fi
197			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
198			continue
199	edhill	1.1	fi
200	edhill	1.2	tdir=`cat $TEMPDIR"/out" \| head -1 \| sed -e 's\|^./\|\|g' \| cut -d '/' -f 1`
201	edhill	1.1	rm -f $TEMPDIR"/out"
202
203			# copy to $OUTDIR and rename if necessary
204	jmc	1.10	sdir=$tdir
205	edhill	1.1	if test -e $OUTDIR"/"$tdir ; then
206			ad=0
207			while test -e $OUTDIR"/"$tdir"_"$ad ; do
208			ad=$(( $ad + 1 ))
209			done
210	jmc	1.10	sdir=$tdir"_"$ad
211	edhill	1.1	fi
212	jmc	1.17	if test $PRT = 2 ; then echo " '$sdir'" ; fi
213	jmc	1.10	mv $TEMPDIR"/"$tdir $OUTDIR"/"$sdir > /dev/null 2>&1
214	jmc	1.15	RETVAL=$?
215			if test "x$RETVAL" != x0 ; then
216			if test "x$ADDRERR" != x ; then
217			echo "parsing email error" > tmp.$$
218			echo "mv $TEMPDIR/$tdir $OUTDIR/$sdir returns error:" $RETVAL >> tmp.$$
219			echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR >> tmp.$$
220			echo -n "in dir: $OUTDIR : " ; ls -l $OUTDIR >> tmp.$$
221			mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
222			rm -f tmp.$$
223			fi
224			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
225			continue
226			fi
227	jmc	1.10	chmod -R a+rx $OUTDIR"/"$sdir > /dev/null 2>&1
228			# gzip $OUTDIR"/"$sdir"/output.txt"
229	edhill	1.1
230			# remove the original file
231			rm -f $INDIR"/"$file
232
233			done
234	jmc	1.10	if test $PRT = 1 ; then echo " done" ; fi
235	edhill	1.1