mitgcm.org/front_content/parse_emails

#! /usr/bin/env bash

# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.12 2008/02/05 22:58:54 jmc Exp $
#
#  The purpose of this script is to parse the emails produced by the
#  MITgcm/verificaton/testreport script and store the data in a
#  reasonable location.


usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo 
    echo "where possible OPTIONS are:"
    echo "  (-h|-help)           print usage"
    echo "  (-s|-silent)         silent mode"
    echo "  (-v|-verbose)        verbose mode"
    echo "  (-i |-ind )DIR       get mpack-created emails from DIR"
    echo "                         [def=\"$INDIR\"]"
    echo "  (-o |-outd )DIR      write the data to DIR"
    echo "                         [def=\"$OUTDIR\"]"
    echo "  (-t |-tempd )DIR     use temporary directory DIR"
    echo "                         [def=\"$TEMPDIR\"]"
    echo "  (-a |-addr )ADDR     send e-mail to ADDR if Error"
    echo "                         [def='"$ADDRERR"']"
    echo 
    exit 1
}

# defaults
INDIR="/u/u2/jmc/Mail/MITgcm-test"
OUTDIR="/u/u0/httpd/html/testing/results/"`date +%Y`"_"`date +%m`
TEMPDIR=./ptmp
MUNPACK=/u/u2/jmc/testing/tools_mpack/munpack
ADDRERR=
PRT=1

#  Parse options
ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
        eval "$ac_prev=\$ac_option"
        ac_prev=
        continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
    
    case $ac_option in
	
	-help | --help | -h | --h)
	    usage ;;
	-s | --s | -silent | --silent)
            PRT=0 ;;
	-v | --v | -verbose | --verbose)
            PRT=2 ;;
	
        -ind | --ind | -i | --i)
            ac_prev=INDIR ;;
        --ind=* | -ind=* | --i=* | -i=*)
            INDIR=$ac_optarg ;;
        
        -outd | --outd | -o | --o)
            ac_prev=OUTDIR ;;
        --outd=* | -outd=* | --o=* | -o=*)
            OUTDIR=$ac_optarg ;;
        
        -tempd | --tempd | -t | --t)
            ac_prev=TEMPDIR ;;
        --tempd=* | -tempd=* | --t=* | -t=*)
            TEMPDIR=$ac_optarg ;;
        -a | --aa | -addr | --addr)
            ac_prev=ADDRERR ;;
        -a=* | --aa=* | -addr=* | --addr=*)
            ADDRERR=$ac_optarg ;;
        
        *)
	    # copy the file list to FL_#
	    echo "Error: don't understand argument \"$ac_option\""
	    usage
            ;;
        
     esac
     
done

if test ! -e $OUTDIR ; then
    mkdir $OUTDIR
    RETVAL=$?
    if test "x$RETVAL" = x ; then
	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
	exit 1
    fi
    chgrp gcmpack $OUTDIR
    chmod 775 $OUTDIR
fi

all_files=`ls -1 $INDIR`
nb_files=`echo "$all_files" | grep -c '^msg\.'`

if test $PRT = 1 ; then
  echo "Using OUTDIR=\"$OUTDIR\""
  echo "Using INDIR=\"$INDIR\""
  echo -n "Unpacking the emails ..."
elif test $nb_files != 0 ; then
  echo "Unpacking $nb_files emails from '$INDIR' to '$OUTDIR'"
  if test "x$ADDRERR" != x ; then
     echo " (send msg to '$ADDRERR' if Error)"
  fi
  if test $PRT = 2 ; then echo -n " dir:" ; fi
fi

for file in $all_files ; do

    #  create local copy
    test -e $TEMPDIR  &&  rm -rf $TEMPDIR
    mkdir $TEMPDIR
    cp $INDIR"/"$file $TEMPDIR

    #  ignore multi-part messages
    grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
    RETVAL=$?
    if test "x$RETVAL" = x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  munpack
    mun=`( cd $TEMPDIR ; $MUNPACK $file | cut -d ' ' -f 1 | head -1 )`
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi

    #  un-tar
    ( cd $TEMPDIR ; tar -xzvf $mun > out )
    RETVAL=$?
    if test "x$RETVAL" != x0 ; then
      if test "x$ADDRERR" != x ; then
        echo "parsing email error" > tmp.$$
        echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
        ls -l $INDIR"/"$file >> tmp.$$
        ls -l $mun >> tmp.$$
        mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
        rm -f tmp.$$
      fi
      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
      continue
    fi
    tdir=`cat $TEMPDIR"/out" | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
    rm -f $TEMPDIR"/out"

    #  copy to $OUTDIR and rename if necessary
    sdir=$tdir
    if test -e $OUTDIR"/"$tdir ; then
	ad=0
	while test -e $OUTDIR"/"$tdir"_"$ad ; do
	    ad=$(( $ad + 1 ))
	done
        sdir=$tdir"_"$ad
    fi
    if test $PRT = 2 ; then echo -n " '$sdir'" ; fi
    mv $TEMPDIR"/"$tdir $OUTDIR"/"$sdir > /dev/null 2>&1
    chmod -R a+rx $OUTDIR"/"$sdir > /dev/null 2>&1
#   gzip $OUTDIR"/"$sdir"/output.txt"

    #  remove the original file
    rm -f $INDIR"/"$file

done
if test $PRT = 1 ; then echo "  done" ; fi
if test $PRT = 2 -a $nb_files != 0 ; then echo "" ; fi

1	edhill	1.1	#! /usr/bin/env bash
2
3	jmc	1.13	# $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.12 2008/02/05 22:58:54 jmc Exp $
4	edhill	1.1	#
5			# The purpose of this script is to parse the emails produced by the
6			# MITgcm/verificaton/testreport script and store the data in a
7			# reasonable location.
8
9
10			usage()
11			{
12			echo
13			echo "Usage: $0 [OPTIONS]"
14			echo
15			echo "where possible OPTIONS are:"
16	jmc	1.10	echo " (-h\|-help) print usage"
17			echo " (-s\|-silent) silent mode"
18			echo " (-v\|-verbose) verbose mode"
19			echo " (-i \|-ind )DIR get mpack-created emails from DIR"
20	edhill	1.1	echo " [def=\"$INDIR\"]"
21	jmc	1.10	echo " (-o \|-outd )DIR write the data to DIR"
22	edhill	1.1	echo " [def=\"$OUTDIR\"]"
23	jmc	1.10	echo " (-t \|-tempd )DIR use temporary directory DIR"
24	edhill	1.1	echo " [def=\"$TEMPDIR\"]"
25	jmc	1.12	echo " (-a \|-addr )ADDR send e-mail to ADDR if Error"
26			echo " [def='"$ADDRERR"']"
27	edhill	1.1	echo
28			exit 1
29			}
30
31			# defaults
32	jmc	1.12	INDIR="/u/u2/jmc/Mail/MITgcm-test"
33			OUTDIR="/u/u0/httpd/html/testing/results/"`date +%Y`"_"`date +%m`
34	edhill	1.1	TEMPDIR=./ptmp
35	jmc	1.13	MUNPACK=/u/u2/jmc/testing/tools_mpack/munpack
36	jmc	1.12	ADDRERR=
37	jmc	1.10	PRT=1
38	edhill	1.1
39			# Parse options
40			ac_prev=
41			for ac_option ; do
42
43			# If the previous option needs an argument, assign it.
44			if test -n "$ac_prev"; then
45			eval "$ac_prev=\$ac_option"
46			ac_prev=
47			continue
48			fi
49
50			ac_optarg=`expr "x$ac_option" : 'x[^=]=\(.\)'`
51
52			case $ac_option in
53
54			-help \| --help \| -h \| --h)
55			usage ;;
56	jmc	1.10	-s \| --s \| -silent \| --silent)
57			PRT=0 ;;
58			-v \| --v \| -verbose \| --verbose)
59			PRT=2 ;;
60	edhill	1.1
61			-ind \| --ind \| -i \| --i)
62			ac_prev=INDIR ;;
63			--ind=* \| -ind=* \| --i=* \| -i=*)
64			INDIR=$ac_optarg ;;
65
66			-outd \| --outd \| -o \| --o)
67			ac_prev=OUTDIR ;;
68			--outd=* \| -outd=* \| --o=* \| -o=*)
69			OUTDIR=$ac_optarg ;;
70
71			-tempd \| --tempd \| -t \| --t)
72			ac_prev=TEMPDIR ;;
73			--tempd=* \| -tempd=* \| --t=* \| -t=*)
74			TEMPDIR=$ac_optarg ;;
75	jmc	1.12	-a \| --aa \| -addr \| --addr)
76			ac_prev=ADDRERR ;;
77			-a=* \| --aa=* \| -addr=* \| --addr=*)
78			ADDRERR=$ac_optarg ;;
79	edhill	1.1
80			*)
81			# copy the file list to FL_#
82			echo "Error: don't understand argument \"$ac_option\""
83			usage
84			;;
85
86			esac
87
88			done
89
90			if test ! -e $OUTDIR ; then
91			mkdir $OUTDIR
92			RETVAL=$?
93			if test "x$RETVAL" = x ; then
94			echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
95			exit 1
96			fi
97	jmc	1.8	chgrp gcmpack $OUTDIR
98			chmod 775 $OUTDIR
99	edhill	1.1	fi
100
101	jmc	1.10	all_files=`ls -1 $INDIR`
102	jmc	1.11	nb_files=`echo "$all_files" \| grep -c '^msg\.'`
103	edhill	1.1
104	jmc	1.10	if test $PRT = 1 ; then
105			echo "Using OUTDIR=\"$OUTDIR\""
106			echo "Using INDIR=\"$INDIR\""
107			echo -n "Unpacking the emails ..."
108			elif test $nb_files != 0 ; then
109			echo "Unpacking $nb_files emails from '$INDIR' to '$OUTDIR'"
110	jmc	1.12	if test "x$ADDRERR" != x ; then
111			echo " (send msg to '$ADDRERR' if Error)"
112			fi
113	jmc	1.10	if test $PRT = 2 ; then echo -n " dir:" ; fi
114			fi
115	edhill	1.1
116			for file in $all_files ; do
117
118			# create local copy
119	edhill	1.2	test -e $TEMPDIR && rm -rf $TEMPDIR
120	edhill	1.1	mkdir $TEMPDIR
121			cp $INDIR"/"$file $TEMPDIR
122
123			# ignore multi-part messages
124			grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
125			RETVAL=$?
126			if test "x$RETVAL" = x0 ; then
127	jmc	1.12	if test "x$ADDRERR" != x ; then
128			echo "parsing email error" > tmp.$$
129			echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
130			ls -l $INDIR"/"$file >> tmp.$$
131			mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
132			rm -f tmp.$$
133			fi
134			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
135			continue
136	edhill	1.1	fi
137
138			# munpack
139			mun=`( cd $TEMPDIR ; $MUNPACK $file \| cut -d ' ' -f 1 \| head -1 )`
140			RETVAL=$?
141			if test "x$RETVAL" != x0 ; then
142	jmc	1.12	if test "x$ADDRERR" != x ; then
143			echo "parsing email error" > tmp.$$
144			echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
145			ls -l $INDIR"/"$file >> tmp.$$
146			mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
147			rm -f tmp.$$
148			fi
149			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
150			continue
151	edhill	1.1	fi
152
153			# un-tar
154			( cd $TEMPDIR ; tar -xzvf $mun > out )
155			RETVAL=$?
156			if test "x$RETVAL" != x0 ; then
157	jmc	1.12	if test "x$ADDRERR" != x ; then
158			echo "parsing email error" > tmp.$$
159			echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
160			ls -l $INDIR"/"$file >> tmp.$$
161			ls -l $mun >> tmp.$$
162			mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
163			rm -f tmp.$$
164			fi
165			mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
166			continue
167	edhill	1.1	fi
168	edhill	1.2	tdir=`cat $TEMPDIR"/out" \| head -1 \| sed -e 's\|^./\|\|g' \| cut -d '/' -f 1`
169	edhill	1.1	rm -f $TEMPDIR"/out"
170
171			# copy to $OUTDIR and rename if necessary
172	jmc	1.10	sdir=$tdir
173	edhill	1.1	if test -e $OUTDIR"/"$tdir ; then
174			ad=0
175			while test -e $OUTDIR"/"$tdir"_"$ad ; do
176			ad=$(( $ad + 1 ))
177			done
178	jmc	1.10	sdir=$tdir"_"$ad
179	edhill	1.1	fi
180	jmc	1.10	if test $PRT = 2 ; then echo -n " '$sdir'" ; fi
181			mv $TEMPDIR"/"$tdir $OUTDIR"/"$sdir > /dev/null 2>&1
182			chmod -R a+rx $OUTDIR"/"$sdir > /dev/null 2>&1
183			# gzip $OUTDIR"/"$sdir"/output.txt"
184	edhill	1.1
185			# remove the original file
186			rm -f $INDIR"/"$file
187
188			done
189	jmc	1.10	if test $PRT = 1 ; then echo " done" ; fi
190			if test $PRT = 2 -a $nb_files != 0 ; then echo "" ; fi
191	edhill	1.1