/[MITgcm]/mitgcm.org/front_content/parse_emails
ViewVC logotype

Annotation of /mitgcm.org/front_content/parse_emails

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.21 - (hide annotations) (download)
Thu Dec 3 18:03:27 2009 UTC (15 years, 7 months ago) by jmc
Branch: MAIN
Changes since 1.20: +27 -11 lines
- untar with option --exclude="._* to filter out small files that some
  MAC OS add to tar files.
- check that the output dir is really a directory

1 edhill 1.1 #! /usr/bin/env bash
2    
3 jmc 1.21 # $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.20 2009/10/01 15:52:13 jmc Exp $
4 edhill 1.1 #
5     # The purpose of this script is to parse the emails produced by the
6     # MITgcm/verificaton/testreport script and store the data in a
7     # reasonable location.
8    
9    
10     usage()
11     {
12     echo
13     echo "Usage: $0 [OPTIONS]"
14 jmc 1.19 echo
15 edhill 1.1 echo "where possible OPTIONS are:"
16 jmc 1.10 echo " (-h|-help) print usage"
17     echo " (-s|-silent) silent mode"
18     echo " (-v|-verbose) verbose mode"
19     echo " (-i |-ind )DIR get mpack-created emails from DIR"
20 edhill 1.1 echo " [def=\"$INDIR\"]"
21 jmc 1.10 echo " (-o |-outd )DIR write the data to DIR"
22 jmc 1.19 echo " [def=\"$BASEDIR/$monthDir\"]"
23 jmc 1.10 echo " (-t |-tempd )DIR use temporary directory DIR"
24 edhill 1.1 echo " [def=\"$TEMPDIR\"]"
25 jmc 1.14 echo " (-u |-unpack )EXE use executable EXE to unpack e-mails"
26     echo " [def=\"$MUNPACK\"]"
27 jmc 1.12 echo " (-a |-addr )ADDR send e-mail to ADDR if Error"
28     echo " [def='"$ADDRERR"']"
29 jmc 1.19 echo
30 edhill 1.1 exit 1
31     }
32    
33     # defaults
34 jmc 1.12 INDIR="/u/u2/jmc/Mail/MITgcm-test"
35 jmc 1.19 BASEDIR="/u/u0/httpd/html/testing/results"
36     monthDir=`date +%Y`"_"`date +%m`
37     OUTDIR=
38 edhill 1.1 TEMPDIR=./ptmp
39 jmc 1.14 MUNPACK=munpack
40 jmc 1.12 ADDRERR=
41 jmc 1.10 PRT=1
42 edhill 1.1
43     # Parse options
44     ac_prev=
45     for ac_option ; do
46    
47     # If the previous option needs an argument, assign it.
48     if test -n "$ac_prev"; then
49     eval "$ac_prev=\$ac_option"
50     ac_prev=
51     continue
52     fi
53    
54     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
55 jmc 1.19
56 edhill 1.1 case $ac_option in
57    
58     -help | --help | -h | --h)
59     usage ;;
60 jmc 1.10 -s | --s | -silent | --silent)
61     PRT=0 ;;
62     -v | --v | -verbose | --verbose)
63     PRT=2 ;;
64 edhill 1.1
65     -ind | --ind | -i | --i)
66     ac_prev=INDIR ;;
67     --ind=* | -ind=* | --i=* | -i=*)
68     INDIR=$ac_optarg ;;
69 jmc 1.19
70 edhill 1.1 -outd | --outd | -o | --o)
71     ac_prev=OUTDIR ;;
72     --outd=* | -outd=* | --o=* | -o=*)
73     OUTDIR=$ac_optarg ;;
74 jmc 1.19
75 edhill 1.1 -tempd | --tempd | -t | --t)
76     ac_prev=TEMPDIR ;;
77     --tempd=* | -tempd=* | --t=* | -t=*)
78     TEMPDIR=$ac_optarg ;;
79 jmc 1.14
80     -u | --u | -unpack | --unpack)
81     ac_prev=MUNPACK ;;
82     -u=* | --u=* | -unpack=* | --unpack=*)
83     MUNPACK=$ac_optarg ;;
84 jmc 1.19
85 jmc 1.14 -a | --a | -addr | --addr)
86 jmc 1.12 ac_prev=ADDRERR ;;
87 jmc 1.14 -a=* | --a=* | -addr=* | --addr=*)
88 jmc 1.12 ADDRERR=$ac_optarg ;;
89 jmc 1.14
90 edhill 1.1 *)
91     # copy the file list to FL_#
92 jmc 1.18 date
93 edhill 1.1 echo "Error: don't understand argument \"$ac_option\""
94     usage
95     ;;
96 jmc 1.19
97 edhill 1.1 esac
98 jmc 1.19
99 edhill 1.1 done
100    
101 jmc 1.14 if test ! -x $MUNPACK ; then
102 jmc 1.18 date
103 jmc 1.14 echo "ERROR: \"$MUNPACK\" is not executable"
104     exit 2
105     fi
106 jmc 1.19 if test "x$OUTDIR" = x ; then
107     OUTDIR="$BASEDIR/$monthDir"
108     else
109     monthDir=0
110     fi
111 edhill 1.1 if test ! -e $OUTDIR ; then
112     mkdir $OUTDIR
113     RETVAL=$?
114 jmc 1.15 if test "x$RETVAL" != x0 ; then
115 jmc 1.18 date
116 edhill 1.1 echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
117 jmc 1.14 exit 3
118 jmc 1.20 else
119     echo "Successfully created new dir: \"$OUTDIR\""
120 edhill 1.1 fi
121 jmc 1.8 chgrp gcmpack $OUTDIR
122     chmod 775 $OUTDIR
123 edhill 1.1 fi
124    
125 jmc 1.10 all_files=`ls -1 $INDIR`
126 jmc 1.11 nb_files=`echo "$all_files" | grep -c '^msg\.'`
127 edhill 1.1
128 jmc 1.10 if test $PRT = 1 ; then
129     echo "Using OUTDIR=\"$OUTDIR\""
130     echo "Using INDIR=\"$INDIR\""
131     echo -n "Unpacking the emails ..."
132     elif test $nb_files != 0 ; then
133 jmc 1.17 echo -n "Unpacking $nb_files emails ("`date`
134 jmc 1.12 if test "x$ADDRERR" != x ; then
135 jmc 1.17 echo -n ", err-msg: '$ADDRERR'"
136 jmc 1.12 fi
137 jmc 1.17 echo ")"
138     echo " from '$INDIR' to '$OUTDIR'"
139 jmc 1.10 fi
140 edhill 1.1
141     for file in $all_files ; do
142    
143 jmc 1.21 #-- create local copy
144 edhill 1.2 test -e $TEMPDIR && rm -rf $TEMPDIR
145 edhill 1.1 mkdir $TEMPDIR
146 jmc 1.15 RETVAL=$?
147     if test "x$RETVAL" = x0 ; then
148     cp $INDIR"/"$file $TEMPDIR
149     RETVAL=$?
150     fi
151     if test "x$RETVAL" != x0 ; then
152     if test "x$ADDRERR" != x ; then
153     echo "parsing email error" > tmp.$$
154     echo " processing file: '$INDIR/$file'" >> tmp.$$
155     echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> tmp.$$
156     echo " returns error $RETVAL" >> tmp.$$
157     mail -s 'parse_emails err_0' $ADDRERR < tmp.$$
158     rm -f tmp.$$
159     fi
160     mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
161     continue
162     fi
163 edhill 1.1
164 jmc 1.21 #-- ignore multi-part messages
165 edhill 1.1 grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
166     RETVAL=$?
167     if test "x$RETVAL" = x0 ; then
168 jmc 1.12 if test "x$ADDRERR" != x ; then
169     echo "parsing email error" > tmp.$$
170     echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
171     ls -l $INDIR"/"$file >> tmp.$$
172     mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
173     rm -f tmp.$$
174     fi
175     mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
176     continue
177 edhill 1.1 fi
178    
179 jmc 1.21 #-- munpack
180 edhill 1.1 mun=`( cd $TEMPDIR ; $MUNPACK $file | cut -d ' ' -f 1 | head -1 )`
181     RETVAL=$?
182     if test "x$RETVAL" != x0 ; then
183 jmc 1.12 if test "x$ADDRERR" != x ; then
184     echo "parsing email error" > tmp.$$
185     echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
186     ls -l $INDIR"/"$file >> tmp.$$
187     mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
188     rm -f tmp.$$
189     fi
190     mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
191     continue
192 edhill 1.1 fi
193    
194 jmc 1.21 #-- un-tar
195     #( cd $TEMPDIR ; tar -xzvf $mun > out )
196     # to remove small files "._mydir" that some MAC OS are adding
197     # (for each file or dir) to a tar-file, use option "--exclude=":
198     ( cd $TEMPDIR ; tar -xzvf $mun --exclude="._*" > out )
199 edhill 1.1 RETVAL=$?
200     if test "x$RETVAL" != x0 ; then
201 jmc 1.12 if test "x$ADDRERR" != x ; then
202     echo "parsing email error" > tmp.$$
203     echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
204     ls -l $INDIR"/"$file >> tmp.$$
205     ls -l $mun >> tmp.$$
206 jmc 1.21 mail -s 'parse_emails err_3a' $ADDRERR < tmp.$$
207 jmc 1.12 rm -f tmp.$$
208     fi
209     mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
210     continue
211 edhill 1.1 fi
212 edhill 1.2 tdir=`cat $TEMPDIR"/out" | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
213 jmc 1.21 #tdir=`( cd $TEMPDIR ; /bin/ls -l | grep '^d' | head -1 | awk '{print $NF}' )`
214     if test -d $TEMPDIR/$tdir ; then
215     rm -f $TEMPDIR"/out"
216     else
217     if test "x$ADDRERR" != x ; then
218     echo "parsing email error" > tmp.$$
219     echo " fail to get a dir output name 'tdir=$tdir'" >> tmp.$$
220     echo " from tar file '$TEMPDIR/$mun'" >> tmp.$$
221     mail -s 'parse_emails err_3b' $ADDRERR < tmp.$$
222     rm -f tmp.$$
223     fi
224     mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
225     continue
226     fi
227 edhill 1.1
228 jmc 1.21 #-- select which Monthly Output Dir:
229 jmc 1.19 locDir=$OUTDIR
230     if test "x$monthDir" != x0 ; then
231     dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
232     | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
233     mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
234     if test "x$mn" != "x$monthDir" ; then
235     if test "x$mn" = x ; then
236     if test $PRT = 2 ; then echo " cannot get month from '$tdir'" ; fi
237     else
238     # could comment out this line:
239     #if test $PRT = 2 ; then echo " chg month: '$mn' for '$tdir'" ; fi
240     locDir="$BASEDIR/$mn"
241     if test ! -d $locDir ; then
242     if test $PRT = 2 ; then echo "NO DIR: '$locDir' => '$tdir' POSTPONED" ; fi
243     if test "x$ADDRERR" != x ; then
244     echo "parsing email error" > tmp.$$
245     echo "no dir '$locDir' for outp. '$tdir'" > tmp.$$
246     ls -l $INDIR"/"$file >> tmp.$$
247     mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
248     rm -f tmp.$$
249     fi
250     continue
251     fi
252     fi
253     fi
254     fi
255    
256 jmc 1.21 #-- copy to $locDir and rename if necessary
257 jmc 1.10 sdir=$tdir
258 jmc 1.19 if test -e $locDir"/"$tdir ; then
259 edhill 1.1 ad=0
260 jmc 1.19 while test -e $locDir"/"$tdir"_"$ad ; do
261 edhill 1.1 ad=$(( $ad + 1 ))
262     done
263 jmc 1.10 sdir=$tdir"_"$ad
264 edhill 1.1 fi
265 jmc 1.19 if test $PRT = 2 ; then
266     if test "x$locDir" = "x$OUTDIR"
267     then echo " '$sdir'"
268     else echo " '$sdir' => '$locDir'"
269     fi
270     fi
271     mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
272 jmc 1.15 RETVAL=$?
273     if test "x$RETVAL" != x0 ; then
274     if test "x$ADDRERR" != x ; then
275     echo "parsing email error" > tmp.$$
276 jmc 1.19 echo "mv $TEMPDIR/$tdir $locDir/$sdir returns error:" $RETVAL >> tmp.$$
277 jmc 1.15 echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR >> tmp.$$
278 jmc 1.19 echo -n "in dir: $OUTDIR : " ; ls -l $locDir >> tmp.$$
279     mail -s 'parse_emails err_5' $ADDRERR < tmp.$$
280 jmc 1.15 rm -f tmp.$$
281     fi
282     mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
283     continue
284     fi
285 jmc 1.19 chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
286 edhill 1.1
287 jmc 1.21 #-- remove the original file
288 edhill 1.1 rm -f $INDIR"/"$file
289    
290     done
291 jmc 1.10 if test $PRT = 1 ; then echo " done" ; fi
292 edhill 1.1

  ViewVC Help
Powered by ViewVC 1.1.22