/[MITgcm]/mitgcm.org/front_content/parse_emails
ViewVC logotype

Contents of /mitgcm.org/front_content/parse_emails

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.26 - (show annotations) (download)
Fri Jan 11 20:49:13 2013 UTC (12 years, 6 months ago) by jmc
Branch: MAIN
Changes since 1.25: +3 -3 lines
always try to use lowest sufix number for output dir

1 #! /usr/bin/env bash
2
3 # $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.25 2011/03/11 18:18:44 jmc Exp $
4 #
5 # The purpose of this script is to parse the emails produced by the
6 # MITgcm/verificaton/testreport script and store the data in a
7 # reasonable location.
8
9
10 usage()
11 {
12 echo
13 echo "Usage: $0 [OPTIONS]"
14 echo
15 echo "where possible OPTIONS are:"
16 echo " (-h|-help) print usage"
17 echo " (-s|-silent) silent mode"
18 echo " (-v|-verbose) verbose mode"
19 echo " (-i |-ind )DIR get mpack-created emails from DIR"
20 echo " [def=\"$INDIR\"]"
21 echo " (-o |-outd )DIR write the data to DIR"
22 echo " [def=\"$BASEDIR/$monthDir\"]"
23 echo " (-t |-tempd )DIR use temporary directory DIR"
24 echo " [def=\"$TEMPDIR\"]"
25 echo " (-u |-unpack )EXE use executable EXE to unpack e-mails"
26 echo " [def=\"$MUNPACK\"]"
27 echo " (-a |-addr )ADDR send e-mail to ADDR if Error"
28 echo " [def='"$ADDRERR"']"
29 echo
30 exit 1
31 }
32
33 # defaults
34 HERE=`pwd`
35 INDIR="/u/u2/jmc/Mail/MITgcm-test"
36 BASEDIR="/u/u0/httpd/html/testing/results"
37 monthDir=`date +%Y`"_"`date +%m`
38 OUTDIR=
39 ADDRERR=
40 MUNPACK=$HERE/munpack
41 UnpTmpD="/var/tmp/m-prts-$USER"
42 TR_LIST='TTT.'$$
43 TEMPDIR=/tmp/prc_emails
44 STDOUT=$TEMPDIR/'outp.'$$
45 ERRMSG=/tmp/tmp.$$
46 PRT=1
47
48 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
49 # Parse options
50 ac_prev=
51 for ac_option ; do
52
53 # If the previous option needs an argument, assign it.
54 if test -n "$ac_prev"; then
55 eval "$ac_prev=\$ac_option"
56 ac_prev=
57 continue
58 fi
59
60 ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
61
62 case $ac_option in
63
64 -help | --help | -h | --h)
65 usage ;;
66 -s | --s | -silent | --silent)
67 PRT=0 ;;
68 -v | --v | -verbose | --verbose)
69 PRT=2 ;;
70
71 -ind | --ind | -i | --i)
72 ac_prev=INDIR ;;
73 --ind=* | -ind=* | --i=* | -i=*)
74 INDIR=$ac_optarg ;;
75
76 -outd | --outd | -o | --o)
77 ac_prev=OUTDIR ;;
78 --outd=* | -outd=* | --o=* | -o=*)
79 OUTDIR=$ac_optarg ;;
80
81 -tempd | --tempd | -t | --t)
82 ac_prev=TEMPDIR ;;
83 --tempd=* | -tempd=* | --t=* | -t=*)
84 TEMPDIR=$ac_optarg ;;
85
86 -u | --u | -unpack | --unpack)
87 ac_prev=MUNPACK ;;
88 -u=* | --u=* | -unpack=* | --unpack=*)
89 MUNPACK=$ac_optarg ;;
90
91 -a | --a | -addr | --addr)
92 ac_prev=ADDRERR ;;
93 -a=* | --a=* | -addr=* | --addr=*)
94 ADDRERR=$ac_optarg ;;
95
96 *)
97 date
98 echo "Error: don't understand argument \"$ac_option\""
99 usage
100 ;;
101
102 esac
103
104 done
105
106 if test ! -x $MUNPACK ; then
107 date
108 echo "ERROR: \"$MUNPACK\" is not executable"
109 exit 2
110 fi
111 if test "x$OUTDIR" = x ; then
112 OUTDIR="$BASEDIR/$monthDir"
113 else
114 monthDir=0
115 fi
116 if test ! -e $OUTDIR ; then
117 mkdir $OUTDIR
118 RETVAL=$?
119 if test "x$RETVAL" != x0 ; then
120 date
121 echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
122 exit 3
123 else
124 echo "Successfully created new dir: \"$OUTDIR\""
125 fi
126 chgrp gcmpack $OUTDIR
127 chmod 775 $OUTDIR
128 fi
129
130 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
131
132 all_msg=`ls -1 $INDIR`
133 nb_files=`echo "$all_msg" | grep -c '^msg\.'`
134
135 if test $PRT = 2 ; then
136 echo "Using OUTDIR=\"$OUTDIR\""
137 echo "Using INDIR=\"$INDIR\""
138 echo -n "Unpacking $nb_files emails ("`date`
139 if test "x$ADDRERR" != x ; then
140 echo -n ", err-msg: '$ADDRERR'"
141 fi
142 echo ")"
143 elif test $nb_files != 0 ; then
144 echo -n "Unpacking $nb_files emails ("`date`
145 if test "x$ADDRERR" != x ; then
146 echo -n ", err-msg: '$ADDRERR'"
147 fi
148 echo ")"
149 echo " from '$INDIR' to '$OUTDIR'"
150 fi
151
152 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
153
154 #-- process list of message to detect group of multi-part msgs:
155 rm -f $TR_LIST
156 touch $TR_LIST
157
158 flag=0
159 for xx in $all_msg
160 do
161 in=`grep -c $xx $TR_LIST`
162 if test $in = 0 ; then
163 np=`grep -c 'Content-Type: message/partial' $INDIR/$xx`
164 if test $np = 0 ; then
165 echo $xx >> $TR_LIST
166 else
167 l=`sed -n '/Content-Type: message\/partial/=' $INDIR/$xx`
168 lp=`expr $l + 1`
169 id=`sed -n "$lp p" $INDIR/$xx`
170 partM=`( cd $INDIR ; grep -c "$id" msg.* | grep -v ':0$' | sed 's/:1$//' )`
171 echo $partM >> $TR_LIST
172 if test "x$ADDRERR" != x ; then flag=1
173 echo "multi-parts message:" $partM >> $ERRMSG
174 ( cd $INDIR ; ls -l $partM ) >> $ERRMSG
175 fi
176 fi
177 fi
178 done
179 if test $flag = 1 ; then
180 mail -s 'parse_emails Multi-parts msg' $ADDRERR < $ERRMSG
181 rm -f $ERRMSG
182 fi
183 #cat $TR_LIST ; echo '----------------------------------'
184
185 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
186
187 #-- process list of individual + group of multi-part messages
188 Nbl=`wc -l $TR_LIST | cut -d ' ' -f 1`
189 n=0
190 while [ $n -lt $Nbl ] ; do
191 n=`expr $n + 1`
192 errFlg=0 ; prcM=''
193 grpM=`sed -n "$n p" $TR_LIST`
194 nm=`sed -n "$n p" $TR_LIST | wc -w | cut -d ' ' -f 1`
195
196 #- create local copy
197 test -e $TEMPDIR && rm -rf $TEMPDIR
198 mkdir $TEMPDIR
199 RETVAL=$?
200 if test "x$RETVAL" = x0 ; then
201 ( cd $INDIR ; cp $grpM $TEMPDIR )
202 RETVAL=$?
203 fi
204 if test "x$RETVAL" != x0 ; then
205 if test "x$ADDRERR" != x ; then
206 echo "error parsing email '$grpM' :" > $ERRMSG
207 echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$grpM $TEMPDIR'" | tee -a $ERRMSG
208 echo " returns error $RETVAL" | tee -a $ERRMSG
209 mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
210 rm -f $ERRMSG
211 fi
212 ( cd $INDIR ; mv -f $grpM ../fail2process )
213 continue
214 fi
215
216 #-check that we have all the parts
217 if [ $PRT -ge 1 -a $nm -gt 1 ] ; then
218 echo " group (nm=$nm) of multi-parts msg: '$grpM'"
219 fi
220 for xx in $grpM ; do
221 np=`grep 'Content-Type: message/partial' $TEMPDIR/$xx \
222 | sed 's/^Content.*total=//' | sed 's/;$//'`
223 if test "x$np" = x -a $nm = 1 ; then np=1 ; fi
224 if test "x$np" != "x$nm" ; then
225 if test $errFlg = 0 ; then errFlg=1
226 echo "error parsing email '$grpM' :" > $ERRMSG
227 fi
228 echo " - Error: Number of parts='$np' in $xx but got '$nm' msg-files" \
229 | tee -a $ERRMSG
230 fi
231 done
232 if test $errFlg = 1 ; then
233 if test "x$ADDRERR" != x ; then
234 mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
235 fi
236 rm -f $ERRMSG
237 ( cd $INDIR ; mv -f $grpM ../fail2process )
238 continue
239 fi
240
241 if test $nm = 1 ; then
242 #- unpack single part message:
243 xx=$grpM
244 prcM=$xx
245 if test $PRT = 2 ; then echo " unpack single-part msg: '$xx'" ; fi
246 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
247 RETVAL=$?
248 if test "x$RETVAL" != x0 ; then errFlg=1
249 echo "error parsing email '$xx' :" > $ERRMSG
250 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
251 ls -l $INDIR"/"$xx >> $ERRMSG
252 else
253 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
254 if test ! -f $TEMPDIR/$trOutp ; then errFlg=1
255 echo "error parsing email '$xx' :" > $ERRMSG
256 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
257 | tee -a $ERRMSG
258 fi
259 fi
260 if test $errFlg = 1 ; then
261 cat $STDOUT | tee -a $ERRMSG
262 echo " <<<-----------------------"
263 if test "x$ADDRERR" != x ; then
264 mail -s 'parse_emails err_2s' $ADDRERR < $ERRMSG
265 fi
266 rm -f $ERRMSG
267 fi
268 rm -f $STDOUT
269
270 else
271 #- unpack group of multi-parts message:
272
273 #-unpack each part, in the right order:
274 j=0
275 while [ $j -lt $nm ] ; do
276 j=`expr $j + 1`
277 #- get the j^th part msg
278 xx=`( cd $TEMPDIR ; grep -c "Content-Type: message/partial; number=$j" $grpM \
279 | grep -v ':0$' | sed 's/:1$//' )`
280 if test $PRT = 2 ; then
281 echo -n " $j : xx='$xx' : "
282 grep 'Content-Type: message/partial' $TEMPDIR/$xx | sed 's/Content-Type: //'
283 fi
284 #- check
285 if test ! -f $TEMPDIR/$xx ; then
286 echo "error parsing email '$xx' :" > $ERRMSG
287 echo " - Error: Missing msg file '$xx' in $TEMPDIR:" | tee -a $ERRMSG
288 ls -l $TEMDIR | tee -a $ERRMSG
289 errFlg=1 ; echo " <<<-----------------------"
290 if test "x$ADDRERR" != x ; then
291 mail -s 'parse_emails err_2g' $ADDRERR < $ERRMSG
292 fi
293 rm -f $ERRMSG
294 else
295 if test $j = 1 ; then prcM=$xx ; else prcM="$prcM $xx" ; fi
296 #- do unpack
297 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
298 RETVAL=$?
299 if test "x$RETVAL" != x0 ; then errFlg=1
300 echo "error parsing email '$xx' :" > $ERRMSG
301 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
302 ls -l $INDIR"/"$xx | tee -a $ERRMSG
303 cat $STDOUT | tee -a $ERRMSG
304 errFlg=1 ; echo " <<<-----------------------"
305 if test "x$ADDRERR" != x ; then
306 mail -s 'parse_emails err_2u' $ADDRERR < $ERRMSG
307 fi
308 rm -f $ERRMSG
309 fi
310 fi
311 if test $errFlg = 1 ; then j=`expr $nm + 1` ; fi
312
313 if [ $j -lt $nm ] ; then
314 #- if not last part:
315 pfix=`cat $STDOUT | tail -1 | awk '{print $NF}'`
316 if test -d $UnpTmpD/$pfix ; then
317 lock=$UnpTmpD/$pfix/CT
318 if test -e $lock ; then
319 if test $PRT = 2 ; then echo " remove lock: $lock" ; fi
320 rm -f $lock
321 else
322 echo "error parsing email '$xx' :" > $ERRMSG
323 echo " - Error: lock file '$lock' not found from $MUNPACK output:" \
324 | tee -a $ERRMSG
325 cat $STDOUT | tee -a $ERRMSG
326 errFlg=1 ; echo " <<<-----------------------"
327 fi
328 else
329 echo "error parsing email '$xx' :" > $ERRMSG
330 echo " - Error: found no dir '$UnpTmpD/$pfix' from $MUNPACK output:" \
331 | tee -a $ERRMSG
332 cat $STDOUT | tee -a $ERRMSG
333 errFlg=1 ; echo " <<<-----------------------"
334 fi
335 elif [ $j -eq $nm ] ; then
336 #- if last part:
337 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
338 if test ! -f $TEMPDIR/$trOutp ; then
339 echo "error parsing email '$xx' :" > $ERRMSG
340 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
341 | tee -a $ERRMSG
342 cat $STDOUT | tee -a $ERRMSG
343 errFlg=1 ; echo " <<<-----------------------"
344 fi
345 fi
346 if [ $errFlg -eq 1 -a $j -le $nm ] ; then j=$nm
347 if test "x$ADDRERR" != x ; then
348 mail -s 'parse_emails err_2m' $ADDRERR < $ERRMSG
349 fi
350 rm -f $ERRMSG
351 fi
352 rm -f $STDOUT
353 done
354 fi
355 #-- in case of error: mv all $grpM msg to "fail2proc"
356 if test $errFlg = 1 ; then
357 ( cd $INDIR ; mv -f $grpM ../fail2process )
358 continue
359 fi
360 if test $PRT = 2 ; then
361 if test -f $TEMPDIR/$trOutp ; then ls -l $TEMPDIR/$trOutp ; fi
362 fi
363
364 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
365
366 #-- un-tar
367 #( cd $TEMPDIR ; tar -xzvf $trOutp > $STDOUT )
368 # to remove small files "._mydir" that some MAC OS are adding
369 # (for each file or dir) to a tar-file, use option "--exclude=":
370 ( cd $TEMPDIR ; tar -xzvf $trOutp --exclude="._*" > $STDOUT )
371 RETVAL=$?
372 if test "x$RETVAL" != x0 ; then
373 echo "parsing email error" > $ERRMSG
374 echo " - Error: tar -xzvf $trOutp returns:" $RETVAL | tee -a $ERRMSG
375 ( cd $INDIR ; ls -l $prcM ) | tee -a $ERRMSG
376 ls -l $TEMPDIR/$trOutp | tee -a $ERRMSG
377 if test "x$ADDRERR" != x ; then
378 mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
379 fi
380 rm -f $ERRMSG
381 ( cd $INDIR ; mv -f $grpM ../fail2process )
382 continue
383 fi
384 tdir=`cat $STDOUT | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
385 if test -d $TEMPDIR/$tdir ; then
386 rm -f $STDOUT
387 else
388 echo "parsing email error" > $ERRMSG
389 echo " - Error: fail to get dir output name 'tdir=$tdir'" | tee -a $ERRMSG
390 echo " (cont) from tar file '$TEMPDIR/$trOutp'" | tee -a $ERRMSG
391 if test "x$ADDRERR" != x ; then
392 mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
393 fi
394 rm -f $ERRMSG
395 ( cd $INDIR ; mv -f $grpM ../fail2process )
396 continue
397 fi
398
399 #-- select which Monthly Output Dir:
400 locDir=$OUTDIR
401 if test "x$monthDir" != x0 ; then
402 dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
403 | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
404 mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
405 if test "x$mn" != "x$monthDir" ; then
406 if test "x$mn" = x ; then
407 if [ $PRT -ge 1 ] ; then echo " cannot get month from '$tdir'" ; fi
408 else
409 locDir="$BASEDIR/$mn"
410 if test ! -d $locDir ; then
411 if [ $PRT -ge 1 ] ; then
412 echo "NO DIR: '$locDir' => '$tdir' POSTPONED"
413 fi
414 if test "x$ADDRERR" != x ; then
415 echo "parsing email error" > $ERRMSG
416 echo " no dir '$locDir' for outp. '$tdir'" >> $ERRMSG
417 ( cd $INDIR ; ls -l $prcM ) >> $ERRMSG
418 mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
419 rm -f $ERRMSG
420 fi
421 continue
422 fi
423 fi
424 fi
425 fi
426
427 #-- copy to $locDir and rename if necessary
428 sdir=$tdir
429 # if test -e $locDir"/"$tdir ; then
430 sdir=`echo $tdir | sed 's/_[0-9]*$//'`
431 ad=0
432 while test -e $locDir"/"$sdir"_"$ad ; do
433 ad=$(( $ad + 1 ))
434 done
435 sdir=$sdir"_"$ad
436 # fi
437 if [ $PRT -ge 1 ] ; then
438 if test "x$locDir" = "x$OUTDIR"
439 then echo " '$sdir' ($prcM)"
440 else echo " '$sdir' ($prcM) => '$locDir'"
441 fi
442 fi
443 mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
444 RETVAL=$?
445 if test "x$RETVAL" != x0 ; then
446 if test "x$ADDRERR" != x ; then
447 echo "parsing email error" > $ERRMSG
448 echo " - Error: mv $TEMPDIR/$tdir $locDir/$sdir returns:" $RETVAL \
449 | tee -a $ERRMSG
450 echo -n " in dir: $TEMPDIR : " ; ls -l $TEMPDIR | tee -a $ERRMSG
451 echo -n " in dir: $OUTDIR : " ; ls -l $locDir | tee -a $ERRMSG
452 mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
453 rm -f $ERRMSG
454 fi
455 ( cd $INDIR ; mv -f $grpM ../fail2process )
456 continue
457 fi
458 chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
459
460 #-- remove the original message files
461 ( cd $INDIR ; rm -f $grpM )
462
463 done
464 rm -f $TR_LIST
465 if test $PRT = 2 ; then echo " done" ; fi

  ViewVC Help
Powered by ViewVC 1.1.22