/[MITgcm]/mitgcm.org/front_content/parse_emails
ViewVC logotype

Contents of /mitgcm.org/front_content/parse_emails

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.29 - (show annotations) (download)
Thu Jan 11 21:46:24 2018 UTC (6 years, 3 months ago) by jmc
Branch: MAIN
CVS Tags: HEAD
Changes since 1.28: +2 -2 lines
modify "TEMPDIR"

1 #! /usr/bin/env bash
2
3 # $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.28 2017/05/25 18:31:04 jmc Exp $
4 #
5 # The purpose of this script is to parse the emails produced by the
6 # MITgcm/verificaton/testreport script and store the data in a
7 # reasonable location.
8
9
10 usage()
11 {
12 echo
13 echo "Usage: $0 [OPTIONS]"
14 echo
15 echo "where possible OPTIONS are:"
16 echo " (-h|-help) print usage"
17 echo " (-s|-silent) silent mode"
18 echo " (-v|-verbose) verbose mode"
19 echo " (-i |-ind )DIR get mpack-created emails from DIR"
20 echo " [def=\"$INDIR\"]"
21 echo " (-o |-outd )DIR write the data to DIR"
22 echo " [def=\"$BASEDIR/$monthDir\"]"
23 echo " (-t |-tempd )DIR use temporary directory DIR"
24 echo " [def=\"$TEMPDIR\"]"
25 echo " (-u |-unpack )EXE use executable EXE to unpack e-mails"
26 echo " [def=\"$MUNPACK\"]"
27 echo " (-a |-addr )ADDR send e-mail to ADDR if Error"
28 echo " [def='"$ADDRERR"']"
29 echo
30 exit 1
31 }
32
33 # defaults
34 HERE=`pwd`
35 INDIR="/u/u2/jmc/Mail/MITgcm-test"
36 BASEDIR="/u/u0/httpd/html/testing/results"
37 monthDir=`date +%Y`"_"`date +%m`
38 OUTDIR=
39 ADDRERR=
40 MUNPACK=$HERE/munpack
41 UnpTmpD="/var/tmp/m-prts-$USER"
42 TR_LIST='TTT.'$$
43 TEMPDIR="/tmp/prc_emails_$USER"
44 STDOUT=$TEMPDIR/'outp.'$$
45 ERRMSG=/tmp/tmp.$$
46 PRT=1
47
48 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
49 # Parse options
50 ac_prev=
51 for ac_option ; do
52
53 # If the previous option needs an argument, assign it.
54 if test -n "$ac_prev"; then
55 eval "$ac_prev=\$ac_option"
56 ac_prev=
57 continue
58 fi
59
60 ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
61
62 case $ac_option in
63
64 -help | --help | -h | --h)
65 usage ;;
66 -s | --s | -silent | --silent)
67 PRT=0 ;;
68 -v | --v | -verbose | --verbose)
69 PRT=2 ;;
70
71 -ind | --ind | -i | --i)
72 ac_prev=INDIR ;;
73 --ind=* | -ind=* | --i=* | -i=*)
74 INDIR=$ac_optarg ;;
75
76 -outd | --outd | -o | --o)
77 ac_prev=OUTDIR ;;
78 --outd=* | -outd=* | --o=* | -o=*)
79 OUTDIR=$ac_optarg ;;
80
81 -tempd | --tempd | -t | --t)
82 ac_prev=TEMPDIR ;;
83 --tempd=* | -tempd=* | --t=* | -t=*)
84 TEMPDIR=$ac_optarg ;;
85
86 -u | --u | -unpack | --unpack)
87 ac_prev=MUNPACK ;;
88 -u=* | --u=* | -unpack=* | --unpack=*)
89 MUNPACK=$ac_optarg ;;
90
91 -a | --a | -addr | --addr)
92 ac_prev=ADDRERR ;;
93 -a=* | --a=* | -addr=* | --addr=*)
94 ADDRERR=$ac_optarg ;;
95
96 *)
97 date
98 echo "Error: don't understand argument \"$ac_option\""
99 usage
100 ;;
101
102 esac
103
104 done
105
106 if test ! -x $MUNPACK ; then
107 date
108 echo "ERROR: \"$MUNPACK\" is not executable"
109 exit 2
110 fi
111 if test "x$OUTDIR" = x ; then
112 OUTDIR="$BASEDIR/$monthDir"
113 else
114 monthDir=0
115 fi
116 if test ! -e $OUTDIR ; then
117 mkdir $OUTDIR
118 RETVAL=$?
119 if test "x$RETVAL" != x0 ; then
120 date
121 echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
122 exit 3
123 else
124 echo "Successfully created new dir: \"$OUTDIR\""
125 fi
126 chgrp gcmpack $OUTDIR
127 chmod 775 $OUTDIR
128 fi
129
130 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
131
132 all_msg=`ls -1 $INDIR`
133 nb_msg=`echo "$all_msg" | grep -c '^msg\.'`
134 nb_tar=`echo "$all_msg" | grep -c '\.tar\.gz$'`
135 nb_files=`expr $nb_msg + $nb_tar`
136
137 if test $PRT = 2 ; then
138 echo "Using OUTDIR=\"$OUTDIR\""
139 echo "Using INDIR=\"$INDIR\""
140 echo -n "Unpacking $nb_msg msg + $nb_tar tar-file ("`date`
141 if test "x$ADDRERR" != x ; then
142 echo -n ", err: $ADDRERR"
143 fi
144 echo ")"
145 elif test $nb_files != 0 ; then
146 echo -n "Unpacking $nb_msg msg + $nb_tar tar-file ("`date`
147 if test "x$ADDRERR" != x ; then
148 echo -n ", err: $ADDRERR"
149 fi
150 echo ")"
151 echo " from '$INDIR' to '$OUTDIR'"
152 fi
153
154 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
155
156 #-- process list of message to detect group of multi-part msgs:
157 rm -f $TR_LIST
158 touch $TR_LIST
159
160 flag=0
161 for xx in $all_msg
162 do
163 in=`grep -c $xx $TR_LIST`
164 if test $in = 0 ; then
165 it=`echo $xx | grep -c '\.tar\.gz$'`
166 if test $it = 1 ; then
167 echo $xx >> $TR_LIST
168 else
169 np=`grep -c 'Content-Type: message/partial' $INDIR/$xx`
170 if test $np = 0 ; then
171 echo $xx >> $TR_LIST
172 else
173 l=`sed -n '/Content-Type: message\/partial/=' $INDIR/$xx`
174 lp=`expr $l + 1`
175 id=`sed -n "$lp p" $INDIR/$xx`
176 partM=`( cd $INDIR ; grep -c "$id" msg.* | grep -v ':0$' | sed 's/:1$//' )`
177 echo $partM >> $TR_LIST
178 if test "x$ADDRERR" != x ; then flag=1
179 echo "multi-parts message:" $partM >> $ERRMSG
180 ( cd $INDIR ; ls -l $partM ) >> $ERRMSG
181 fi
182 fi
183 fi
184 fi
185 done
186 if test $flag = 1 ; then
187 mail -s 'parse_emails Multi-parts msg' $ADDRERR < $ERRMSG
188 rm -f $ERRMSG
189 fi
190 #cat $TR_LIST ; echo '----------------------------------'
191
192 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
193
194 #-- process list of individual + group of multi-part messages
195 Nbl=`wc -l $TR_LIST | cut -d ' ' -f 1`
196 n=0
197 while [ $n -lt $Nbl ] ; do
198 n=`expr $n + 1`
199 errFlg=0 ; prcM=''
200 grpM=`sed -n "$n p" $TR_LIST`
201 nm=`sed -n "$n p" $TR_LIST | wc -w | cut -d ' ' -f 1`
202
203 #- create local copy
204 test -e $TEMPDIR && rm -rf $TEMPDIR
205 mkdir $TEMPDIR
206 RETVAL=$?
207 if test "x$RETVAL" = x0 ; then
208 ( cd $INDIR ; cp $grpM $TEMPDIR )
209 RETVAL=$?
210 fi
211 if test "x$RETVAL" != x0 ; then
212 if test "x$ADDRERR" != x ; then
213 echo "error parsing email '$grpM' :" > $ERRMSG
214 echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$grpM $TEMPDIR'" | tee -a $ERRMSG
215 echo " returns error $RETVAL" | tee -a $ERRMSG
216 mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
217 rm -f $ERRMSG
218 fi
219 ( cd $INDIR ; mv -f $grpM ../fail2process )
220 continue
221 fi
222
223 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
224 it=`echo $grpM | grep -c '\.tar\.gz$'`
225 if test $it = 1 ; then
226 #- nothing to do: already a tar file!
227 trOutp=$grpM ; prcM=$grpM
228 else
229
230 #-check that we have all the parts
231 if [ $PRT -ge 1 -a $nm -gt 1 ] ; then
232 echo " group (nm=$nm) of multi-parts msg: '$grpM'"
233 fi
234 for xx in $grpM ; do
235 np=`grep 'Content-Type: message/partial' $TEMPDIR/$xx \
236 | sed 's/^Content.*total=//' | sed 's/;$//'`
237 if test "x$np" = x -a $nm = 1 ; then np=1 ; fi
238 if test "x$np" != "x$nm" ; then
239 if test $errFlg = 0 ; then errFlg=1
240 echo "error parsing email '$grpM' :" > $ERRMSG
241 fi
242 echo " - Error: Number of parts='$np' in $xx but got '$nm' msg-files" \
243 | tee -a $ERRMSG
244 fi
245 done
246 if test $errFlg = 1 ; then
247 if test "x$ADDRERR" != x ; then
248 mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
249 fi
250 rm -f $ERRMSG
251 ( cd $INDIR ; mv -f $grpM ../fail2process )
252 continue
253 fi
254
255 if test $nm = 1 ; then
256 #- unpack single part message:
257 xx=$grpM
258 prcM=$xx
259 if test $PRT = 2 ; then echo " unpack single-part msg: '$xx'" ; fi
260 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
261 RETVAL=$?
262 if test "x$RETVAL" != x0 ; then errFlg=1
263 echo "error parsing email '$xx' :" > $ERRMSG
264 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
265 ls -l $INDIR"/"$xx >> $ERRMSG
266 else
267 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
268 if test ! -f $TEMPDIR/$trOutp ; then errFlg=1
269 echo "error parsing email '$xx' :" > $ERRMSG
270 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
271 | tee -a $ERRMSG
272 fi
273 fi
274 if test $errFlg = 1 ; then
275 cat $STDOUT | tee -a $ERRMSG
276 echo " <<<-----------------------"
277 if test "x$ADDRERR" != x ; then
278 mail -s 'parse_emails err_2s' $ADDRERR < $ERRMSG
279 fi
280 rm -f $ERRMSG
281 fi
282 rm -f $STDOUT
283
284 else
285 #- unpack group of multi-parts message:
286
287 #-unpack each part, in the right order:
288 j=0
289 while [ $j -lt $nm ] ; do
290 j=`expr $j + 1`
291 #- get the j^th part msg
292 xx=`( cd $TEMPDIR ; grep -c "Content-Type: message/partial; number=$j" $grpM \
293 | grep -v ':0$' | sed 's/:1$//' )`
294 if test $PRT = 2 ; then
295 echo -n " $j : xx='$xx' : "
296 grep 'Content-Type: message/partial' $TEMPDIR/$xx | sed 's/Content-Type: //'
297 fi
298 #- check
299 if test ! -f $TEMPDIR/$xx ; then
300 echo "error parsing email '$xx' :" > $ERRMSG
301 echo " - Error: Missing msg file '$xx' in $TEMPDIR:" | tee -a $ERRMSG
302 ls -l $TEMDIR | tee -a $ERRMSG
303 errFlg=1 ; echo " <<<-----------------------"
304 if test "x$ADDRERR" != x ; then
305 mail -s 'parse_emails err_2g' $ADDRERR < $ERRMSG
306 fi
307 rm -f $ERRMSG
308 else
309 if test $j = 1 ; then prcM=$xx ; else prcM="$prcM $xx" ; fi
310 #- do unpack
311 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
312 RETVAL=$?
313 if test "x$RETVAL" != x0 ; then errFlg=1
314 echo "error parsing email '$xx' :" > $ERRMSG
315 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
316 ls -l $INDIR"/"$xx | tee -a $ERRMSG
317 cat $STDOUT | tee -a $ERRMSG
318 errFlg=1 ; echo " <<<-----------------------"
319 if test "x$ADDRERR" != x ; then
320 mail -s 'parse_emails err_2u' $ADDRERR < $ERRMSG
321 fi
322 rm -f $ERRMSG
323 fi
324 fi
325 if test $errFlg = 1 ; then j=`expr $nm + 1` ; fi
326
327 if [ $j -lt $nm ] ; then
328 #- if not last part:
329 pfix=`cat $STDOUT | tail -1 | awk '{print $NF}'`
330 if test -d $UnpTmpD/$pfix ; then
331 lock=$UnpTmpD/$pfix/CT
332 if test -e $lock ; then
333 if test $PRT = 2 ; then echo " remove lock: $lock" ; fi
334 rm -f $lock
335 else
336 echo "error parsing email '$xx' :" > $ERRMSG
337 echo " - Error: lock file '$lock' not found from $MUNPACK output:" \
338 | tee -a $ERRMSG
339 cat $STDOUT | tee -a $ERRMSG
340 errFlg=1 ; echo " <<<-----------------------"
341 fi
342 else
343 echo "error parsing email '$xx' :" > $ERRMSG
344 echo " - Error: found no dir '$UnpTmpD/$pfix' from $MUNPACK output:" \
345 | tee -a $ERRMSG
346 cat $STDOUT | tee -a $ERRMSG
347 errFlg=1 ; echo " <<<-----------------------"
348 fi
349 elif [ $j -eq $nm ] ; then
350 #- if last part:
351 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
352 if test ! -f $TEMPDIR/$trOutp ; then
353 echo "error parsing email '$xx' :" > $ERRMSG
354 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
355 | tee -a $ERRMSG
356 cat $STDOUT | tee -a $ERRMSG
357 errFlg=1 ; echo " <<<-----------------------"
358 fi
359 fi
360 if [ $errFlg -eq 1 -a $j -le $nm ] ; then j=$nm
361 if test "x$ADDRERR" != x ; then
362 mail -s 'parse_emails err_2m' $ADDRERR < $ERRMSG
363 fi
364 rm -f $ERRMSG
365 fi
366 rm -f $STDOUT
367 done
368 fi
369 #-- in case of error: mv all $grpM msg to "fail2proc"
370 if test $errFlg = 1 ; then
371 ( cd $INDIR ; mv -f $grpM ../fail2process )
372 continue
373 fi
374 if test $PRT = 2 ; then
375 if test -f $TEMPDIR/$trOutp ; then ls -l $TEMPDIR/$trOutp ; fi
376 fi
377
378 fi
379 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
380
381 #-- un-tar
382 #( cd $TEMPDIR ; tar -xzvf $trOutp > $STDOUT )
383 # to remove small files "._mydir" that some MAC OS are adding
384 # (for each file or dir) to a tar-file, use option "--exclude=":
385 ( cd $TEMPDIR ; tar -xzvf $trOutp --exclude="._*" > $STDOUT 2> $ERRMSG )
386 RETVAL=$?
387 #echo "--- content of file $STDOUT (STDOUT):"
388 #cat $STDOUT
389 #echo "--- content of file $ERRMSG (ERRMSG):"
390 #cat $ERRMSG
391 if test "x$RETVAL" != x0 ; then
392 echo "parsing email error" >> $ERRMSG
393 echo " - Error: tar -xzvf $trOutp returns:" $RETVAL | tee -a $ERRMSG
394 ( cd $INDIR ; ls -l $prcM ) | tee -a $ERRMSG
395 ls -l $TEMPDIR/$trOutp | tee -a $ERRMSG
396 if test "x$ADDRERR" != x ; then
397 mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
398 fi
399 rm -f $ERRMSG
400 ( cd $INDIR ; mv -f $grpM ../fail2process )
401 continue
402 fi
403 test -f $ERRMSG && rm -f $ERRMSG
404 #--
405 tdir=`cat $STDOUT | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
406 if test -d $TEMPDIR/$tdir ; then
407 rm -f $STDOUT
408 else
409 echo "parsing email error" > $ERRMSG
410 echo " - Error: fail to get dir output name 'tdir=$tdir'" | tee -a $ERRMSG
411 echo " (cont) from tar file '$TEMPDIR/$trOutp'" | tee -a $ERRMSG
412 if test "x$ADDRERR" != x ; then
413 mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
414 fi
415 rm -f $ERRMSG
416 ( cd $INDIR ; mv -f $grpM ../fail2process )
417 continue
418 fi
419
420 #-- select which Monthly Output Dir:
421 locDir=$OUTDIR
422 if test "x$monthDir" != x0 ; then
423 dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
424 | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
425 mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
426 if test "x$mn" != "x$monthDir" ; then
427 if test "x$mn" = x ; then
428 if [ $PRT -ge 1 ] ; then echo " cannot get month from '$tdir'" ; fi
429 else
430 locDir="$BASEDIR/$mn"
431 if test ! -d $locDir ; then
432 if [ $PRT -ge 1 ] ; then
433 echo "NO DIR: '$locDir' => '$tdir' POSTPONED"
434 fi
435 if test "x$ADDRERR" != x ; then
436 echo "parsing email error" > $ERRMSG
437 echo " no dir '$locDir' for outp. '$tdir'" >> $ERRMSG
438 ( cd $INDIR ; ls -l $prcM ) >> $ERRMSG
439 mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
440 rm -f $ERRMSG
441 fi
442 continue
443 fi
444 fi
445 fi
446 fi
447
448 #-- copy to $locDir and rename if necessary
449 sdir=$tdir
450 # if test -e $locDir"/"$tdir ; then
451 sdir=`echo $tdir | sed 's/_[0-9]*$//'`
452 ad=0
453 while test -e $locDir"/"$sdir"_"$ad ; do
454 ad=$(( $ad + 1 ))
455 done
456 sdir=$sdir"_"$ad
457 # fi
458 if [ $PRT -ge 1 ] ; then
459 if test "x$locDir" = "x$OUTDIR"
460 then echo " '$sdir' ($prcM)"
461 else echo " '$sdir' ($prcM) => '$locDir'"
462 fi
463 fi
464 mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
465 RETVAL=$?
466 if test "x$RETVAL" != x0 ; then
467 if test "x$ADDRERR" != x ; then
468 echo "parsing email error" > $ERRMSG
469 echo " - Error: mv $TEMPDIR/$tdir $locDir/$sdir returns:" $RETVAL \
470 | tee -a $ERRMSG
471 echo -n " in dir: $TEMPDIR : " ; ls -l $TEMPDIR | tee -a $ERRMSG
472 echo -n " in dir: $OUTDIR : " ; ls -l $locDir | tee -a $ERRMSG
473 mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
474 rm -f $ERRMSG
475 fi
476 ( cd $INDIR ; mv -f $grpM ../fail2process )
477 continue
478 fi
479 chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
480
481 #-- remove the original message files
482 ( cd $INDIR ; rm -f $grpM )
483
484 done
485 rm -f $TR_LIST
486 if test $PRT = 2 ; then echo " done" ; fi

  ViewVC Help
Powered by ViewVC 1.1.22