/[MITgcm]/mitgcm.org/front_content/get+parse_msg
ViewVC logotype

Contents of /mitgcm.org/front_content/get+parse_msg

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.3 - (show annotations) (download)
Thu Jan 11 21:46:24 2018 UTC (3 years, 8 months ago) by jmc
Branch: MAIN
Changes since 1.2: +2 -2 lines
modify "TEMPDIR"

1 #! /usr/bin/env bash
2
3 # $Header: /u/gcmpack/mitgcm.org/front_content/get+parse_msg,v 1.2 2018/01/06 20:51:40 jmc Exp $
4 #
5 # The purpose of this script is to parse the emails produced by the
6 # MITgcm/verificaton/testreport script and store the data in a
7 # reasonable location.
8
9 usage()
10 {
11 echo
12 echo "Usage: $0 [OPTIONS]"
13 echo
14 echo "where possible OPTIONS are:"
15 echo " (-h|-help) print usage"
16 echo " (-s|-silent) silent mode"
17 echo " (-v|-verbose) verbose mode"
18 echo " (-i |-ind )DIR get mpack-created emails from DIR"
19 echo " [def=\"$INDIR\"]"
20 echo " (-o |-outd )DIR write the data to DIR"
21 echo " [def=\"$BASEDIR/$monthDir\"]"
22 echo " (-t |-tempd )DIR use temporary directory DIR"
23 echo " [def=\"$TEMPDIR\"]"
24 echo " (-u |-unpack )EXE use executable EXE to unpack e-mails"
25 echo " [def=\"$MUNPACK\"]"
26 echo " (-a |-addr )ADDR send e-mail to ADDR if Error"
27 echo " [def='"$ADDRERR"']"
28 echo
29 exit 1
30 }
31
32 # defaults
33 HERE=`pwd`
34 #INDIR="/u/u2/jmc/Mail/MITgcm-test"
35 #BASEDIR="/u/u0/httpd/html/testing/results"
36 RHOST="jm_c@mitgcm-mm.mit.edu" ; M_DIR="Mail/MITgcm-test"
37 INDIR="/home/jm_c/testing/MITgcm-test"
38 BASEDIR="/net/zany.mit.edu/data/ORWELL/export-9/mitgcm-testing/results"
39 monthDir=`date +%Y`"_"`date +%m`
40 OUTDIR=
41 ADDRERR=
42 MUNPACK=$HERE/munpack
43 UnpTmpD="/var/tmp/m-prts-$USER"
44 TR_LIST='TTT.'$$
45 TEMPDIR="/tmp/prc_emails_$USER"
46 STDOUT=$TEMPDIR/'outp.'$$
47 ERRMSG=/tmp/tmp.$$
48 PRT=1
49
50 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
51 # Parse options
52 ac_prev=
53 for ac_option ; do
54
55 # If the previous option needs an argument, assign it.
56 if test -n "$ac_prev"; then
57 eval "$ac_prev=\$ac_option"
58 ac_prev=
59 continue
60 fi
61
62 ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
63
64 case $ac_option in
65
66 -help | --help | -h | --h)
67 usage ;;
68 -s | --s | -silent | --silent)
69 PRT=0 ;;
70 -v | --v | -verbose | --verbose)
71 PRT=2 ;;
72
73 -ind | --ind | -i | --i)
74 ac_prev=INDIR ;;
75 --ind=* | -ind=* | --i=* | -i=*)
76 INDIR=$ac_optarg ;;
77
78 -outd | --outd | -o | --o)
79 ac_prev=OUTDIR ;;
80 --outd=* | -outd=* | --o=* | -o=*)
81 OUTDIR=$ac_optarg ;;
82
83 -tempd | --tempd | -t | --t)
84 ac_prev=TEMPDIR ;;
85 --tempd=* | -tempd=* | --t=* | -t=*)
86 TEMPDIR=$ac_optarg ;;
87
88 -u | --u | -unpack | --unpack)
89 ac_prev=MUNPACK ;;
90 -u=* | --u=* | -unpack=* | --unpack=*)
91 MUNPACK=$ac_optarg ;;
92
93 -a | --a | -addr | --addr)
94 ac_prev=ADDRERR ;;
95 -a=* | --a=* | -addr=* | --addr=*)
96 ADDRERR=$ac_optarg ;;
97
98 *)
99 date
100 echo "Error: don't understand argument \"$ac_option\""
101 usage
102 ;;
103
104 esac
105
106 done
107
108 if test ! -x $MUNPACK ; then
109 date
110 echo "ERROR: \"$MUNPACK\" is not executable"
111 exit 2
112 fi
113 if test "x$OUTDIR" = x ; then
114 OUTDIR="$BASEDIR/$monthDir"
115 else
116 monthDir=0
117 fi
118 if test ! -e $OUTDIR ; then
119 mkdir $OUTDIR
120 RETVAL=$?
121 if test "x$RETVAL" != x0 ; then
122 date
123 echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
124 exit 3
125 else
126 echo "Successfully created new dir: \"$OUTDIR\""
127 fi
128 chgrp gcmpack $OUTDIR
129 chmod 775 $OUTDIR
130 fi
131
132 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
133 #- 1) check that $INDIR is empty
134 nbMsg=`ls -1 $INDIR | wc -l`
135 if test $nbMsg != 0 ; then
136 date
137 echo "ERROR: '$INDIR' is not empty ! --> exit"
138 exit 4
139 fi
140
141 nbMsg=0; ORIG=${RHOST}:$M_DIR
142 #- 2) scp all msg from ORIG to INDIR
143 cd $INDIR
144 scp -p ${RHOST}:$M_DIR/\* . > $STDOUT 2>&1
145 RETVAL=$?
146 if test "x$RETVAL" != x0 ; then
147 #echo " RETVAL='$RETVAL'"
148 RETVAL=`grep -v '^scp: No match\.$' $STDOUT | wc -l`
149 fi
150 if test "x$RETVAL" != x0 ; then
151 date
152 cat $STDOUT
153 echo "ERROR: when trying to scp msg from $ORIG to $INDIR"
154 rm -f $STDOUT
155 exit 5
156 fi
157 rm -f $STDOUT
158 nbMsg=`ls -1 | wc -l`
159 if test $nbMsg != 0 ; then
160 echo "> scp $nbMsg msg from '$ORIG' to '$INDIR'"
161 listMsg=`ls -1`
162 fi
163 cd $HERE
164 #echo " nbMsg='$nbMsg' ; listMsg='$listMsg'"
165
166 #- 3) remove from ORIG all msg that we got here (this way it should be safe)
167 if test $nbMsg != 0 ; then
168 #echo "ssh $RHOST cd $M_DIR \; /bin/rm -f $listMsg"
169 ssh $RHOST cd $M_DIR \; /bin/rm -f $listMsg
170 RETVAL=$?
171 if test "x$RETVAL" != x0 ; then
172 echo "ERROR ( $RETVAL ) when trying to ssh+rm $nbMsg msg from $ORIG"
173 #- note: Checking for return-value fron ssh command is not good enough,
174 # therefore we do not terminate if non-zero
175 #exit 6
176 fi
177 #- This is the log file that current "check_outp" (run on baudelaire) is checking:
178 logpfx="prc_emails_" ; sfx=`date +%m%d`
179 logfile="/net/zany/data/ORWELL/export-7/u/jmc/testing/logs/${logpfx}$sfx"
180 #- update log-file to trigger a new "check_outp":
181 if test -e $logfile ; then touch $logfile ; fi
182 fi
183 #echo 'Start processing msg locally'
184 #exit
185
186 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
187
188 all_msg=`ls -1 $INDIR`
189 nb_msg=`echo "$all_msg" | grep -c '^msg\.'`
190 nb_tar=`echo "$all_msg" | grep -c '\.tar\.gz$'`
191 nb_files=`expr $nb_msg + $nb_tar`
192
193 if test $PRT = 2 ; then
194 echo "Using OUTDIR=\"$OUTDIR\""
195 echo "Using INDIR=\"$INDIR\""
196 echo -n "Unpacking $nb_msg msg + $nb_tar tar-file ("`date`
197 if test "x$ADDRERR" != x ; then
198 echo -n ", err: $ADDRERR"
199 fi
200 echo ")"
201 elif test $nb_files != 0 ; then
202 echo -n "Unpacking $nb_msg msg + $nb_tar tar-file ("`date`
203 if test "x$ADDRERR" != x ; then
204 echo -n ", err: $ADDRERR"
205 fi
206 echo ")"
207 #echo " from '$INDIR' to '$OUTDIR'"
208 echo " to: '$OUTDIR'"
209 fi
210
211 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
212
213 #-- process list of message to detect group of multi-part msgs:
214 rm -f $TR_LIST
215 touch $TR_LIST
216
217 flag=0
218 for xx in $all_msg
219 do
220 in=`grep -c $xx $TR_LIST`
221 if test $in = 0 ; then
222 it=`echo $xx | grep -c '\.tar\.gz$'`
223 if test $it = 1 ; then
224 echo $xx >> $TR_LIST
225 else
226 np=`grep -c 'Content-Type: message/partial' $INDIR/$xx`
227 if test $np = 0 ; then
228 echo $xx >> $TR_LIST
229 else
230 l=`sed -n '/Content-Type: message\/partial/=' $INDIR/$xx`
231 lp=`expr $l + 1`
232 id=`sed -n "$lp p" $INDIR/$xx`
233 partM=`( cd $INDIR ; grep -c "$id" msg.* | grep -v ':0$' | sed 's/:1$//' )`
234 echo $partM >> $TR_LIST
235 if test "x$ADDRERR" != x ; then flag=1
236 echo "multi-parts message:" $partM >> $ERRMSG
237 ( cd $INDIR ; ls -l $partM ) >> $ERRMSG
238 fi
239 fi
240 fi
241 fi
242 done
243 if test $flag = 1 ; then
244 mail -s 'parse_emails Multi-parts msg' $ADDRERR < $ERRMSG
245 rm -f $ERRMSG
246 fi
247 #cat $TR_LIST ; echo '----------------------------------'
248
249 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
250
251 #-- process list of individual + group of multi-part messages
252 Nbl=`wc -l $TR_LIST | cut -d ' ' -f 1`
253 n=0
254 while [ $n -lt $Nbl ] ; do
255 n=`expr $n + 1`
256 errFlg=0 ; prcM=''
257 grpM=`sed -n "$n p" $TR_LIST`
258 nm=`sed -n "$n p" $TR_LIST | wc -w | cut -d ' ' -f 1`
259
260 #- create local copy
261 test -e $TEMPDIR && rm -rf $TEMPDIR
262 mkdir $TEMPDIR
263 RETVAL=$?
264 if test "x$RETVAL" = x0 ; then
265 ( cd $INDIR ; cp $grpM $TEMPDIR )
266 RETVAL=$?
267 fi
268 if test "x$RETVAL" != x0 ; then
269 if test "x$ADDRERR" != x ; then
270 echo "error parsing email '$grpM' :" > $ERRMSG
271 echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$grpM $TEMPDIR'" | tee -a $ERRMSG
272 echo " returns error $RETVAL" | tee -a $ERRMSG
273 mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
274 rm -f $ERRMSG
275 fi
276 ( cd $INDIR ; mv -f $grpM ../fail2process )
277 continue
278 fi
279
280 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
281 it=`echo $grpM | grep -c '\.tar\.gz$'`
282 if test $it = 1 ; then
283 #- nothing to do: already a tar file!
284 trOutp=$grpM ; prcM=$grpM
285 else
286
287 #-check that we have all the parts
288 if [ $PRT -ge 1 -a $nm -gt 1 ] ; then
289 echo " group (nm=$nm) of multi-parts msg: '$grpM'"
290 fi
291 for xx in $grpM ; do
292 np=`grep 'Content-Type: message/partial' $TEMPDIR/$xx \
293 | sed 's/^Content.*total=//' | sed 's/;$//'`
294 if test "x$np" = x -a $nm = 1 ; then np=1 ; fi
295 if test "x$np" != "x$nm" ; then
296 if test $errFlg = 0 ; then errFlg=1
297 echo "error parsing email '$grpM' :" > $ERRMSG
298 fi
299 echo " - Error: Number of parts='$np' in $xx but got '$nm' msg-files" \
300 | tee -a $ERRMSG
301 fi
302 done
303 if test $errFlg = 1 ; then
304 if test "x$ADDRERR" != x ; then
305 mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
306 fi
307 rm -f $ERRMSG
308 ( cd $INDIR ; mv -f $grpM ../fail2process )
309 continue
310 fi
311
312 if test $nm = 1 ; then
313 #- unpack single part message:
314 xx=$grpM
315 prcM=$xx
316 if test $PRT = 2 ; then echo " unpack single-part msg: '$xx'" ; fi
317 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
318 RETVAL=$?
319 if test "x$RETVAL" != x0 ; then errFlg=1
320 echo "error parsing email '$xx' :" > $ERRMSG
321 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
322 ls -l $INDIR"/"$xx >> $ERRMSG
323 else
324 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
325 if test ! -f $TEMPDIR/$trOutp ; then errFlg=1
326 echo "error parsing email '$xx' :" > $ERRMSG
327 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
328 | tee -a $ERRMSG
329 fi
330 fi
331 if test $errFlg = 1 ; then
332 cat $STDOUT | tee -a $ERRMSG
333 echo " <<<-----------------------"
334 if test "x$ADDRERR" != x ; then
335 mail -s 'parse_emails err_2s' $ADDRERR < $ERRMSG
336 fi
337 rm -f $ERRMSG
338 fi
339 rm -f $STDOUT
340
341 else
342 #- unpack group of multi-parts message:
343
344 #-unpack each part, in the right order:
345 j=0
346 while [ $j -lt $nm ] ; do
347 j=`expr $j + 1`
348 #- get the j^th part msg
349 xx=`( cd $TEMPDIR ; grep -c "Content-Type: message/partial; number=$j" $grpM \
350 | grep -v ':0$' | sed 's/:1$//' )`
351 if test $PRT = 2 ; then
352 echo -n " $j : xx='$xx' : "
353 grep 'Content-Type: message/partial' $TEMPDIR/$xx | sed 's/Content-Type: //'
354 fi
355 #- check
356 if test ! -f $TEMPDIR/$xx ; then
357 echo "error parsing email '$xx' :" > $ERRMSG
358 echo " - Error: Missing msg file '$xx' in $TEMPDIR:" | tee -a $ERRMSG
359 ls -l $TEMDIR | tee -a $ERRMSG
360 errFlg=1 ; echo " <<<-----------------------"
361 if test "x$ADDRERR" != x ; then
362 mail -s 'parse_emails err_2g' $ADDRERR < $ERRMSG
363 fi
364 rm -f $ERRMSG
365 else
366 if test $j = 1 ; then prcM=$xx ; else prcM="$prcM $xx" ; fi
367 #- do unpack
368 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
369 RETVAL=$?
370 if test "x$RETVAL" != x0 ; then errFlg=1
371 echo "error parsing email '$xx' :" > $ERRMSG
372 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
373 ls -l $INDIR"/"$xx | tee -a $ERRMSG
374 cat $STDOUT | tee -a $ERRMSG
375 errFlg=1 ; echo " <<<-----------------------"
376 if test "x$ADDRERR" != x ; then
377 mail -s 'parse_emails err_2u' $ADDRERR < $ERRMSG
378 fi
379 rm -f $ERRMSG
380 fi
381 fi
382 if test $errFlg = 1 ; then j=`expr $nm + 1` ; fi
383
384 if [ $j -lt $nm ] ; then
385 #- if not last part:
386 pfix=`cat $STDOUT | tail -1 | awk '{print $NF}'`
387 if test -d $UnpTmpD/$pfix ; then
388 lock=$UnpTmpD/$pfix/CT
389 if test -e $lock ; then
390 if test $PRT = 2 ; then echo " remove lock: $lock" ; fi
391 rm -f $lock
392 else
393 echo "error parsing email '$xx' :" > $ERRMSG
394 echo " - Error: lock file '$lock' not found from $MUNPACK output:" \
395 | tee -a $ERRMSG
396 cat $STDOUT | tee -a $ERRMSG
397 errFlg=1 ; echo " <<<-----------------------"
398 fi
399 else
400 echo "error parsing email '$xx' :" > $ERRMSG
401 echo " - Error: found no dir '$UnpTmpD/$pfix' from $MUNPACK output:" \
402 | tee -a $ERRMSG
403 cat $STDOUT | tee -a $ERRMSG
404 errFlg=1 ; echo " <<<-----------------------"
405 fi
406 elif [ $j -eq $nm ] ; then
407 #- if last part:
408 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
409 if test ! -f $TEMPDIR/$trOutp ; then
410 echo "error parsing email '$xx' :" > $ERRMSG
411 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
412 | tee -a $ERRMSG
413 cat $STDOUT | tee -a $ERRMSG
414 errFlg=1 ; echo " <<<-----------------------"
415 fi
416 fi
417 if [ $errFlg -eq 1 -a $j -le $nm ] ; then j=$nm
418 if test "x$ADDRERR" != x ; then
419 mail -s 'parse_emails err_2m' $ADDRERR < $ERRMSG
420 fi
421 rm -f $ERRMSG
422 fi
423 rm -f $STDOUT
424 done
425 fi
426 #-- in case of error: mv all $grpM msg to "fail2proc"
427 if test $errFlg = 1 ; then
428 ( cd $INDIR ; mv -f $grpM ../fail2process )
429 continue
430 fi
431 if test $PRT = 2 ; then
432 if test -f $TEMPDIR/$trOutp ; then ls -l $TEMPDIR/$trOutp ; fi
433 fi
434
435 fi
436 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
437
438 #-- un-tar
439 #( cd $TEMPDIR ; tar -xzvf $trOutp > $STDOUT )
440 # to remove small files "._mydir" that some MAC OS are adding
441 # (for each file or dir) to a tar-file, use option "--exclude=":
442 ( cd $TEMPDIR ; tar -xzvf $trOutp --exclude="._*" > $STDOUT 2> $ERRMSG )
443 RETVAL=$?
444 #echo "--- content of file $STDOUT (STDOUT):"
445 #cat $STDOUT
446 #echo "--- content of file $ERRMSG (ERRMSG):"
447 #cat $ERRMSG
448 if test "x$RETVAL" != x0 ; then
449 echo "parsing email error" >> $ERRMSG
450 echo " - Error: tar -xzvf $trOutp returns:" $RETVAL | tee -a $ERRMSG
451 ( cd $INDIR ; ls -l $prcM ) | tee -a $ERRMSG
452 ls -l $TEMPDIR/$trOutp | tee -a $ERRMSG
453 if test "x$ADDRERR" != x ; then
454 mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
455 fi
456 rm -f $ERRMSG
457 ( cd $INDIR ; mv -f $grpM ../fail2process )
458 continue
459 fi
460 test -f $ERRMSG && rm -f $ERRMSG
461 #--
462 tdir=`cat $STDOUT | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
463 if test -d $TEMPDIR/$tdir ; then
464 rm -f $STDOUT
465 else
466 echo "parsing email error" > $ERRMSG
467 echo " - Error: fail to get dir output name 'tdir=$tdir'" | tee -a $ERRMSG
468 echo " (cont) from tar file '$TEMPDIR/$trOutp'" | tee -a $ERRMSG
469 if test "x$ADDRERR" != x ; then
470 mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
471 fi
472 rm -f $ERRMSG
473 ( cd $INDIR ; mv -f $grpM ../fail2process )
474 continue
475 fi
476
477 #-- select which Monthly Output Dir:
478 locDir=$OUTDIR
479 if test "x$monthDir" != x0 ; then
480 dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
481 | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
482 mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
483 if test "x$mn" != "x$monthDir" ; then
484 if test "x$mn" = x ; then
485 if [ $PRT -ge 1 ] ; then echo " cannot get month from '$tdir'" ; fi
486 else
487 locDir="$BASEDIR/$mn"
488 if test ! -d $locDir ; then
489 if [ $PRT -ge 1 ] ; then
490 echo "NO DIR: '$locDir' => '$tdir' POSTPONED"
491 fi
492 if test "x$ADDRERR" != x ; then
493 echo "parsing email error" > $ERRMSG
494 echo " no dir '$locDir' for outp. '$tdir'" >> $ERRMSG
495 ( cd $INDIR ; ls -l $prcM ) >> $ERRMSG
496 mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
497 rm -f $ERRMSG
498 fi
499 continue
500 fi
501 fi
502 fi
503 fi
504
505 #-- copy to $locDir and rename if necessary
506 sdir=$tdir
507 # if test -e $locDir"/"$tdir ; then
508 sdir=`echo $tdir | sed 's/_[0-9]*$//'`
509 ad=0
510 while test -e $locDir"/"$sdir"_"$ad ; do
511 ad=$(( $ad + 1 ))
512 done
513 sdir=$sdir"_"$ad
514 # fi
515 if [ $PRT -ge 1 ] ; then
516 if test "x$locDir" = "x$OUTDIR"
517 then echo " '$sdir' ($prcM)"
518 else echo " '$sdir' ($prcM) => '$locDir'"
519 fi
520 fi
521 mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
522 RETVAL=$?
523 if test "x$RETVAL" != x0 ; then
524 if test "x$ADDRERR" != x ; then
525 echo "parsing email error" > $ERRMSG
526 echo " - Error: mv $TEMPDIR/$tdir $locDir/$sdir returns:" $RETVAL \
527 | tee -a $ERRMSG
528 echo -n " in dir: $TEMPDIR : " ; ls -l $TEMPDIR | tee -a $ERRMSG
529 echo -n " in dir: $OUTDIR : " ; ls -l $locDir | tee -a $ERRMSG
530 mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
531 rm -f $ERRMSG
532 fi
533 ( cd $INDIR ; mv -f $grpM ../fail2process )
534 continue
535 fi
536 chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
537
538 #-- remove the original message files
539 ( cd $INDIR ; rm -f $grpM )
540
541 done
542 rm -f $TR_LIST
543 if test $PRT = 2 ; then echo " done" ; fi

  ViewVC Help
Powered by ViewVC 1.1.22