/[MITgcm]/mitgcm.org/front_content/get+parse_msg
ViewVC logotype

Contents of /mitgcm.org/front_content/get+parse_msg

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (show annotations) (download)
Wed Jan 3 23:17:49 2018 UTC (3 years, 8 months ago) by jmc
Branch: MAIN
new script to scp and parse (similar to parse_emails) testreport-output messages

1 #! /usr/bin/env bash
2
3 # $Header: /u/gcmpack/mitgcm.org/front_content/parse_emails,v 1.28 2017/05/25 18:31:04 jmc Exp $
4 #
5 # The purpose of this script is to parse the emails produced by the
6 # MITgcm/verificaton/testreport script and store the data in a
7 # reasonable location.
8
9
10 usage()
11 {
12 echo
13 echo "Usage: $0 [OPTIONS]"
14 echo
15 echo "where possible OPTIONS are:"
16 echo " (-h|-help) print usage"
17 echo " (-s|-silent) silent mode"
18 echo " (-v|-verbose) verbose mode"
19 echo " (-i |-ind )DIR get mpack-created emails from DIR"
20 echo " [def=\"$INDIR\"]"
21 echo " (-o |-outd )DIR write the data to DIR"
22 echo " [def=\"$BASEDIR/$monthDir\"]"
23 echo " (-t |-tempd )DIR use temporary directory DIR"
24 echo " [def=\"$TEMPDIR\"]"
25 echo " (-u |-unpack )EXE use executable EXE to unpack e-mails"
26 echo " [def=\"$MUNPACK\"]"
27 echo " (-a |-addr )ADDR send e-mail to ADDR if Error"
28 echo " [def='"$ADDRERR"']"
29 echo
30 exit 1
31 }
32
33 # defaults
34 HERE=`pwd`
35 #INDIR="/u/u2/jmc/Mail/MITgcm-test"
36 #BASEDIR="/u/u0/httpd/html/testing/results"
37 RHOST="jm_c@mitgcm-mm.mit.edu" ; M_DIR="Mail/MITgcm-test"
38 INDIR="/home/jm_c/testing/MITgcm-test"
39 BASEDIR="/net/zany.mit.edu/data/ORWELL/export-9/mitgcm-testing/results"
40 monthDir=`date +%Y`"_"`date +%m`
41 OUTDIR=
42 ADDRERR=
43 MUNPACK=$HERE/munpack
44 UnpTmpD="/var/tmp/m-prts-$USER"
45 TR_LIST='TTT.'$$
46 TEMPDIR=/tmp/prc_emails
47 STDOUT=$TEMPDIR/'outp.'$$
48 ERRMSG=/tmp/tmp.$$
49 PRT=1
50
51 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
52 # Parse options
53 ac_prev=
54 for ac_option ; do
55
56 # If the previous option needs an argument, assign it.
57 if test -n "$ac_prev"; then
58 eval "$ac_prev=\$ac_option"
59 ac_prev=
60 continue
61 fi
62
63 ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
64
65 case $ac_option in
66
67 -help | --help | -h | --h)
68 usage ;;
69 -s | --s | -silent | --silent)
70 PRT=0 ;;
71 -v | --v | -verbose | --verbose)
72 PRT=2 ;;
73
74 -ind | --ind | -i | --i)
75 ac_prev=INDIR ;;
76 --ind=* | -ind=* | --i=* | -i=*)
77 INDIR=$ac_optarg ;;
78
79 -outd | --outd | -o | --o)
80 ac_prev=OUTDIR ;;
81 --outd=* | -outd=* | --o=* | -o=*)
82 OUTDIR=$ac_optarg ;;
83
84 -tempd | --tempd | -t | --t)
85 ac_prev=TEMPDIR ;;
86 --tempd=* | -tempd=* | --t=* | -t=*)
87 TEMPDIR=$ac_optarg ;;
88
89 -u | --u | -unpack | --unpack)
90 ac_prev=MUNPACK ;;
91 -u=* | --u=* | -unpack=* | --unpack=*)
92 MUNPACK=$ac_optarg ;;
93
94 -a | --a | -addr | --addr)
95 ac_prev=ADDRERR ;;
96 -a=* | --a=* | -addr=* | --addr=*)
97 ADDRERR=$ac_optarg ;;
98
99 *)
100 date
101 echo "Error: don't understand argument \"$ac_option\""
102 usage
103 ;;
104
105 esac
106
107 done
108
109 if test ! -x $MUNPACK ; then
110 date
111 echo "ERROR: \"$MUNPACK\" is not executable"
112 exit 2
113 fi
114 if test "x$OUTDIR" = x ; then
115 OUTDIR="$BASEDIR/$monthDir"
116 else
117 monthDir=0
118 fi
119 if test ! -e $OUTDIR ; then
120 mkdir $OUTDIR
121 RETVAL=$?
122 if test "x$RETVAL" != x0 ; then
123 date
124 echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
125 exit 3
126 else
127 echo "Successfully created new dir: \"$OUTDIR\""
128 fi
129 chgrp gcmpack $OUTDIR
130 chmod 775 $OUTDIR
131 fi
132
133 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
134 #- 1) check that $INDIR is empty
135 nbMsg=`ls -1 $INDIR | wc -l`
136 if test $nbMsg != 0 ; then
137 date
138 echo "ERROR: '$INDIR' is not empty ! --> exit"
139 exit 4
140 fi
141
142 nbMsg=0; ORIG=${RHOST}:$M_DIR
143 #- 2) scp all msg from ORIG to INDIR
144 cd $INDIR
145 scp -p ${RHOST}:$M_DIR/\* . > $STDOUT 2>&1
146 RETVAL=$?
147 if test "x$RETVAL" != x0 ; then
148 #echo " RETVAL='$RETVAL'"
149 RETVAL=`grep -v '^scp: No match\.$' $STDOUT | wc -l`
150 fi
151 if test "x$RETVAL" != x0 ; then
152 date
153 cat $STDOUT
154 echo "ERROR: when trying to scp msg from $ORIG to $INDIR"
155 rm -f $STDOUT
156 exit 5
157 fi
158 rm -f $STDOUT
159 nbMsg=`ls -1 | wc -l`
160 if test $nbMsg != 0 ; then
161 echo "> scp $nbMsg msg from '$ORIG' to '$INDIR'"
162 listMsg=`ls -1`
163 fi
164 cd $HERE
165 #echo " nbMsg='$nbMsg' ; listMsg='$listMsg'"
166
167 #- 3) remove from ORIG all msg that we got here (this way it should be safe)
168 if test $nbMsg != 0 ; then
169 #echo "ssh $RHOST cd $M_DIR \; /bin/rm -f $listMsg"
170 ssh $RHOST cd $M_DIR \; /bin/rm -f $listMsg
171 RETVAL=$?
172 if test "x$RETVAL" != x0 ; then
173 echo "ERROR ( $RETVAL ) when trying to ssh+rm $nbMsg msg from $ORIG"
174 #- note: Checking for return-value fron ssh command is not good enough,
175 # therefore we do not terminate if non-zero
176 #exit 6
177 fi
178 fi
179 #echo 'Start processing msg locally'
180 #exit
181
182 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
183
184 all_msg=`ls -1 $INDIR`
185 nb_msg=`echo "$all_msg" | grep -c '^msg\.'`
186 nb_tar=`echo "$all_msg" | grep -c '\.tar\.gz$'`
187 nb_files=`expr $nb_msg + $nb_tar`
188
189 if test $PRT = 2 ; then
190 echo "Using OUTDIR=\"$OUTDIR\""
191 echo "Using INDIR=\"$INDIR\""
192 echo -n "Unpacking $nb_msg msg + $nb_tar tar-file ("`date`
193 if test "x$ADDRERR" != x ; then
194 echo -n ", err: $ADDRERR"
195 fi
196 echo ")"
197 elif test $nb_files != 0 ; then
198 echo -n "Unpacking $nb_msg msg + $nb_tar tar-file ("`date`
199 if test "x$ADDRERR" != x ; then
200 echo -n ", err: $ADDRERR"
201 fi
202 echo ")"
203 #echo " from '$INDIR' to '$OUTDIR'"
204 echo " to: '$OUTDIR'"
205 fi
206
207 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
208
209 #-- process list of message to detect group of multi-part msgs:
210 rm -f $TR_LIST
211 touch $TR_LIST
212
213 flag=0
214 for xx in $all_msg
215 do
216 in=`grep -c $xx $TR_LIST`
217 if test $in = 0 ; then
218 it=`echo $xx | grep -c '\.tar\.gz$'`
219 if test $it = 1 ; then
220 echo $xx >> $TR_LIST
221 else
222 np=`grep -c 'Content-Type: message/partial' $INDIR/$xx`
223 if test $np = 0 ; then
224 echo $xx >> $TR_LIST
225 else
226 l=`sed -n '/Content-Type: message\/partial/=' $INDIR/$xx`
227 lp=`expr $l + 1`
228 id=`sed -n "$lp p" $INDIR/$xx`
229 partM=`( cd $INDIR ; grep -c "$id" msg.* | grep -v ':0$' | sed 's/:1$//' )`
230 echo $partM >> $TR_LIST
231 if test "x$ADDRERR" != x ; then flag=1
232 echo "multi-parts message:" $partM >> $ERRMSG
233 ( cd $INDIR ; ls -l $partM ) >> $ERRMSG
234 fi
235 fi
236 fi
237 fi
238 done
239 if test $flag = 1 ; then
240 mail -s 'parse_emails Multi-parts msg' $ADDRERR < $ERRMSG
241 rm -f $ERRMSG
242 fi
243 #cat $TR_LIST ; echo '----------------------------------'
244
245 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
246
247 #-- process list of individual + group of multi-part messages
248 Nbl=`wc -l $TR_LIST | cut -d ' ' -f 1`
249 n=0
250 while [ $n -lt $Nbl ] ; do
251 n=`expr $n + 1`
252 errFlg=0 ; prcM=''
253 grpM=`sed -n "$n p" $TR_LIST`
254 nm=`sed -n "$n p" $TR_LIST | wc -w | cut -d ' ' -f 1`
255
256 #- create local copy
257 test -e $TEMPDIR && rm -rf $TEMPDIR
258 mkdir $TEMPDIR
259 RETVAL=$?
260 if test "x$RETVAL" = x0 ; then
261 ( cd $INDIR ; cp $grpM $TEMPDIR )
262 RETVAL=$?
263 fi
264 if test "x$RETVAL" != x0 ; then
265 if test "x$ADDRERR" != x ; then
266 echo "error parsing email '$grpM' :" > $ERRMSG
267 echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$grpM $TEMPDIR'" | tee -a $ERRMSG
268 echo " returns error $RETVAL" | tee -a $ERRMSG
269 mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
270 rm -f $ERRMSG
271 fi
272 ( cd $INDIR ; mv -f $grpM ../fail2process )
273 continue
274 fi
275
276 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
277 it=`echo $grpM | grep -c '\.tar\.gz$'`
278 if test $it = 1 ; then
279 #- nothing to do: already a tar file!
280 trOutp=$grpM ; prcM=$grpM
281 else
282
283 #-check that we have all the parts
284 if [ $PRT -ge 1 -a $nm -gt 1 ] ; then
285 echo " group (nm=$nm) of multi-parts msg: '$grpM'"
286 fi
287 for xx in $grpM ; do
288 np=`grep 'Content-Type: message/partial' $TEMPDIR/$xx \
289 | sed 's/^Content.*total=//' | sed 's/;$//'`
290 if test "x$np" = x -a $nm = 1 ; then np=1 ; fi
291 if test "x$np" != "x$nm" ; then
292 if test $errFlg = 0 ; then errFlg=1
293 echo "error parsing email '$grpM' :" > $ERRMSG
294 fi
295 echo " - Error: Number of parts='$np' in $xx but got '$nm' msg-files" \
296 | tee -a $ERRMSG
297 fi
298 done
299 if test $errFlg = 1 ; then
300 if test "x$ADDRERR" != x ; then
301 mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
302 fi
303 rm -f $ERRMSG
304 ( cd $INDIR ; mv -f $grpM ../fail2process )
305 continue
306 fi
307
308 if test $nm = 1 ; then
309 #- unpack single part message:
310 xx=$grpM
311 prcM=$xx
312 if test $PRT = 2 ; then echo " unpack single-part msg: '$xx'" ; fi
313 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
314 RETVAL=$?
315 if test "x$RETVAL" != x0 ; then errFlg=1
316 echo "error parsing email '$xx' :" > $ERRMSG
317 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
318 ls -l $INDIR"/"$xx >> $ERRMSG
319 else
320 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
321 if test ! -f $TEMPDIR/$trOutp ; then errFlg=1
322 echo "error parsing email '$xx' :" > $ERRMSG
323 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
324 | tee -a $ERRMSG
325 fi
326 fi
327 if test $errFlg = 1 ; then
328 cat $STDOUT | tee -a $ERRMSG
329 echo " <<<-----------------------"
330 if test "x$ADDRERR" != x ; then
331 mail -s 'parse_emails err_2s' $ADDRERR < $ERRMSG
332 fi
333 rm -f $ERRMSG
334 fi
335 rm -f $STDOUT
336
337 else
338 #- unpack group of multi-parts message:
339
340 #-unpack each part, in the right order:
341 j=0
342 while [ $j -lt $nm ] ; do
343 j=`expr $j + 1`
344 #- get the j^th part msg
345 xx=`( cd $TEMPDIR ; grep -c "Content-Type: message/partial; number=$j" $grpM \
346 | grep -v ':0$' | sed 's/:1$//' )`
347 if test $PRT = 2 ; then
348 echo -n " $j : xx='$xx' : "
349 grep 'Content-Type: message/partial' $TEMPDIR/$xx | sed 's/Content-Type: //'
350 fi
351 #- check
352 if test ! -f $TEMPDIR/$xx ; then
353 echo "error parsing email '$xx' :" > $ERRMSG
354 echo " - Error: Missing msg file '$xx' in $TEMPDIR:" | tee -a $ERRMSG
355 ls -l $TEMDIR | tee -a $ERRMSG
356 errFlg=1 ; echo " <<<-----------------------"
357 if test "x$ADDRERR" != x ; then
358 mail -s 'parse_emails err_2g' $ADDRERR < $ERRMSG
359 fi
360 rm -f $ERRMSG
361 else
362 if test $j = 1 ; then prcM=$xx ; else prcM="$prcM $xx" ; fi
363 #- do unpack
364 ( cd $TEMPDIR ; $MUNPACK $xx > $STDOUT 2>&1 )
365 RETVAL=$?
366 if test "x$RETVAL" != x0 ; then errFlg=1
367 echo "error parsing email '$xx' :" > $ERRMSG
368 echo " - Error: $MUNPACK $xx returns: $RETVAL" | tee -a $ERRMSG
369 ls -l $INDIR"/"$xx | tee -a $ERRMSG
370 cat $STDOUT | tee -a $ERRMSG
371 errFlg=1 ; echo " <<<-----------------------"
372 if test "x$ADDRERR" != x ; then
373 mail -s 'parse_emails err_2u' $ADDRERR < $ERRMSG
374 fi
375 rm -f $ERRMSG
376 fi
377 fi
378 if test $errFlg = 1 ; then j=`expr $nm + 1` ; fi
379
380 if [ $j -lt $nm ] ; then
381 #- if not last part:
382 pfix=`cat $STDOUT | tail -1 | awk '{print $NF}'`
383 if test -d $UnpTmpD/$pfix ; then
384 lock=$UnpTmpD/$pfix/CT
385 if test -e $lock ; then
386 if test $PRT = 2 ; then echo " remove lock: $lock" ; fi
387 rm -f $lock
388 else
389 echo "error parsing email '$xx' :" > $ERRMSG
390 echo " - Error: lock file '$lock' not found from $MUNPACK output:" \
391 | tee -a $ERRMSG
392 cat $STDOUT | tee -a $ERRMSG
393 errFlg=1 ; echo " <<<-----------------------"
394 fi
395 else
396 echo "error parsing email '$xx' :" > $ERRMSG
397 echo " - Error: found no dir '$UnpTmpD/$pfix' from $MUNPACK output:" \
398 | tee -a $ERRMSG
399 cat $STDOUT | tee -a $ERRMSG
400 errFlg=1 ; echo " <<<-----------------------"
401 fi
402 elif [ $j -eq $nm ] ; then
403 #- if last part:
404 trOutp=`tail -1 $STDOUT | cut -d ' ' -f 1`
405 if test ! -f $TEMPDIR/$trOutp ; then
406 echo "error parsing email '$xx' :" > $ERRMSG
407 echo " - Error: Missing output file '$trOutp' from $MUNPACK output:" \
408 | tee -a $ERRMSG
409 cat $STDOUT | tee -a $ERRMSG
410 errFlg=1 ; echo " <<<-----------------------"
411 fi
412 fi
413 if [ $errFlg -eq 1 -a $j -le $nm ] ; then j=$nm
414 if test "x$ADDRERR" != x ; then
415 mail -s 'parse_emails err_2m' $ADDRERR < $ERRMSG
416 fi
417 rm -f $ERRMSG
418 fi
419 rm -f $STDOUT
420 done
421 fi
422 #-- in case of error: mv all $grpM msg to "fail2proc"
423 if test $errFlg = 1 ; then
424 ( cd $INDIR ; mv -f $grpM ../fail2process )
425 continue
426 fi
427 if test $PRT = 2 ; then
428 if test -f $TEMPDIR/$trOutp ; then ls -l $TEMPDIR/$trOutp ; fi
429 fi
430
431 fi
432 #---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
433
434 #-- un-tar
435 #( cd $TEMPDIR ; tar -xzvf $trOutp > $STDOUT )
436 # to remove small files "._mydir" that some MAC OS are adding
437 # (for each file or dir) to a tar-file, use option "--exclude=":
438 ( cd $TEMPDIR ; tar -xzvf $trOutp --exclude="._*" > $STDOUT 2> $ERRMSG )
439 RETVAL=$?
440 #echo "--- content of file $STDOUT (STDOUT):"
441 #cat $STDOUT
442 #echo "--- content of file $ERRMSG (ERRMSG):"
443 #cat $ERRMSG
444 if test "x$RETVAL" != x0 ; then
445 echo "parsing email error" >> $ERRMSG
446 echo " - Error: tar -xzvf $trOutp returns:" $RETVAL | tee -a $ERRMSG
447 ( cd $INDIR ; ls -l $prcM ) | tee -a $ERRMSG
448 ls -l $TEMPDIR/$trOutp | tee -a $ERRMSG
449 if test "x$ADDRERR" != x ; then
450 mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
451 fi
452 rm -f $ERRMSG
453 ( cd $INDIR ; mv -f $grpM ../fail2process )
454 continue
455 fi
456 test -f $ERRMSG && rm -f $ERRMSG
457 #--
458 tdir=`cat $STDOUT | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
459 if test -d $TEMPDIR/$tdir ; then
460 rm -f $STDOUT
461 else
462 echo "parsing email error" > $ERRMSG
463 echo " - Error: fail to get dir output name 'tdir=$tdir'" | tee -a $ERRMSG
464 echo " (cont) from tar file '$TEMPDIR/$trOutp'" | tee -a $ERRMSG
465 if test "x$ADDRERR" != x ; then
466 mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
467 fi
468 rm -f $ERRMSG
469 ( cd $INDIR ; mv -f $grpM ../fail2process )
470 continue
471 fi
472
473 #-- select which Monthly Output Dir:
474 locDir=$OUTDIR
475 if test "x$monthDir" != x0 ; then
476 dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
477 | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
478 mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
479 if test "x$mn" != "x$monthDir" ; then
480 if test "x$mn" = x ; then
481 if [ $PRT -ge 1 ] ; then echo " cannot get month from '$tdir'" ; fi
482 else
483 locDir="$BASEDIR/$mn"
484 if test ! -d $locDir ; then
485 if [ $PRT -ge 1 ] ; then
486 echo "NO DIR: '$locDir' => '$tdir' POSTPONED"
487 fi
488 if test "x$ADDRERR" != x ; then
489 echo "parsing email error" > $ERRMSG
490 echo " no dir '$locDir' for outp. '$tdir'" >> $ERRMSG
491 ( cd $INDIR ; ls -l $prcM ) >> $ERRMSG
492 mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
493 rm -f $ERRMSG
494 fi
495 continue
496 fi
497 fi
498 fi
499 fi
500
501 #-- copy to $locDir and rename if necessary
502 sdir=$tdir
503 # if test -e $locDir"/"$tdir ; then
504 sdir=`echo $tdir | sed 's/_[0-9]*$//'`
505 ad=0
506 while test -e $locDir"/"$sdir"_"$ad ; do
507 ad=$(( $ad + 1 ))
508 done
509 sdir=$sdir"_"$ad
510 # fi
511 if [ $PRT -ge 1 ] ; then
512 if test "x$locDir" = "x$OUTDIR"
513 then echo " '$sdir' ($prcM)"
514 else echo " '$sdir' ($prcM) => '$locDir'"
515 fi
516 fi
517 mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
518 RETVAL=$?
519 if test "x$RETVAL" != x0 ; then
520 if test "x$ADDRERR" != x ; then
521 echo "parsing email error" > $ERRMSG
522 echo " - Error: mv $TEMPDIR/$tdir $locDir/$sdir returns:" $RETVAL \
523 | tee -a $ERRMSG
524 echo -n " in dir: $TEMPDIR : " ; ls -l $TEMPDIR | tee -a $ERRMSG
525 echo -n " in dir: $OUTDIR : " ; ls -l $locDir | tee -a $ERRMSG
526 mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
527 rm -f $ERRMSG
528 fi
529 ( cd $INDIR ; mv -f $grpM ../fail2process )
530 continue
531 fi
532 chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
533
534 #-- remove the original message files
535 ( cd $INDIR ; rm -f $grpM )
536
537 done
538 rm -f $TR_LIST
539 if test $PRT = 2 ; then echo " done" ; fi

  ViewVC Help
Powered by ViewVC 1.1.22