--- mitgcm.org/front_content/parse_emails	2008/02/28 20:03:45	1.15
+++ mitgcm.org/front_content/parse_emails	2009/12/03 18:03:27	1.21
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/front_content/parse_emails,v 1.15 2008/02/28 20:03:45 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/front_content/parse_emails,v 1.21 2009/12/03 18:03:27 jmc Exp $
 #
 #  The purpose of this script is to parse the emails produced by the
 #  MITgcm/verificaton/testreport script and store the data in a
@@ -11,7 +11,7 @@
 {
     echo
     echo "Usage:  $0 [OPTIONS]"
-    echo 
+    echo
     echo "where possible OPTIONS are:"
     echo "  (-h|-help)           print usage"
     echo "  (-s|-silent)         silent mode"
@@ -19,20 +19,22 @@
     echo "  (-i |-ind )DIR       get mpack-created emails from DIR"
     echo "                         [def=\"$INDIR\"]"
     echo "  (-o |-outd )DIR      write the data to DIR"
-    echo "                         [def=\"$OUTDIR\"]"
+    echo "                         [def=\"$BASEDIR/$monthDir\"]"
     echo "  (-t |-tempd )DIR     use temporary directory DIR"
     echo "                         [def=\"$TEMPDIR\"]"
     echo "  (-u |-unpack )EXE    use executable EXE to unpack e-mails"
     echo "                         [def=\"$MUNPACK\"]"
     echo "  (-a |-addr )ADDR     send e-mail to ADDR if Error"
     echo "                         [def='"$ADDRERR"']"
-    echo 
+    echo
     exit 1
 }
 
 # defaults
 INDIR="/u/u2/jmc/Mail/MITgcm-test"
-OUTDIR="/u/u0/httpd/html/testing/results/"`date +%Y`"_"`date +%m`
+BASEDIR="/u/u0/httpd/html/testing/results"
+monthDir=`date +%Y`"_"`date +%m`
+OUTDIR=
 TEMPDIR=./ptmp
 MUNPACK=munpack
 ADDRERR=
@@ -50,7 +52,7 @@
     fi
 
     ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
-    
+
     case $ac_option in
 	
 	-help | --help | -h | --h)
@@ -64,12 +66,12 @@
             ac_prev=INDIR ;;
         --ind=* | -ind=* | --i=* | -i=*)
             INDIR=$ac_optarg ;;
-        
+
         -outd | --outd | -o | --o)
             ac_prev=OUTDIR ;;
         --outd=* | -outd=* | --o=* | -o=*)
             OUTDIR=$ac_optarg ;;
-        
+
         -tempd | --tempd | -t | --t)
             ac_prev=TEMPDIR ;;
         --tempd=* | -tempd=* | --t=* | -t=*)
@@ -79,7 +81,7 @@
             ac_prev=MUNPACK ;;
         -u=* | --u=* | -unpack=* | --unpack=*)
             MUNPACK=$ac_optarg ;;
-        
+
         -a | --a | -addr | --addr)
             ac_prev=ADDRERR ;;
         -a=* | --a=* | -addr=* | --addr=*)
@@ -87,24 +89,34 @@
 
         *)
 	    # copy the file list to FL_#
+	    date
 	    echo "Error: don't understand argument \"$ac_option\""
 	    usage
             ;;
-        
+
      esac
-     
+
 done
 
 if test ! -x $MUNPACK ; then
+	date
 	echo "ERROR: \"$MUNPACK\" is not executable"
 	exit 2
 fi
+if test "x$OUTDIR" = x ; then
+  OUTDIR="$BASEDIR/$monthDir"
+else
+  monthDir=0
+fi
 if test ! -e $OUTDIR ; then
     mkdir $OUTDIR
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
+	date
 	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
 	exit 3
+    else
+	echo "Successfully created new dir: \"$OUTDIR\""
     fi
     chgrp gcmpack $OUTDIR
     chmod 775 $OUTDIR
@@ -118,16 +130,17 @@
   echo "Using INDIR=\"$INDIR\""
   echo -n "Unpacking the emails ..."
 elif test $nb_files != 0 ; then
-  echo "Unpacking $nb_files emails from '$INDIR' to '$OUTDIR'"
+  echo -n "Unpacking $nb_files emails ("`date`
   if test "x$ADDRERR" != x ; then
-     echo " (send msg to '$ADDRERR' if Error)"
+     echo -n ", err-msg: '$ADDRERR'"
   fi
-  if test $PRT = 2 ; then echo -n " dir:" ; fi
+  echo ")"
+  echo " from '$INDIR' to '$OUTDIR'"
 fi
 
 for file in $all_files ; do
 
-    #  create local copy
+    #-- create local copy
     test -e $TEMPDIR  &&  rm -rf $TEMPDIR
     mkdir $TEMPDIR
     RETVAL=$?
@@ -148,7 +161,7 @@
       continue
     fi
 
-    #  ignore multi-part messages
+    #-- ignore multi-part messages
     grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
     RETVAL=$?
     if test "x$RETVAL" = x0 ; then
@@ -163,7 +176,7 @@
       continue
     fi
 
-    #  munpack
+    #-- munpack
     mun=`( cd $TEMPDIR ; $MUNPACK $file | cut -d ' ' -f 1 | head -1 )`
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
@@ -178,8 +191,11 @@
       continue
     fi
 
-    #  un-tar
-    ( cd $TEMPDIR ; tar -xzvf $mun > out )
+    #-- un-tar
+    #( cd $TEMPDIR ; tar -xzvf $mun > out )
+    #   to remove small files "._mydir" that some MAC OS are adding
+    #   (for each file or dir) to a tar-file, use option "--exclude=":
+    ( cd $TEMPDIR ; tar -xzvf $mun --exclude="._*" > out )
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
       if test "x$ADDRERR" != x ; then
@@ -187,46 +203,90 @@
         echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
         ls -l $INDIR"/"$file >> tmp.$$
         ls -l $mun >> tmp.$$
-        mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
+        mail -s 'parse_emails err_3a' $ADDRERR < tmp.$$
         rm -f tmp.$$
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
     tdir=`cat $TEMPDIR"/out" | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
-    rm -f $TEMPDIR"/out"
+   #tdir=`( cd $TEMPDIR ; /bin/ls -l | grep '^d' | head -1 | awk '{print $NF}' )`
+    if test -d $TEMPDIR/$tdir ; then
+      rm -f $TEMPDIR"/out"
+    else
+      if test "x$ADDRERR" != x ; then
+        echo "parsing email error" > tmp.$$
+        echo " fail to get a dir output name 'tdir=$tdir'" >> tmp.$$
+        echo " from tar file '$TEMPDIR/$mun'" >> tmp.$$
+        mail -s 'parse_emails err_3b' $ADDRERR < tmp.$$
+        rm -f tmp.$$
+      fi
+      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
+      continue
+    fi
 
-    #  copy to $OUTDIR and rename if necessary
+    #-- select which Monthly Output Dir:
+    locDir=$OUTDIR
+    if test "x$monthDir" != x0 ; then
+      dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
+               | grep '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' | tail -1`
+      mn=`echo $dd | sed 's/..$//' | sed 's/..$/_&/'`
+      if test "x$mn" != "x$monthDir" ; then
+       if test "x$mn" = x ; then
+        if test $PRT = 2 ; then echo " cannot get month from '$tdir'" ; fi
+       else
+       #  could comment out this line:
+       #if test $PRT = 2 ; then echo " chg month: '$mn' for '$tdir'" ; fi
+        locDir="$BASEDIR/$mn"
+        if test ! -d $locDir ; then
+          if test $PRT = 2 ; then echo "NO DIR: '$locDir' => '$tdir' POSTPONED" ; fi
+          if test "x$ADDRERR" != x ; then
+            echo "parsing email error" > tmp.$$
+            echo "no dir '$locDir' for outp. '$tdir'" > tmp.$$
+            ls -l $INDIR"/"$file >> tmp.$$
+            mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
+            rm -f tmp.$$
+          fi
+          continue
+        fi
+       fi
+      fi
+    fi
+
+    #-- copy to $locDir and rename if necessary
     sdir=$tdir
-    if test -e $OUTDIR"/"$tdir ; then
+    if test -e $locDir"/"$tdir ; then
 	ad=0
-	while test -e $OUTDIR"/"$tdir"_"$ad ; do
+	while test -e $locDir"/"$tdir"_"$ad ; do
 	    ad=$(( $ad + 1 ))
 	done
         sdir=$tdir"_"$ad
     fi
-    if test $PRT = 2 ; then echo -n " '$sdir'" ; fi
-    mv $TEMPDIR"/"$tdir $OUTDIR"/"$sdir > /dev/null 2>&1
+    if test $PRT = 2 ; then
+      if test "x$locDir" = "x$OUTDIR"
+      then echo " '$sdir'"
+      else echo " '$sdir' => '$locDir'"
+      fi
+    fi
+    mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
       if test "x$ADDRERR" != x ; then
         echo "parsing email error" > tmp.$$
-        echo "mv $TEMPDIR/$tdir $OUTDIR/$sdir returns error:" $RETVAL >> tmp.$$
+        echo "mv $TEMPDIR/$tdir $locDir/$sdir returns error:" $RETVAL >> tmp.$$
         echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR  >> tmp.$$
-        echo -n "in dir: $OUTDIR : " ; ls -l $OUTDIR  >> tmp.$$
-        mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
+        echo -n "in dir: $OUTDIR : " ; ls -l $locDir  >> tmp.$$
+        mail -s 'parse_emails err_5' $ADDRERR < tmp.$$
         rm -f tmp.$$
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
-    chmod -R a+rx $OUTDIR"/"$sdir > /dev/null 2>&1
-#   gzip $OUTDIR"/"$sdir"/output.txt"
+    chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
 
-    #  remove the original file
+    #-- remove the original file
     rm -f $INDIR"/"$file
 
 done
 if test $PRT = 1 ; then echo "  done" ; fi
-if test $PRT = 2 -a $nb_files != 0 ; then echo "" ; fi