--- mitgcm.org/front_content/parse_emails	2009/02/02 19:16:45	1.19
+++ mitgcm.org/front_content/parse_emails	2010/10/31 02:25:05	1.24
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/front_content/parse_emails,v 1.19 2009/02/02 19:16:45 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/front_content/parse_emails,v 1.24 2010/10/31 02:25:05 jmc Exp $
 #
 #  The purpose of this script is to parse the emails produced by the
 #  MITgcm/verificaton/testreport script and store the data in a
@@ -35,7 +35,8 @@
 BASEDIR="/u/u0/httpd/html/testing/results"
 monthDir=`date +%Y`"_"`date +%m`
 OUTDIR=
-TEMPDIR=./ptmp
+TEMPDIR=/tmp/prc_emails
+ERRMSG=/tmp/tmp.$$
 MUNPACK=munpack
 ADDRERR=
 PRT=1
@@ -115,6 +116,8 @@
 	date
 	echo "ERROR: directory \"$OUTDIR\" doesn't exist and can't be created"
 	exit 3
+    else
+	echo "Successfully created new dir: \"$OUTDIR\""
     fi
     chgrp gcmpack $OUTDIR
     chmod 775 $OUTDIR
@@ -138,7 +141,7 @@
 
 for file in $all_files ; do
 
-    #  create local copy
+    #-- create local copy
     test -e $TEMPDIR  &&  rm -rf $TEMPDIR
     mkdir $TEMPDIR
     RETVAL=$?
@@ -148,66 +151,81 @@
     fi
     if test "x$RETVAL" != x0 ; then
       if test "x$ADDRERR" != x ; then
-        echo "parsing email error" > tmp.$$
-        echo " processing file: '$INDIR/$file'" >> tmp.$$
-        echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> tmp.$$
-        echo " returns error $RETVAL" >> tmp.$$
-        mail -s 'parse_emails err_0' $ADDRERR < tmp.$$
-        rm -f tmp.$$
+        echo "parsing email error" > $ERRMSG
+        echo " processing file: '$INDIR/$file'" >> $ERRMSG
+        echo -n "'mkdir $TEMPDIR' or 'cp $INDIR/$file $TEMPDIR'" >> $ERRMSG
+        echo " returns error $RETVAL" >> $ERRMSG
+        mail -s 'parse_emails err_0' $ADDRERR < $ERRMSG
+        rm -f $ERRMSG
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
 
-    #  ignore multi-part messages
+    #-- ignore multi-part messages
     grep "Content-Type: message/partial" $INDIR"/"$file > /dev/null 2>&1
     RETVAL=$?
     if test "x$RETVAL" = x0 ; then
       if test "x$ADDRERR" != x ; then
-        echo "parsing email error" > tmp.$$
-        echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> tmp.$$
-        ls -l $INDIR"/"$file >> tmp.$$
-        mail -s 'parse_emails err_1' $ADDRERR < tmp.$$
-        rm -f tmp.$$
+        echo "parsing email error" > $ERRMSG
+        echo 'grep "Content-Type: message/partial" returns error:' $RETVAL >> $ERRMSG
+        ls -l $INDIR"/"$file >> $ERRMSG
+        mail -s 'parse_emails err_1' $ADDRERR < $ERRMSG
+        rm -f $ERRMSG
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
 
-    #  munpack
+    #-- munpack
     mun=`( cd $TEMPDIR ; $MUNPACK $file | cut -d ' ' -f 1 | head -1 )`
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
       if test "x$ADDRERR" != x ; then
-        echo "parsing email error" > tmp.$$
-        echo "$MUNPACK $file returns error: $RETVAL" >> tmp.$$
-        ls -l $INDIR"/"$file >> tmp.$$
-        mail -s 'parse_emails err_2' $ADDRERR < tmp.$$
-        rm -f tmp.$$
+        echo "parsing email error" > $ERRMSG
+        echo "$MUNPACK $file returns error: $RETVAL" >> $ERRMSG
+        ls -l $INDIR"/"$file >> $ERRMSG
+        mail -s 'parse_emails err_2' $ADDRERR < $ERRMSG
+        rm -f $ERRMSG
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
 
-    #  un-tar
-    ( cd $TEMPDIR ; tar -xzvf $mun > out )
+    #-- un-tar
+    #( cd $TEMPDIR ; tar -xzvf $mun > out )
+    #   to remove small files "._mydir" that some MAC OS are adding
+    #   (for each file or dir) to a tar-file, use option "--exclude=":
+    ( cd $TEMPDIR ; tar -xzvf $mun --exclude="._*" > out )
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
       if test "x$ADDRERR" != x ; then
-        echo "parsing email error" > tmp.$$
-        echo "tar -xzvf $mun returns error:" $RETVAL >> tmp.$$
-        ls -l $INDIR"/"$file >> tmp.$$
-        ls -l $mun >> tmp.$$
-        mail -s 'parse_emails err_3' $ADDRERR < tmp.$$
-        rm -f tmp.$$
+        echo "parsing email error" > $ERRMSG
+        echo "tar -xzvf $mun returns error:" $RETVAL >> $ERRMSG
+        ls -l $INDIR"/"$file >> $ERRMSG
+        ls -l $mun >> $ERRMSG
+        mail -s 'parse_emails err_3a' $ADDRERR < $ERRMSG
+        rm -f $ERRMSG
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
     tdir=`cat $TEMPDIR"/out" | head -1 | sed -e 's|^./||g' | cut -d '/' -f 1`
-    rm -f $TEMPDIR"/out"
+    if test -d $TEMPDIR/$tdir ; then
+      rm -f $TEMPDIR"/out"
+    else
+      if test "x$ADDRERR" != x ; then
+        echo "parsing email error" > $ERRMSG
+        echo " fail to get a dir output name 'tdir=$tdir'" >> $ERRMSG
+        echo " from tar file '$TEMPDIR/$mun'" >> $ERRMSG
+        mail -s 'parse_emails err_3b' $ADDRERR < $ERRMSG
+        rm -f $ERRMSG
+      fi
+      mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
+      continue
+    fi
 
-    #  select which Monthly Output Dir:
+    #-- select which Monthly Output Dir:
     locDir=$OUTDIR
     if test "x$monthDir" != x0 ; then
       dd=`echo $tdir | sed 's/_/ /g' | awk '{ for(i=1;i<=NF;i++) print $i }'\
@@ -223,11 +241,11 @@
         if test ! -d $locDir ; then
           if test $PRT = 2 ; then echo "NO DIR: '$locDir' => '$tdir' POSTPONED" ; fi
           if test "x$ADDRERR" != x ; then
-            echo "parsing email error" > tmp.$$
-            echo "no dir '$locDir' for outp. '$tdir'" > tmp.$$
-            ls -l $INDIR"/"$file >> tmp.$$
-            mail -s 'parse_emails err_4' $ADDRERR < tmp.$$
-            rm -f tmp.$$
+            echo "parsing email error" > $ERRMSG
+            echo "no dir '$locDir' for outp. '$tdir'" > $ERRMSG
+            ls -l $INDIR"/"$file >> $ERRMSG
+            mail -s 'parse_emails err_4' $ADDRERR < $ERRMSG
+            rm -f $ERRMSG
           fi
           continue
         fi
@@ -235,38 +253,39 @@
       fi
     fi
 
-    #  copy to $locDir and rename if necessary
+    #-- copy to $locDir and rename if necessary
     sdir=$tdir
     if test -e $locDir"/"$tdir ; then
+        sdir=`echo $tdir | sed 's/_[0-9]*$//'`
 	ad=0
-	while test -e $locDir"/"$tdir"_"$ad ; do
+	while test -e $locDir"/"$sdir"_"$ad ; do
 	    ad=$(( $ad + 1 ))
 	done
-        sdir=$tdir"_"$ad
+        sdir=$sdir"_"$ad
     fi
     if test $PRT = 2 ; then
       if test "x$locDir" = "x$OUTDIR"
-      then echo " '$sdir'"
-      else echo " '$sdir' => '$locDir'"
+      then echo " '$sdir' ($file)"
+      else echo " '$sdir' ($file) => '$locDir'"
       fi
     fi
     mv $TEMPDIR"/"$tdir $locDir"/"$sdir > /dev/null 2>&1
     RETVAL=$?
     if test "x$RETVAL" != x0 ; then
       if test "x$ADDRERR" != x ; then
-        echo "parsing email error" > tmp.$$
-        echo "mv $TEMPDIR/$tdir $locDir/$sdir returns error:" $RETVAL >> tmp.$$
-        echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR  >> tmp.$$
-        echo -n "in dir: $OUTDIR : " ; ls -l $locDir  >> tmp.$$
-        mail -s 'parse_emails err_5' $ADDRERR < tmp.$$
-        rm -f tmp.$$
+        echo "parsing email error" > $ERRMSG
+        echo "mv $TEMPDIR/$tdir $locDir/$sdir returns error:" $RETVAL >> $ERRMSG
+        echo -n "in dir: $TEMPDIR : " ; ls -l $TEMPDIR  >> $ERRMSG
+        echo -n "in dir: $OUTDIR : " ; ls -l $locDir  >> $ERRMSG
+        mail -s 'parse_emails err_5' $ADDRERR < $ERRMSG
+        rm -f $ERRMSG
       fi
       mv -f $INDIR"/"$file $INDIR"/../fail2process/"$file
       continue
     fi
     chmod -R a+rx $locDir"/"$sdir > /dev/null 2>&1
 
-    #  remove the original file
+    #-- remove the original file
     rm -f $INDIR"/"$file
 
 done