--- MITgcm/eesupp/src/eedie.F	1998/04/22 19:15:30	1.1.1.1
+++ MITgcm/eesupp/src/eedie.F	2012/10/11 19:15:18	1.13
@@ -1,81 +1,110 @@
-C $Id: eedie.F,v 1.1.1.1 1998/04/22 19:15:30 cnh Exp $
+C $Header: /home/ubuntu/mnt/e9_copy/MITgcm/eesupp/src/eedie.F,v 1.13 2012/10/11 19:15:18 jmc Exp $
+C $Name:  $
 
 #include "CPP_EEOPTIONS.h"
+#ifdef USE_LIBHPM
+# include "f_hpm.h"
+#endif
 
-CStartOfInterface
+CBOP
       SUBROUTINE EEDIE
-C     /==========================================================\
+C     *==========================================================*
 C     | SUBROUTINE EEDIE                                         |
 C     | o Close execution "environment", particularly perform    |
 C     |   steps to terminate parallel processing.                |
-C     |==========================================================|
+C     *==========================================================*
 C     | Note: This routine can also be compiled with CPP         |
 C     | directives set so that no multi-processing is initialised|
 C     | This is OK and should work fine.                         |
-C     \==========================================================/
+C     *==========================================================*
+      IMPLICIT NONE
 
 C     == Global variables ==
 #include "SIZE.h"
 #include "EEPARAMS.h"
 #include "EESUPPORT.h"
-CEndOfInterface
+CEOP
 
 C     == Local variables ==
-C     msgBuf       - I/O Buffer
-C     nThreadsDone - Used to count number of completed threads.
-C     I            - Loop counter.
+C     msgBuf       :: I/O Buffer
+C     nThreadsDone :: Used to count number of completed threads.
+C     I            :: Loop counter.
       CHARACTER*(MAX_LEN_MBUF) msgBuf
       INTEGER nThreadsDone
       INTEGER I
 #ifdef ALLOW_USE_MPI
-C     mpiRC            - Error code reporting variable used
-C                        with MPI.
+C     mpiRC        :: Error code reporting variable used with MPI.
       INTEGER mpiRC
 #endif /* ALLOW_USE_MPI */
 
+      IF ( eeBootError ) THEN
+C--   Skip ended threads counting if earlier error was found
+        WRITE(msgBuf,'(2A)')
+     &   'EEDIE: earlier error in multi-proc/thread setting'
+        CALL PRINT_ERROR( msgBuf, 1 )
+        fatalError = .TRUE.
+
+      ELSE
 C--   Check that all the threads have ended
 C     No thread should reach this loop before all threads have set
 C     threadIsComplete to TRUE. If they do then either there is a bug
-C     in the code or the behaviour of the parallel compiler directives 
-C     are not right for this code. In the latter case different directives
-C     may be available or the compiler itself may have a bug or you may
-C     need a different parallel compiler for main.F
-      nThreadsDone = 0
-      DO I = 1, nThreads
-       IF ( threadIsComplete(I) ) nThreadsDone = nThreadsDone+1
-      ENDDO
-      IF ( nThreadsDone .LT. nThreads ) THEN
-       WRITE(msgBuf,'(A,I,A,I,A)')
-     & 'S/R EEDIE: Only ',nThreadsDone,
-     & ' threads have completed, ',nThreads,' are expected for this configuration!'
-       WRITE(0,*) msgBuf
-       WRITE(0,*) 'Possibly you have different values of setenv PARALLEL and nThreads?'
-       eeEndError = .TRUE.
-       fatalError = .TRUE.
+C     in the code or the behaviour of the parallel compiler directives
+C     are not right for this code. In the latter case different
+C     directives may be available or the compiler itself may have a
+C     bug or you may need a different parallel compiler for main.F
+        nThreadsDone = 0
+        DO I = 1, nThreads
+         IF ( threadIsComplete(I) ) nThreadsDone = nThreadsDone+1
+        ENDDO
+        IF ( nThreadsDone .LT. nThreads ) THEN
+         WRITE(msgBuf,'(A,I5,A)')
+     &    'S/R EEDIE: Only',nThreadsDone,' threads have completed,'
+         CALL PRINT_ERROR( msgBuf, 1 )
+         WRITE(msgBuf,'(A,I5,A)')
+     &    'S/R EEDIE:',nThreads,' are expected for this config !'
+         CALL PRINT_ERROR( msgBuf, 1 )
+         eeEndError = .TRUE.
+         fatalError = .TRUE.
+        ENDIF
+
+C--   end if/else eebootError
       ENDIF
 
+#ifdef USE_LIBHPM
+      CALL F_HPMTERMINATE(myProcId)
+#endif
+
+C--   Flush IO-unit before MPI termination
+      CALL MDS_FLUSH( errorMessageUnit, 1 )
+c#ifdef ALLOW_USE_MPI
+      CALL MDS_FLUSH( standardMessageUnit, 1 )
+c#endif /* ALLOW_USE_MPI */
+
 #ifdef ALLOW_USE_MPI
-C--
+C- Note: since MPI_INIT is always called, better to also always terminate MPI
+C        (even if usingMPI=F) --> comment out test on usingMPI
+c     IF ( usingMPI ) THEN
+
 C--   MPI style multiple-process termination
 C--   ======================================
-#ifndef ALWAYS_USE_MPI
-      IF ( usingMPI ) THEN
+#ifdef COMPONENT_MODULE
+       IF ( useCoupler) CALL MPI_BARRIER( MPI_COMM_WORLD, mpiRC )
+#endif
+#ifdef ALLOW_OASIS
+       IF ( useOASIS ) CALL OASIS_FINALIZE
 #endif
        CALL MPI_FINALIZE  ( mpiRC )
        IF ( mpiRC .NE. MPI_SUCCESS ) THEN
         eeEndError = .TRUE.
         fatalError = .TRUE.
-        WRITE(msgBuf,'(A,I)')
+        WRITE(msgBuf,'(A,I5)')
      &       'S/R FIN_PROCS: MPI_FINALIZE return code',
      &       mpiRC
         CALL PRINT_ERROR( msgBuf, 1 )
        ENDIF
-C
-#ifndef ALWAYS_USE_MPI
-      ENDIF
-#endif
-#endif /* ALLOW_USE_MPI */
 
+c     ENDIF
+#endif /* ALLOW_USE_MPI */
 
       RETURN
       END