| 1 |
C $Id$ |
| 2 |
C |
| 3 |
C /==========================================================\ |
| 4 |
C | EESUPPORT.h | |
| 5 |
C |==========================================================| |
| 6 |
C | Support data structures for the MITgcm UV "execution | |
| 7 |
C | environment" code. This data should be private to the | |
| 8 |
C | execution environment routines. Data which needs to be | |
| 9 |
C | accessed directly by the numerical model goes in | |
| 10 |
C | EEPARAMS.h. | |
| 11 |
C \==========================================================/ |
| 12 |
|
| 13 |
C ERROR_HEADER - String which prefixes error messages |
| 14 |
CHARACTER*(*) ERROR_HEADER |
| 15 |
PARAMETER ( ERROR_HEADER = ' *** ERROR ***' ) |
| 16 |
C PROCESS_HEADER - String which prefixes processor number |
| 17 |
CHARACTER*(*) PROCESS_HEADER |
| 18 |
PARAMETER ( PROCESS_HEADER = 'PID.TID' ) |
| 19 |
|
| 20 |
C COMM_NONE - No edge communication |
| 21 |
C COMM_MPI - Use MPI to communicate edges |
| 22 |
C COMM_SHMPG - Use shm get/put to communicate edges |
| 23 |
C COMM_SHARED - Use true shared memory to communicate edges |
| 24 |
INTEGER COMM_NONE |
| 25 |
PARAMETER ( COMM_NONE = 0 ) |
| 26 |
INTEGER COMM_MPI |
| 27 |
PARAMETER ( COMM_MPI = 1 ) |
| 28 |
INTEGER COMM_SHMPG |
| 29 |
PARAMETER ( COMM_SHMPG = 2 ) |
| 30 |
INTEGER COMM_SHARED |
| 31 |
PARAMETER ( COMM_SHARED = 3 ) |
| 32 |
|
| 33 |
C-- COMMON /EESUPP_C/ Execution environment support character variables |
| 34 |
C myProcessStr - String identifying my process number |
| 35 |
COMMON /EESUPP_C/ myProcessStr |
| 36 |
CHARACTER*128 myProcessStr |
| 37 |
|
| 38 |
C-- COMMON /EESUPP_L/ Execution environment support logical variables |
| 39 |
C initMPError - Flag indicating error during multi-processing |
| 40 |
C initialisation. |
| 41 |
C finMPError - Flag indicating error during multi-processing |
| 42 |
C termination. |
| 43 |
C ThError - Thread detected an error. |
| 44 |
C usingMPI - Flag controlling use of MPI routines. This flag |
| 45 |
C allows either MPI or threads to be used in a |
| 46 |
C shared memory environment which can be a useful |
| 47 |
C debugging/performance analysis tool. |
| 48 |
C usingSyncMessages - Flag that causes blocking communication to be used |
| 49 |
C if possible. When false non-blocking EXCH routines |
| 50 |
C will be used if possible. |
| 51 |
C notUsingXPeriodicity - Flag indicating no X/Y boundary wrap around |
| 52 |
C notUsingYPeriodicity This affects the communication routines but |
| 53 |
C is generally ignored in the numerical model |
| 54 |
C code. |
| 55 |
C threadIsRunning, threadIsComplete - Flags used to check for correct behaviour |
| 56 |
C of multi-threaded code. |
| 57 |
C threadIsRunning is used to check that the |
| 58 |
C threads we need are running. This catches the |
| 59 |
C situation where a program eedata file has nTthreads |
| 60 |
C greater than the setenv PARALLEL or NCPUS variable. |
| 61 |
C threadIsComplete is used to flag that a thread has |
| 62 |
C reached the end of the model. This is used as a check to |
| 63 |
C trap problems that might occur if one thread "escapes" |
| 64 |
C the main.F master loop. This should not happen |
| 65 |
C if the multi-threading compilation tools works right. |
| 66 |
C But (see for example KAP) this is not always the case! |
| 67 |
COMMON /EESUPP_L/ thError, threadIsRunning, threadIsComplete, |
| 68 |
& allMyEdgesAreSharedMemory, usingMPI, usingSyncMessages, |
| 69 |
& notUsingXPeriodicity, notUsingYPeriodicity |
| 70 |
LOGICAL thError(MAX_NO_THREADS) |
| 71 |
LOGICAL threadIsRunning(MAX_NO_THREADS) |
| 72 |
LOGICAL threadIsComplete(MAX_NO_THREADS) |
| 73 |
LOGICAL allMyEdgesAreSharedMemory(MAX_NO_THREADS) |
| 74 |
LOGICAL usingMPI |
| 75 |
LOGICAL usingSyncMessages |
| 76 |
LOGICAL notUsingXPeriodicity |
| 77 |
LOGICAL notUsingYPeriodicity |
| 78 |
|
| 79 |
|
| 80 |
C-- COMMON /EESUPP_I/ Parallel support integer globals |
| 81 |
C pidW - "Process" ID of neighbor to West |
| 82 |
C pidE - " " " East |
| 83 |
C pidN - " " " North |
| 84 |
C pidS - " " " South |
| 85 |
C pidNE - " " " North-East |
| 86 |
C pidNW - " " " North-West |
| 87 |
C pidSE - " " " South-East |
| 88 |
C pidSW - " " " South-West |
| 89 |
C Note: pid[XY] is not necessairily the UNIX |
| 90 |
C process id - it is just an identifying |
| 91 |
C number. |
| 92 |
C commW - Communication method at thread edge to the |
| 93 |
C commE west (W), east (E), south (S), north (N). |
| 94 |
C commS |
| 95 |
C commN |
| 96 |
C myThrS - Thread number of neighboring thread, used |
| 97 |
C myThrN to match senders with receivers of |
| 98 |
C myThrW messages. |
| 99 |
C myThrE |
| 100 |
C myThrSW |
| 101 |
C myThrSE |
| 102 |
C myThrNW |
| 103 |
C myThrNE |
| 104 |
C nTx, nTy - No. threads in X and Y. This assumes a simple cartesian |
| 105 |
C gridding of the threads which is not required elsewhere |
| 106 |
C but that makes it easier. |
| 107 |
COMMON /EESUPP_I/ |
| 108 |
& pidW, pidE, pidS, pidN, pidSE, pidSW, pidNE, pidNW, |
| 109 |
& commW, commN, commS, commE, |
| 110 |
& myThrS, myThrN, myThrW, myThrE, |
| 111 |
& myThrNE, myThrNW, myThrSE, myThrSW |
| 112 |
INTEGER commW(MAX_NO_THREADS) |
| 113 |
INTEGER commE(MAX_NO_THREADS) |
| 114 |
INTEGER commN(MAX_NO_THREADS) |
| 115 |
INTEGER commS(MAX_NO_THREADS) |
| 116 |
INTEGER pidW |
| 117 |
INTEGER pidE |
| 118 |
INTEGER pidS |
| 119 |
INTEGER pidN |
| 120 |
INTEGER pidSE |
| 121 |
INTEGER pidSW |
| 122 |
INTEGER pidNE |
| 123 |
INTEGER pidNW |
| 124 |
INTEGER myThrS(MAX_NO_THREADS) |
| 125 |
INTEGER myThrN(MAX_NO_THREADS) |
| 126 |
INTEGER myThrW(MAX_NO_THREADS) |
| 127 |
INTEGER myThrE(MAX_NO_THREADS) |
| 128 |
INTEGER myThrSW(MAX_NO_THREADS) |
| 129 |
INTEGER myThrNW(MAX_NO_THREADS) |
| 130 |
INTEGER myThrNE(MAX_NO_THREADS) |
| 131 |
INTEGER myThrSE(MAX_NO_THREADS) |
| 132 |
|
| 133 |
#ifdef ALLOW_USE_MPI |
| 134 |
C-- Include MPI standard Fortran header file |
| 135 |
#include "mpif.h" |
| 136 |
#define _mpiTRUE_ 1 |
| 137 |
#define _mpiFALSE_ 0 |
| 138 |
|
| 139 |
C-- COMMON /EESUPP_MPI_I/ MPI parallel support integer globals |
| 140 |
C mpiPidW - MPI process id for west neighbor. |
| 141 |
C mpiPidE - MPI process id for east neighbor. |
| 142 |
C mpiPidN - MPI process id for north neighbor. |
| 143 |
C mpiPidS - MPI process id for south neighbor. |
| 144 |
C mpiPidNW - MPI process id for northwest neighbor. |
| 145 |
C mpiPidNE - MPI process id for northeast neighbor. |
| 146 |
C mpiPidSW - MPI process id for southwest neighbor. |
| 147 |
C mpiPidSE - MPI process id for southeast neighbor. |
| 148 |
C mpiPidIO - MPI process to use for IO. |
| 149 |
C mpiNprocs - No. of MPI processes. |
| 150 |
C mpiMyId - MPI process id of me. |
| 151 |
C mpiComm - MPI communicator to use. |
| 152 |
C mpiPx - My MPI proc. grid X coord |
| 153 |
C mpiPy - My MPI proc. grid Y coord |
| 154 |
C mpiXGlobalLo - My bottom-left (south-west) x-coordinate in |
| 155 |
C global domain. |
| 156 |
C mpiYGlobalLo - My bottom-left (south-west) y-coordinate in |
| 157 |
C global domain. |
| 158 |
C mpiTypeXFaceBlock_xy_r4 - Primitives for communicating edge |
| 159 |
C mpiTypeXFaceBlock_xy_r8 of a block. |
| 160 |
C mpiTypeYFaceBlock_xy_r4 XFace is used in east-west transfer |
| 161 |
C mpiTypeYFaceBlock_xy_r8 YFace is used in nrth-south transfer |
| 162 |
C mpiTypeXFaceBlock_xyz_r4 xy is used in two-dimensional arrays |
| 163 |
C mpiTypeXFaceBlock_xyz_r8 xyz is used with three-dimensional arrays |
| 164 |
C mpiTypeYFaceBlock_xyz_r4 r4 is used for real*4 data |
| 165 |
C mpiTypeYFaceBlock_xyz_r8 r8 is used for real*8 data |
| 166 |
C mpiTypeXFaceThread_xy_r4 - Composites of the above primitives |
| 167 |
C mpiTypeXFaceThread_xy_r8 for communicating edges of all blocks |
| 168 |
C mpiTypeYFaceThread_xy_r4 owned by a thread. |
| 169 |
C mpiTypeYFaceThread_xy_r8 |
| 170 |
C mpiTypeXFaceThread_xyz_r4 |
| 171 |
C mpiTypeXFaceThread_xyz_r8 |
| 172 |
C mpiTypeYFaceThread_xyz_r4 |
| 173 |
C mpiTypeYFaceBlock_xyz_r8 |
| 174 |
C mpiTagE - Tags are needed to mark requests when MPI is running |
| 175 |
C mpiTagW between multithreaded processes or when the same process. |
| 176 |
C mpiTagS is a neighbor in more than one direction. The tags ensure that |
| 177 |
C mpiTagN a thread will get the message it is looking for. |
| 178 |
C mpiTagSW The scheme adopted is to tag messages according to |
| 179 |
C mpiTagSE the direction they are travelling. Thus a message |
| 180 |
C mpiTagNW travelling east is tagged mpiTagE. However, in a |
| 181 |
C mpiTagNE multi-threaded environemnt several messages could |
| 182 |
C be travelling east from the same process at the |
| 183 |
C same time. The tag is therefore modified to |
| 184 |
C be mpiTag[EWS...]*nThreads+myThid. This requires that |
| 185 |
C each thread also know the thread ids of its "neighbor" |
| 186 |
C threads. |
| 187 |
COMMON /EESUPP_MPI_I/ |
| 188 |
& mpiPidW, mpiPidE, mpiPidS, mpiPidN, |
| 189 |
& mpiPidSE, mpiPidSW, mpiPidNE, mpiPidNW, |
| 190 |
& mpiPidIo, mpiMyId, mpiNProcs, mpiComm, |
| 191 |
& mpiPx, mpiPy, mpiXGlobalLo, mpiYGlobalLo, |
| 192 |
& mpiTypeXFaceBlock_xy_r4, mpiTypeXFaceBlock_xy_r8, |
| 193 |
& mpiTypeYFaceBlock_xy_r4, mpiTypeYFaceBlock_xy_r8, |
| 194 |
& mpiTypeXFaceBlock_xyz_r4, mpiTypeXFaceBlock_xyz_r8, |
| 195 |
& mpiTypeYFaceBlock_xyz_r4, mpiTypeYFaceBlock_xyz_r8, |
| 196 |
& mpiTypeXFaceThread_xy_r4, mpiTypeXFaceThread_xy_r8, |
| 197 |
& mpiTypeYFaceThread_xy_r4, mpiTypeYFaceThread_xy_r8, |
| 198 |
& mpiTypeXFaceThread_xyz_r4, mpiTypeXFaceThread_xyz_r8, |
| 199 |
& mpiTypeYFaceThread_xyz_r4, mpiTypeYFaceThread_xyz_r8, |
| 200 |
& mpiTagE, mpiTagW, mpiTagN, mpiTagS, |
| 201 |
& mpiTagSE, mpiTagSW, mpiTagNW, mpiTagNE |
| 202 |
|
| 203 |
INTEGER mpiPidW |
| 204 |
INTEGER mpiPidE |
| 205 |
INTEGER mpiPidS |
| 206 |
INTEGER mpiPidN |
| 207 |
INTEGER mpiPidSW |
| 208 |
INTEGER mpiPidSE |
| 209 |
INTEGER mpiPidNW |
| 210 |
INTEGER mpiPidNE |
| 211 |
INTEGER mpiPidIO |
| 212 |
INTEGER mpiMyId |
| 213 |
INTEGER mpiNProcs |
| 214 |
INTEGER mpiComm |
| 215 |
INTEGER mpiPx |
| 216 |
INTEGER mpiPy |
| 217 |
INTEGER mpiXGlobalLo |
| 218 |
INTEGER mpiYGlobalLo |
| 219 |
INTEGER mpiTypeXFaceBlock_xy_r4 |
| 220 |
INTEGER mpiTypeXFaceBlock_xy_r8 |
| 221 |
INTEGER mpiTypeYFaceBlock_xy_r4 |
| 222 |
INTEGER mpiTypeYFaceBlock_xy_r8 |
| 223 |
INTEGER mpiTypeXFaceBlock_xyz_r4 |
| 224 |
INTEGER mpiTypeXFaceBlock_xyz_r8 |
| 225 |
INTEGER mpiTypeYFaceBlock_xyz_r4 |
| 226 |
INTEGER mpiTypeYFaceBlock_xyz_r8 |
| 227 |
INTEGER mpiTypeXFaceThread_xy_r4(MAX_NO_THREADS) |
| 228 |
INTEGER mpiTypeXFaceThread_xy_r8(MAX_NO_THREADS) |
| 229 |
INTEGER mpiTypeYFaceThread_xy_r4(MAX_NO_THREADS) |
| 230 |
INTEGER mpiTypeYFaceThread_xy_r8(MAX_NO_THREADS) |
| 231 |
INTEGER mpiTypeXFaceThread_xyz_r4(MAX_NO_THREADS) |
| 232 |
INTEGER mpiTypeXFaceThread_xyz_r8(MAX_NO_THREADS) |
| 233 |
INTEGER mpiTypeYFaceThread_xyz_r4(MAX_NO_THREADS) |
| 234 |
INTEGER mpiTypeYFaceThread_xyz_r8(MAX_NO_THREADS) |
| 235 |
INTEGER mpiTagNW |
| 236 |
INTEGER mpiTagNE |
| 237 |
INTEGER mpiTagSW |
| 238 |
INTEGER mpiTagSE |
| 239 |
INTEGER mpiTagW |
| 240 |
INTEGER mpiTagE |
| 241 |
INTEGER mpiTagN |
| 242 |
INTEGER mpiTagS |
| 243 |
#endif /* ALLOW_USE_MPI */ |
| 244 |
|
Parent Directory
| 
Revision Log
| 
Revision Graph