| 1 |
cnh |
1.3 |
C $Header: /u/gcmpack/models/MITgcmUV/eesupp/inc/EESUPPORT.h,v 1.2 1998/04/23 20:59:35 cnh Exp $ |
| 2 |
cnh |
1.1 |
C |
| 3 |
|
|
C /==========================================================\ |
| 4 |
|
|
C | EESUPPORT.h | |
| 5 |
|
|
C |==========================================================| |
| 6 |
|
|
C | Support data structures for the MITgcm UV "execution | |
| 7 |
|
|
C | environment" code. This data should be private to the | |
| 8 |
|
|
C | execution environment routines. Data which needs to be | |
| 9 |
|
|
C | accessed directly by the numerical model goes in | |
| 10 |
|
|
C | EEPARAMS.h. | |
| 11 |
|
|
C \==========================================================/ |
| 12 |
|
|
|
| 13 |
|
|
C ERROR_HEADER - String which prefixes error messages |
| 14 |
|
|
CHARACTER*(*) ERROR_HEADER |
| 15 |
|
|
PARAMETER ( ERROR_HEADER = ' *** ERROR ***' ) |
| 16 |
|
|
C PROCESS_HEADER - String which prefixes processor number |
| 17 |
|
|
CHARACTER*(*) PROCESS_HEADER |
| 18 |
|
|
PARAMETER ( PROCESS_HEADER = 'PID.TID' ) |
| 19 |
|
|
|
| 20 |
|
|
C COMM_NONE - No edge communication |
| 21 |
|
|
C COMM_MPI - Use MPI to communicate edges |
| 22 |
|
|
C COMM_SHMPG - Use shm get/put to communicate edges |
| 23 |
|
|
C COMM_SHARED - Use true shared memory to communicate edges |
| 24 |
|
|
INTEGER COMM_NONE |
| 25 |
|
|
PARAMETER ( COMM_NONE = 0 ) |
| 26 |
|
|
INTEGER COMM_MPI |
| 27 |
|
|
PARAMETER ( COMM_MPI = 1 ) |
| 28 |
|
|
INTEGER COMM_SHMPG |
| 29 |
|
|
PARAMETER ( COMM_SHMPG = 2 ) |
| 30 |
|
|
INTEGER COMM_SHARED |
| 31 |
|
|
PARAMETER ( COMM_SHARED = 3 ) |
| 32 |
|
|
|
| 33 |
|
|
C-- COMMON /EESUPP_C/ Execution environment support character variables |
| 34 |
|
|
C myProcessStr - String identifying my process number |
| 35 |
|
|
COMMON /EESUPP_C/ myProcessStr |
| 36 |
|
|
CHARACTER*128 myProcessStr |
| 37 |
|
|
|
| 38 |
|
|
C-- COMMON /EESUPP_L/ Execution environment support logical variables |
| 39 |
|
|
C initMPError - Flag indicating error during multi-processing |
| 40 |
|
|
C initialisation. |
| 41 |
|
|
C finMPError - Flag indicating error during multi-processing |
| 42 |
|
|
C termination. |
| 43 |
|
|
C ThError - Thread detected an error. |
| 44 |
|
|
C usingMPI - Flag controlling use of MPI routines. This flag |
| 45 |
|
|
C allows either MPI or threads to be used in a |
| 46 |
|
|
C shared memory environment which can be a useful |
| 47 |
|
|
C debugging/performance analysis tool. |
| 48 |
|
|
C usingSyncMessages - Flag that causes blocking communication to be used |
| 49 |
|
|
C if possible. When false non-blocking EXCH routines |
| 50 |
|
|
C will be used if possible. |
| 51 |
|
|
C notUsingXPeriodicity - Flag indicating no X/Y boundary wrap around |
| 52 |
|
|
C notUsingYPeriodicity This affects the communication routines but |
| 53 |
|
|
C is generally ignored in the numerical model |
| 54 |
|
|
C code. |
| 55 |
|
|
C threadIsRunning, threadIsComplete - Flags used to check for correct behaviour |
| 56 |
|
|
C of multi-threaded code. |
| 57 |
|
|
C threadIsRunning is used to check that the |
| 58 |
|
|
C threads we need are running. This catches the |
| 59 |
|
|
C situation where a program eedata file has nTthreads |
| 60 |
|
|
C greater than the setenv PARALLEL or NCPUS variable. |
| 61 |
|
|
C threadIsComplete is used to flag that a thread has |
| 62 |
|
|
C reached the end of the model. This is used as a check to |
| 63 |
|
|
C trap problems that might occur if one thread "escapes" |
| 64 |
|
|
C the main.F master loop. This should not happen |
| 65 |
|
|
C if the multi-threading compilation tools works right. |
| 66 |
|
|
C But (see for example KAP) this is not always the case! |
| 67 |
cnh |
1.3 |
C printMapIncludesZeros - Flag that controls whether character constant map code ignores |
| 68 |
|
|
C exact zero values. |
| 69 |
cnh |
1.1 |
COMMON /EESUPP_L/ thError, threadIsRunning, threadIsComplete, |
| 70 |
|
|
& allMyEdgesAreSharedMemory, usingMPI, usingSyncMessages, |
| 71 |
cnh |
1.3 |
& notUsingXPeriodicity, notUsingYPeriodicity, |
| 72 |
|
|
& printMapIncludesZeros |
| 73 |
cnh |
1.1 |
LOGICAL thError(MAX_NO_THREADS) |
| 74 |
|
|
LOGICAL threadIsRunning(MAX_NO_THREADS) |
| 75 |
|
|
LOGICAL threadIsComplete(MAX_NO_THREADS) |
| 76 |
|
|
LOGICAL allMyEdgesAreSharedMemory(MAX_NO_THREADS) |
| 77 |
|
|
LOGICAL usingMPI |
| 78 |
|
|
LOGICAL usingSyncMessages |
| 79 |
|
|
LOGICAL notUsingXPeriodicity |
| 80 |
|
|
LOGICAL notUsingYPeriodicity |
| 81 |
cnh |
1.3 |
LOGICAL printMapIncludesZeros |
| 82 |
cnh |
1.1 |
|
| 83 |
|
|
|
| 84 |
|
|
C-- COMMON /EESUPP_I/ Parallel support integer globals |
| 85 |
|
|
C pidW - "Process" ID of neighbor to West |
| 86 |
|
|
C pidE - " " " East |
| 87 |
|
|
C pidN - " " " North |
| 88 |
|
|
C pidS - " " " South |
| 89 |
|
|
C pidNE - " " " North-East |
| 90 |
|
|
C pidNW - " " " North-West |
| 91 |
|
|
C pidSE - " " " South-East |
| 92 |
|
|
C pidSW - " " " South-West |
| 93 |
|
|
C Note: pid[XY] is not necessairily the UNIX |
| 94 |
|
|
C process id - it is just an identifying |
| 95 |
|
|
C number. |
| 96 |
|
|
C commW - Communication method at thread edge to the |
| 97 |
|
|
C commE west (W), east (E), south (S), north (N). |
| 98 |
|
|
C commS |
| 99 |
|
|
C commN |
| 100 |
|
|
C myThrS - Thread number of neighboring thread, used |
| 101 |
|
|
C myThrN to match senders with receivers of |
| 102 |
|
|
C myThrW messages. |
| 103 |
|
|
C myThrE |
| 104 |
|
|
C myThrSW |
| 105 |
|
|
C myThrSE |
| 106 |
|
|
C myThrNW |
| 107 |
|
|
C myThrNE |
| 108 |
|
|
C nTx, nTy - No. threads in X and Y. This assumes a simple cartesian |
| 109 |
|
|
C gridding of the threads which is not required elsewhere |
| 110 |
|
|
C but that makes it easier. |
| 111 |
|
|
COMMON /EESUPP_I/ |
| 112 |
|
|
& pidW, pidE, pidS, pidN, pidSE, pidSW, pidNE, pidNW, |
| 113 |
|
|
& commW, commN, commS, commE, |
| 114 |
|
|
& myThrS, myThrN, myThrW, myThrE, |
| 115 |
|
|
& myThrNE, myThrNW, myThrSE, myThrSW |
| 116 |
|
|
INTEGER commW(MAX_NO_THREADS) |
| 117 |
|
|
INTEGER commE(MAX_NO_THREADS) |
| 118 |
|
|
INTEGER commN(MAX_NO_THREADS) |
| 119 |
|
|
INTEGER commS(MAX_NO_THREADS) |
| 120 |
|
|
INTEGER pidW |
| 121 |
|
|
INTEGER pidE |
| 122 |
|
|
INTEGER pidS |
| 123 |
|
|
INTEGER pidN |
| 124 |
|
|
INTEGER pidSE |
| 125 |
|
|
INTEGER pidSW |
| 126 |
|
|
INTEGER pidNE |
| 127 |
|
|
INTEGER pidNW |
| 128 |
|
|
INTEGER myThrS(MAX_NO_THREADS) |
| 129 |
|
|
INTEGER myThrN(MAX_NO_THREADS) |
| 130 |
|
|
INTEGER myThrW(MAX_NO_THREADS) |
| 131 |
|
|
INTEGER myThrE(MAX_NO_THREADS) |
| 132 |
|
|
INTEGER myThrSW(MAX_NO_THREADS) |
| 133 |
|
|
INTEGER myThrNW(MAX_NO_THREADS) |
| 134 |
|
|
INTEGER myThrNE(MAX_NO_THREADS) |
| 135 |
|
|
INTEGER myThrSE(MAX_NO_THREADS) |
| 136 |
|
|
|
| 137 |
|
|
#ifdef ALLOW_USE_MPI |
| 138 |
|
|
C-- Include MPI standard Fortran header file |
| 139 |
|
|
#include "mpif.h" |
| 140 |
|
|
#define _mpiTRUE_ 1 |
| 141 |
|
|
#define _mpiFALSE_ 0 |
| 142 |
|
|
|
| 143 |
|
|
C-- COMMON /EESUPP_MPI_I/ MPI parallel support integer globals |
| 144 |
|
|
C mpiPidW - MPI process id for west neighbor. |
| 145 |
|
|
C mpiPidE - MPI process id for east neighbor. |
| 146 |
|
|
C mpiPidN - MPI process id for north neighbor. |
| 147 |
|
|
C mpiPidS - MPI process id for south neighbor. |
| 148 |
|
|
C mpiPidNW - MPI process id for northwest neighbor. |
| 149 |
|
|
C mpiPidNE - MPI process id for northeast neighbor. |
| 150 |
|
|
C mpiPidSW - MPI process id for southwest neighbor. |
| 151 |
|
|
C mpiPidSE - MPI process id for southeast neighbor. |
| 152 |
|
|
C mpiPidIO - MPI process to use for IO. |
| 153 |
|
|
C mpiNprocs - No. of MPI processes. |
| 154 |
|
|
C mpiMyId - MPI process id of me. |
| 155 |
|
|
C mpiComm - MPI communicator to use. |
| 156 |
|
|
C mpiPx - My MPI proc. grid X coord |
| 157 |
|
|
C mpiPy - My MPI proc. grid Y coord |
| 158 |
|
|
C mpiXGlobalLo - My bottom-left (south-west) x-coordinate in |
| 159 |
|
|
C global domain. |
| 160 |
|
|
C mpiYGlobalLo - My bottom-left (south-west) y-coordinate in |
| 161 |
|
|
C global domain. |
| 162 |
|
|
C mpiTypeXFaceBlock_xy_r4 - Primitives for communicating edge |
| 163 |
|
|
C mpiTypeXFaceBlock_xy_r8 of a block. |
| 164 |
|
|
C mpiTypeYFaceBlock_xy_r4 XFace is used in east-west transfer |
| 165 |
|
|
C mpiTypeYFaceBlock_xy_r8 YFace is used in nrth-south transfer |
| 166 |
|
|
C mpiTypeXFaceBlock_xyz_r4 xy is used in two-dimensional arrays |
| 167 |
|
|
C mpiTypeXFaceBlock_xyz_r8 xyz is used with three-dimensional arrays |
| 168 |
|
|
C mpiTypeYFaceBlock_xyz_r4 r4 is used for real*4 data |
| 169 |
|
|
C mpiTypeYFaceBlock_xyz_r8 r8 is used for real*8 data |
| 170 |
|
|
C mpiTypeXFaceThread_xy_r4 - Composites of the above primitives |
| 171 |
|
|
C mpiTypeXFaceThread_xy_r8 for communicating edges of all blocks |
| 172 |
|
|
C mpiTypeYFaceThread_xy_r4 owned by a thread. |
| 173 |
|
|
C mpiTypeYFaceThread_xy_r8 |
| 174 |
|
|
C mpiTypeXFaceThread_xyz_r4 |
| 175 |
|
|
C mpiTypeXFaceThread_xyz_r8 |
| 176 |
|
|
C mpiTypeYFaceThread_xyz_r4 |
| 177 |
|
|
C mpiTypeYFaceBlock_xyz_r8 |
| 178 |
|
|
C mpiTagE - Tags are needed to mark requests when MPI is running |
| 179 |
|
|
C mpiTagW between multithreaded processes or when the same process. |
| 180 |
|
|
C mpiTagS is a neighbor in more than one direction. The tags ensure that |
| 181 |
|
|
C mpiTagN a thread will get the message it is looking for. |
| 182 |
|
|
C mpiTagSW The scheme adopted is to tag messages according to |
| 183 |
|
|
C mpiTagSE the direction they are travelling. Thus a message |
| 184 |
|
|
C mpiTagNW travelling east is tagged mpiTagE. However, in a |
| 185 |
|
|
C mpiTagNE multi-threaded environemnt several messages could |
| 186 |
|
|
C be travelling east from the same process at the |
| 187 |
|
|
C same time. The tag is therefore modified to |
| 188 |
|
|
C be mpiTag[EWS...]*nThreads+myThid. This requires that |
| 189 |
|
|
C each thread also know the thread ids of its "neighbor" |
| 190 |
|
|
C threads. |
| 191 |
|
|
COMMON /EESUPP_MPI_I/ |
| 192 |
|
|
& mpiPidW, mpiPidE, mpiPidS, mpiPidN, |
| 193 |
|
|
& mpiPidSE, mpiPidSW, mpiPidNE, mpiPidNW, |
| 194 |
|
|
& mpiPidIo, mpiMyId, mpiNProcs, mpiComm, |
| 195 |
|
|
& mpiPx, mpiPy, mpiXGlobalLo, mpiYGlobalLo, |
| 196 |
|
|
& mpiTypeXFaceBlock_xy_r4, mpiTypeXFaceBlock_xy_r8, |
| 197 |
|
|
& mpiTypeYFaceBlock_xy_r4, mpiTypeYFaceBlock_xy_r8, |
| 198 |
|
|
& mpiTypeXFaceBlock_xyz_r4, mpiTypeXFaceBlock_xyz_r8, |
| 199 |
|
|
& mpiTypeYFaceBlock_xyz_r4, mpiTypeYFaceBlock_xyz_r8, |
| 200 |
|
|
& mpiTypeXFaceThread_xy_r4, mpiTypeXFaceThread_xy_r8, |
| 201 |
|
|
& mpiTypeYFaceThread_xy_r4, mpiTypeYFaceThread_xy_r8, |
| 202 |
|
|
& mpiTypeXFaceThread_xyz_r4, mpiTypeXFaceThread_xyz_r8, |
| 203 |
|
|
& mpiTypeYFaceThread_xyz_r4, mpiTypeYFaceThread_xyz_r8, |
| 204 |
|
|
& mpiTagE, mpiTagW, mpiTagN, mpiTagS, |
| 205 |
|
|
& mpiTagSE, mpiTagSW, mpiTagNW, mpiTagNE |
| 206 |
|
|
|
| 207 |
|
|
INTEGER mpiPidW |
| 208 |
|
|
INTEGER mpiPidE |
| 209 |
|
|
INTEGER mpiPidS |
| 210 |
|
|
INTEGER mpiPidN |
| 211 |
|
|
INTEGER mpiPidSW |
| 212 |
|
|
INTEGER mpiPidSE |
| 213 |
|
|
INTEGER mpiPidNW |
| 214 |
|
|
INTEGER mpiPidNE |
| 215 |
|
|
INTEGER mpiPidIO |
| 216 |
|
|
INTEGER mpiMyId |
| 217 |
|
|
INTEGER mpiNProcs |
| 218 |
|
|
INTEGER mpiComm |
| 219 |
|
|
INTEGER mpiPx |
| 220 |
|
|
INTEGER mpiPy |
| 221 |
|
|
INTEGER mpiXGlobalLo |
| 222 |
|
|
INTEGER mpiYGlobalLo |
| 223 |
|
|
INTEGER mpiTypeXFaceBlock_xy_r4 |
| 224 |
|
|
INTEGER mpiTypeXFaceBlock_xy_r8 |
| 225 |
|
|
INTEGER mpiTypeYFaceBlock_xy_r4 |
| 226 |
|
|
INTEGER mpiTypeYFaceBlock_xy_r8 |
| 227 |
|
|
INTEGER mpiTypeXFaceBlock_xyz_r4 |
| 228 |
|
|
INTEGER mpiTypeXFaceBlock_xyz_r8 |
| 229 |
|
|
INTEGER mpiTypeYFaceBlock_xyz_r4 |
| 230 |
|
|
INTEGER mpiTypeYFaceBlock_xyz_r8 |
| 231 |
|
|
INTEGER mpiTypeXFaceThread_xy_r4(MAX_NO_THREADS) |
| 232 |
|
|
INTEGER mpiTypeXFaceThread_xy_r8(MAX_NO_THREADS) |
| 233 |
|
|
INTEGER mpiTypeYFaceThread_xy_r4(MAX_NO_THREADS) |
| 234 |
|
|
INTEGER mpiTypeYFaceThread_xy_r8(MAX_NO_THREADS) |
| 235 |
|
|
INTEGER mpiTypeXFaceThread_xyz_r4(MAX_NO_THREADS) |
| 236 |
|
|
INTEGER mpiTypeXFaceThread_xyz_r8(MAX_NO_THREADS) |
| 237 |
|
|
INTEGER mpiTypeYFaceThread_xyz_r4(MAX_NO_THREADS) |
| 238 |
|
|
INTEGER mpiTypeYFaceThread_xyz_r8(MAX_NO_THREADS) |
| 239 |
|
|
INTEGER mpiTagNW |
| 240 |
|
|
INTEGER mpiTagNE |
| 241 |
|
|
INTEGER mpiTagSW |
| 242 |
|
|
INTEGER mpiTagSE |
| 243 |
|
|
INTEGER mpiTagW |
| 244 |
|
|
INTEGER mpiTagE |
| 245 |
|
|
INTEGER mpiTagN |
| 246 |
|
|
INTEGER mpiTagS |
| 247 |
|
|
#endif /* ALLOW_USE_MPI */ |
| 248 |
|
|
|
Parent Directory
| 
Revision Log
| 
Revision Graph