| 1 |
C $Header: /u/gcmpack/MITgcm/eesupp/inc/EESUPPORT.h,v 1.9 2005/11/05 00:49:47 jmc Exp $ |
| 2 |
C $Name: $ |
| 3 |
CBOP |
| 4 |
C !ROUTINE: EESUPPORT.h |
| 5 |
C !INTERFACE: |
| 6 |
C include "EESUPPORT.h" |
| 7 |
C |
| 8 |
C !DESCRIPTION: |
| 9 |
C *==========================================================* |
| 10 |
C | EESUPPORT.h | |
| 11 |
C *==========================================================* |
| 12 |
C | Support data structures for the MITgcm UV ``execution | |
| 13 |
C | environment'' code. This data should be private to the | |
| 14 |
C | execution environment routines. Data which needs to be | |
| 15 |
C | accessed directly by a numerical model goes in | |
| 16 |
C | EEPARAMS.h. | |
| 17 |
C *==========================================================* |
| 18 |
CEOP |
| 19 |
|
| 20 |
C ERROR_HEADER - String which prefixes error messages |
| 21 |
CHARACTER*(*) ERROR_HEADER |
| 22 |
PARAMETER ( ERROR_HEADER = ' *** ERROR ***' ) |
| 23 |
C PROCESS_HEADER - String which prefixes processor number |
| 24 |
CHARACTER*(*) PROCESS_HEADER |
| 25 |
PARAMETER ( PROCESS_HEADER = 'PID.TID' ) |
| 26 |
|
| 27 |
C MAX_NUM_COMM_MODES - Maximum number of communication modes |
| 28 |
C COMM_NONE - No edge communication |
| 29 |
C COMM_MSG - Use messages to communicate edges |
| 30 |
C COMM_PUT - Use put to communicate edges |
| 31 |
C COMM_GET - Use get to communicate edges |
| 32 |
C Note - commName holds an identifying name for each communication |
| 33 |
C mode. The COMM_ parameters are used to index commName |
| 34 |
C so the COMM_ parameters need to be in the range |
| 35 |
C 1 : MAX_NUM_COMM_MODES. |
| 36 |
INTEGER MAX_NUM_COMM_MODES |
| 37 |
PARAMETER ( MAX_NUM_COMM_MODES = 4 ) |
| 38 |
INTEGER COMM_NONE |
| 39 |
PARAMETER ( COMM_NONE = 1 ) |
| 40 |
INTEGER COMM_MSG |
| 41 |
PARAMETER ( COMM_MSG = 2 ) |
| 42 |
INTEGER COMM_PUT |
| 43 |
PARAMETER ( COMM_PUT = 3 ) |
| 44 |
INTEGER COMM_GET |
| 45 |
PARAMETER ( COMM_GET = 4 ) |
| 46 |
COMMON /EESUPP_COMMNAME/ commName |
| 47 |
CHARACTER*10 commName(MAX_NUM_COMM_MODES) |
| 48 |
|
| 49 |
C Tile identifiers |
| 50 |
C Tiles have a number that is unique over the global domain. |
| 51 |
C A tile that is not there has its number set to NULL_TILE |
| 52 |
INTEGER NULL_TILE |
| 53 |
PARAMETER ( NULL_TILE = -1 ) |
| 54 |
|
| 55 |
|
| 56 |
C-- COMMON /EESUPP_C/ Execution environment support character variables |
| 57 |
C myProcessStr - String identifying my process number |
| 58 |
COMMON /EESUPP_C/ myProcessStr |
| 59 |
CHARACTER*128 myProcessStr |
| 60 |
|
| 61 |
C-- COMMON /EESUPP_L/ Execution environment support logical variables |
| 62 |
C initMPError - Flag indicating error during multi-processing |
| 63 |
C initialisation. |
| 64 |
C finMPError - Flag indicating error during multi-processing |
| 65 |
C termination. |
| 66 |
C ThError - Thread detected an error. |
| 67 |
C usingMPI - Flag controlling use of MPI routines. This flag |
| 68 |
C allows either MPI or threads to be used in a |
| 69 |
C shared memory environment which can be a useful |
| 70 |
C debugging/performance analysis tool. |
| 71 |
C usingSyncMessages - Flag that causes blocking communication to be used |
| 72 |
C if possible. When false non-blocking EXCH routines |
| 73 |
C will be used if possible. |
| 74 |
C notUsingXPeriodicity - Flag indicating no X/Y boundary wrap around |
| 75 |
C notUsingYPeriodicity This affects the communication routines but |
| 76 |
C is generally ignored in the numerical model |
| 77 |
C code. |
| 78 |
C threadIsRunning, threadIsComplete - Flags used to check for correct behaviour |
| 79 |
C of multi-threaded code. |
| 80 |
C threadIsRunning is used to check that the |
| 81 |
C threads we need are running. This catches the |
| 82 |
C situation where a program eedata file has nTthreads |
| 83 |
C greater than the setenv PARALLEL or NCPUS variable. |
| 84 |
C threadIsComplete is used to flag that a thread has |
| 85 |
C reached the end of the model. This is used as a check to |
| 86 |
C trap problems that might occur if one thread "escapes" |
| 87 |
C the main.F master loop. This should not happen |
| 88 |
C if the multi-threading compilation tools works right. |
| 89 |
C But (see for example KAP) this is not always the case! |
| 90 |
COMMON /EESUPP_L/ thError, threadIsRunning, threadIsComplete, |
| 91 |
& allMyEdgesAreSharedMemory, usingMPI, usingSyncMessages, |
| 92 |
& notUsingXPeriodicity, notUsingYPeriodicity |
| 93 |
LOGICAL thError(MAX_NO_THREADS) |
| 94 |
LOGICAL threadIsRunning(MAX_NO_THREADS) |
| 95 |
LOGICAL threadIsComplete(MAX_NO_THREADS) |
| 96 |
LOGICAL allMyEdgesAreSharedMemory(MAX_NO_THREADS) |
| 97 |
LOGICAL usingMPI |
| 98 |
LOGICAL usingSyncMessages |
| 99 |
LOGICAL notUsingXPeriodicity |
| 100 |
LOGICAL notUsingYPeriodicity |
| 101 |
|
| 102 |
C-- COMMON /EESUPP_I/ Parallel support integer globals |
| 103 |
C pidW - Process ID of neighbor to West |
| 104 |
C pidE - ditto East |
| 105 |
C pidN - ditto North |
| 106 |
C pidS - ditto South |
| 107 |
C Note: pid[XY] is not necessairily the UNIX |
| 108 |
C process id - it is just an identifying |
| 109 |
C number. |
| 110 |
C myPid - My own process id |
| 111 |
C nProcs - Number of processes |
| 112 |
C westCommunicationMode - Mode of communication for each tile face |
| 113 |
C eastCommunicationMode |
| 114 |
C northCommunicationMode |
| 115 |
C southCommunicationMode |
| 116 |
C bi0 - Low cartesian tile index for this process |
| 117 |
C bj0 Note - In a tile distribution with holes bi0 and bj0 |
| 118 |
C are not useful. Neighboring tile indices must |
| 119 |
C be derived some other way. |
| 120 |
C tileNo - Tile identification number for my tile and |
| 121 |
C tileNo[WENS] my N,S,E,W neighbor tiles. |
| 122 |
C tilePid[WENS] - Process identification number for |
| 123 |
C my N,S,E,W neighbor tiles. |
| 124 |
C nTx, nTy - No. threads in X and Y. This assumes a simple |
| 125 |
C cartesian gridding of the threads which is not |
| 126 |
C required elsewhere but that makes it easier. |
| 127 |
COMMON /EESUPP_I/ |
| 128 |
& myPid, nProcs, pidW, pidE, pidN, pidS, |
| 129 |
& tileCommModeW, tileCommModeE, |
| 130 |
& tileCommModeN, tileCommModeS, |
| 131 |
& tileNo, tileNoW, tileNoE, tileNoS, tileNoN, |
| 132 |
& tilePidW, tilePidE, tilePidS, tilePidN, |
| 133 |
& tileBiW, tileBiE, tileBiS, tileBiN, |
| 134 |
& tileBjW, tileBjE, tileBjS, tileBjN, |
| 135 |
& tileTagSendW, tileTagSendE, tileTagSendS, tileTagSendN, |
| 136 |
& tileTagRecvW, tileTagRecvE, tileTagRecvS, tileTagRecvN |
| 137 |
INTEGER myPid |
| 138 |
INTEGER nProcs |
| 139 |
INTEGER pidW |
| 140 |
INTEGER pidE |
| 141 |
INTEGER pidN |
| 142 |
INTEGER pidS |
| 143 |
INTEGER tileCommModeW ( nSx, nSy ) |
| 144 |
INTEGER tileCommModeE ( nSx, nSy ) |
| 145 |
INTEGER tileCommModeN ( nSx, nSy ) |
| 146 |
INTEGER tileCommModeS ( nSx, nSy ) |
| 147 |
INTEGER tileNo( nSx, nSy ) |
| 148 |
INTEGER tileNoW( nSx, nSy ) |
| 149 |
INTEGER tileNoE( nSx, nSy ) |
| 150 |
INTEGER tileNoN( nSx, nSy ) |
| 151 |
INTEGER tileNoS( nSx, nSy ) |
| 152 |
INTEGER tilePidW( nSx, nSy ) |
| 153 |
INTEGER tilePidE( nSx, nSy ) |
| 154 |
INTEGER tilePidN( nSx, nSy ) |
| 155 |
INTEGER tilePidS( nSx, nSy ) |
| 156 |
INTEGER tileBiW( nSx, nSy ) |
| 157 |
INTEGER tileBiE( nSx, nSy ) |
| 158 |
INTEGER tileBiN( nSx, nSy ) |
| 159 |
INTEGER tileBiS( nSx, nSy ) |
| 160 |
INTEGER tileBjW( nSx, nSy ) |
| 161 |
INTEGER tileBjE( nSx, nSy ) |
| 162 |
INTEGER tileBjN( nSx, nSy ) |
| 163 |
INTEGER tileBjS( nSx, nSy ) |
| 164 |
INTEGER tileTagSendW( nSx, nSy ) |
| 165 |
INTEGER tileTagSendE( nSx, nSy ) |
| 166 |
INTEGER tileTagSendN( nSx, nSy ) |
| 167 |
INTEGER tileTagSendS( nSx, nSy ) |
| 168 |
INTEGER tileTagRecvW( nSx, nSy ) |
| 169 |
INTEGER tileTagRecvE( nSx, nSy ) |
| 170 |
INTEGER tileTagRecvN( nSx, nSy ) |
| 171 |
INTEGER tileTagRecvS( nSx, nSy ) |
| 172 |
|
| 173 |
#ifdef ALLOW_USE_MPI |
| 174 |
C-- Include MPI standard Fortran header file |
| 175 |
#include "mpif.h" |
| 176 |
#define _mpiTRUE_ 1 |
| 177 |
#define _mpiFALSE_ 0 |
| 178 |
|
| 179 |
C-- COMMON /EESUPP_MPI_I/ MPI parallel support integer globals |
| 180 |
C mpiPidW - MPI process id for west neighbor. |
| 181 |
C mpiPidE - MPI process id for east neighbor. |
| 182 |
C mpiPidN - MPI process id for north neighbor. |
| 183 |
C mpiPidS - MPI process id for south neighbor. |
| 184 |
C mpiPidNW - MPI process id for northwest neighbor. |
| 185 |
C mpiPidNE - MPI process id for northeast neighbor. |
| 186 |
C mpiPidSW - MPI process id for southwest neighbor. |
| 187 |
C mpiPidSE - MPI process id for southeast neighbor. |
| 188 |
C mpiPidIO - MPI process to use for IO. |
| 189 |
C mpiNprocs - No. of MPI processes. |
| 190 |
C mpiMyId - MPI process id of me. |
| 191 |
C mpiComm - MPI communicator to use. |
| 192 |
C mpiPx - My MPI proc. grid X coord |
| 193 |
C mpiPy - My MPI proc. grid Y coord |
| 194 |
C mpiXGlobalLo - My bottom-left (south-west) x-coordinate in |
| 195 |
C global domain. |
| 196 |
C mpiYGlobalLo - My bottom-left (south-west) y-coordinate in |
| 197 |
C global domain. |
| 198 |
C mpiTypeXFaceBlock_xy_r4 - Primitives for communicating edge |
| 199 |
C mpiTypeXFaceBlock_xy_r8 of a block. |
| 200 |
C mpiTypeYFaceBlock_xy_r4 XFace is used in east-west transfer |
| 201 |
C mpiTypeYFaceBlock_xy_r8 YFace is used in nrth-south transfer |
| 202 |
C mpiTypeXFaceBlock_xyz_r4 xy is used in two-dimensional arrays |
| 203 |
C mpiTypeXFaceBlock_xyz_r8 xyz is used with three-dimensional arrays |
| 204 |
C mpiTypeYFaceBlock_xyz_r4 r4 is used for real*4 data |
| 205 |
C mpiTypeYFaceBlock_xyz_r8 r8 is used for real*8 data |
| 206 |
C mpiTypeXFaceThread_xy_r4 - Composites of the above primitives |
| 207 |
C mpiTypeXFaceThread_xy_r8 for communicating edges of all blocks |
| 208 |
C mpiTypeYFaceThread_xy_r4 owned by a thread. |
| 209 |
C mpiTypeYFaceThread_xy_r8 |
| 210 |
C mpiTypeXFaceThread_xyz_r4 |
| 211 |
C mpiTypeXFaceThread_xyz_r8 |
| 212 |
C mpiTypeYFaceThread_xyz_r4 |
| 213 |
C mpiTypeYFaceBlock_xyz_r8 |
| 214 |
C mpiTagE - Tags are needed to mark requests when MPI is running |
| 215 |
C mpiTagW between multithreaded processes or when the same process. |
| 216 |
C mpiTagS is a neighbor in more than one direction. The tags ensure that |
| 217 |
C mpiTagN a thread will get the message it is looking for. |
| 218 |
C mpiTagSW The scheme adopted is to tag messages according to |
| 219 |
C mpiTagSE the direction they are travelling. Thus a message |
| 220 |
C mpiTagNW travelling east is tagged mpiTagE. However, in a |
| 221 |
C mpiTagNE multi-threaded environemnt several messages could |
| 222 |
C be travelling east from the same process at the |
| 223 |
C same time. The tag is therefore modified to |
| 224 |
C be mpiTag[EWS...]*nThreads+myThid. This requires that |
| 225 |
C each thread also know the thread ids of its "neighbor" |
| 226 |
C threads. |
| 227 |
COMMON /EESUPP_MPI_I/ |
| 228 |
& mpiPidW, mpiPidE, mpiPidS, mpiPidN, |
| 229 |
& mpiPidSE, mpiPidSW, mpiPidNE, mpiPidNW, |
| 230 |
& mpiPidIo, mpiMyId, mpiNProcs, mpiComm, |
| 231 |
& mpiPx, mpiPy, mpiXGlobalLo, mpiYGlobalLo, |
| 232 |
& mpiTypeXFaceBlock_xy_r4, mpiTypeXFaceBlock_xy_r8, |
| 233 |
& mpiTypeYFaceBlock_xy_r4, mpiTypeYFaceBlock_xy_r8, |
| 234 |
& mpiTypeXFaceBlock_xyz_r4, mpiTypeXFaceBlock_xyz_r8, |
| 235 |
& mpiTypeYFaceBlock_xyz_r4, mpiTypeYFaceBlock_xyz_r8, |
| 236 |
& mpiTypeXFaceThread_xy_r4, mpiTypeXFaceThread_xy_r8, |
| 237 |
& mpiTypeYFaceThread_xy_r4, mpiTypeYFaceThread_xy_r8, |
| 238 |
& mpiTypeXFaceThread_xyz_r4, mpiTypeXFaceThread_xyz_r8, |
| 239 |
& mpiTypeYFaceThread_xyz_r4, mpiTypeYFaceThread_xyz_r8, |
| 240 |
& mpiTagE, mpiTagW, mpiTagN, mpiTagS, |
| 241 |
& mpiTagSE, mpiTagSW, mpiTagNW, mpiTagNE |
| 242 |
|
| 243 |
INTEGER mpiPidW |
| 244 |
INTEGER mpiPidE |
| 245 |
INTEGER mpiPidS |
| 246 |
INTEGER mpiPidN |
| 247 |
INTEGER mpiPidSW |
| 248 |
INTEGER mpiPidSE |
| 249 |
INTEGER mpiPidNW |
| 250 |
INTEGER mpiPidNE |
| 251 |
INTEGER mpiPidIO |
| 252 |
INTEGER mpiMyId |
| 253 |
INTEGER mpiNProcs |
| 254 |
INTEGER mpiComm |
| 255 |
INTEGER mpiPx |
| 256 |
INTEGER mpiPy |
| 257 |
INTEGER mpiXGlobalLo |
| 258 |
INTEGER mpiYGlobalLo |
| 259 |
INTEGER mpiTypeXFaceBlock_xy_r4 |
| 260 |
INTEGER mpiTypeXFaceBlock_xy_r8 |
| 261 |
INTEGER mpiTypeYFaceBlock_xy_r4 |
| 262 |
INTEGER mpiTypeYFaceBlock_xy_r8 |
| 263 |
INTEGER mpiTypeXFaceBlock_xyz_r4 |
| 264 |
INTEGER mpiTypeXFaceBlock_xyz_r8 |
| 265 |
INTEGER mpiTypeYFaceBlock_xyz_r4 |
| 266 |
INTEGER mpiTypeYFaceBlock_xyz_r8 |
| 267 |
INTEGER mpiTypeXFaceThread_xy_r4(MAX_NO_THREADS) |
| 268 |
INTEGER mpiTypeXFaceThread_xy_r8(MAX_NO_THREADS) |
| 269 |
INTEGER mpiTypeYFaceThread_xy_r4(MAX_NO_THREADS) |
| 270 |
INTEGER mpiTypeYFaceThread_xy_r8(MAX_NO_THREADS) |
| 271 |
INTEGER mpiTypeXFaceThread_xyz_r4(MAX_NO_THREADS) |
| 272 |
INTEGER mpiTypeXFaceThread_xyz_r8(MAX_NO_THREADS) |
| 273 |
INTEGER mpiTypeYFaceThread_xyz_r4(MAX_NO_THREADS) |
| 274 |
INTEGER mpiTypeYFaceThread_xyz_r8(MAX_NO_THREADS) |
| 275 |
INTEGER mpiTagNW |
| 276 |
INTEGER mpiTagNE |
| 277 |
INTEGER mpiTagSW |
| 278 |
INTEGER mpiTagSE |
| 279 |
INTEGER mpiTagW |
| 280 |
INTEGER mpiTagE |
| 281 |
INTEGER mpiTagN |
| 282 |
INTEGER mpiTagS |
| 283 |
|
| 284 |
C-- COMMON /MPI_FULLMAP_I/ holds integer arrays of the full list of MPI process |
| 285 |
C mpi_myXGlobalLo :: List of all processors bottom-left X-index in global domain |
| 286 |
C mpi_myYGlobalLo :: List of all processors bottom-left Y-index in global domain |
| 287 |
C Note: needed for mpi gather/scatter routines & singleCpuIO. |
| 288 |
COMMON /MPI_FULLMAP_I/ |
| 289 |
& mpi_myXGlobalLo, mpi_myYGlobalLo |
| 290 |
INTEGER mpi_myXGlobalLo(nPx*nPy) |
| 291 |
INTEGER mpi_myYGlobalLo(nPx*nPy) |
| 292 |
|
| 293 |
C MPI communicator describing this model realization |
| 294 |
COMMON /MPI_COMMS/ |
| 295 |
& MPI_COMM_MODEL |
| 296 |
INTEGER MPI_COMM_MODEL |
| 297 |
|
| 298 |
#endif /* ALLOW_USE_MPI */ |
Parent Directory
| 
Revision Log
| 
Revision Graph