| 1 |
jmc |
1.10 |
C $Header: /u/gcmpack/MITgcm/eesupp/inc/EESUPPORT.h,v 1.9 2005/11/05 00:49:47 jmc Exp $ |
| 2 |
cnh |
1.7 |
C $Name: $ |
| 3 |
|
|
CBOP |
| 4 |
jmc |
1.9 |
C !ROUTINE: EESUPPORT.h |
| 5 |
cnh |
1.7 |
C !INTERFACE: |
| 6 |
|
|
C include "EESUPPORT.h" |
| 7 |
cnh |
1.1 |
C |
| 8 |
cnh |
1.7 |
C !DESCRIPTION: |
| 9 |
|
|
C *==========================================================* |
| 10 |
cnh |
1.1 |
C | EESUPPORT.h | |
| 11 |
cnh |
1.7 |
C *==========================================================* |
| 12 |
afe |
1.8 |
C | Support data structures for the MITgcm UV ``execution | |
| 13 |
|
|
C | environment'' code. This data should be private to the | |
| 14 |
cnh |
1.1 |
C | execution environment routines. Data which needs to be | |
| 15 |
cnh |
1.7 |
C | accessed directly by a numerical model goes in | |
| 16 |
cnh |
1.1 |
C | EEPARAMS.h. | |
| 17 |
cnh |
1.7 |
C *==========================================================* |
| 18 |
|
|
CEOP |
| 19 |
cnh |
1.1 |
|
| 20 |
|
|
C ERROR_HEADER - String which prefixes error messages |
| 21 |
|
|
CHARACTER*(*) ERROR_HEADER |
| 22 |
|
|
PARAMETER ( ERROR_HEADER = ' *** ERROR ***' ) |
| 23 |
|
|
C PROCESS_HEADER - String which prefixes processor number |
| 24 |
|
|
CHARACTER*(*) PROCESS_HEADER |
| 25 |
|
|
PARAMETER ( PROCESS_HEADER = 'PID.TID' ) |
| 26 |
|
|
|
| 27 |
cnh |
1.4 |
C MAX_NUM_COMM_MODES - Maximum number of communication modes |
| 28 |
cnh |
1.1 |
C COMM_NONE - No edge communication |
| 29 |
cnh |
1.4 |
C COMM_MSG - Use messages to communicate edges |
| 30 |
|
|
C COMM_PUT - Use put to communicate edges |
| 31 |
|
|
C COMM_GET - Use get to communicate edges |
| 32 |
|
|
C Note - commName holds an identifying name for each communication |
| 33 |
|
|
C mode. The COMM_ parameters are used to index commName |
| 34 |
|
|
C so the COMM_ parameters need to be in the range |
| 35 |
|
|
C 1 : MAX_NUM_COMM_MODES. |
| 36 |
|
|
INTEGER MAX_NUM_COMM_MODES |
| 37 |
|
|
PARAMETER ( MAX_NUM_COMM_MODES = 4 ) |
| 38 |
cnh |
1.1 |
INTEGER COMM_NONE |
| 39 |
cnh |
1.4 |
PARAMETER ( COMM_NONE = 1 ) |
| 40 |
|
|
INTEGER COMM_MSG |
| 41 |
|
|
PARAMETER ( COMM_MSG = 2 ) |
| 42 |
|
|
INTEGER COMM_PUT |
| 43 |
|
|
PARAMETER ( COMM_PUT = 3 ) |
| 44 |
|
|
INTEGER COMM_GET |
| 45 |
|
|
PARAMETER ( COMM_GET = 4 ) |
| 46 |
|
|
COMMON /EESUPP_COMMNAME/ commName |
| 47 |
|
|
CHARACTER*10 commName(MAX_NUM_COMM_MODES) |
| 48 |
|
|
|
| 49 |
|
|
C Tile identifiers |
| 50 |
|
|
C Tiles have a number that is unique over the global domain. |
| 51 |
|
|
C A tile that is not there has its number set to NULL_TILE |
| 52 |
|
|
INTEGER NULL_TILE |
| 53 |
|
|
PARAMETER ( NULL_TILE = -1 ) |
| 54 |
|
|
|
| 55 |
cnh |
1.1 |
|
| 56 |
|
|
C-- COMMON /EESUPP_C/ Execution environment support character variables |
| 57 |
|
|
C myProcessStr - String identifying my process number |
| 58 |
|
|
COMMON /EESUPP_C/ myProcessStr |
| 59 |
|
|
CHARACTER*128 myProcessStr |
| 60 |
|
|
|
| 61 |
|
|
C-- COMMON /EESUPP_L/ Execution environment support logical variables |
| 62 |
|
|
C initMPError - Flag indicating error during multi-processing |
| 63 |
|
|
C initialisation. |
| 64 |
|
|
C finMPError - Flag indicating error during multi-processing |
| 65 |
|
|
C termination. |
| 66 |
|
|
C ThError - Thread detected an error. |
| 67 |
|
|
C usingMPI - Flag controlling use of MPI routines. This flag |
| 68 |
|
|
C allows either MPI or threads to be used in a |
| 69 |
|
|
C shared memory environment which can be a useful |
| 70 |
|
|
C debugging/performance analysis tool. |
| 71 |
|
|
C usingSyncMessages - Flag that causes blocking communication to be used |
| 72 |
|
|
C if possible. When false non-blocking EXCH routines |
| 73 |
|
|
C will be used if possible. |
| 74 |
|
|
C notUsingXPeriodicity - Flag indicating no X/Y boundary wrap around |
| 75 |
|
|
C notUsingYPeriodicity This affects the communication routines but |
| 76 |
|
|
C is generally ignored in the numerical model |
| 77 |
|
|
C code. |
| 78 |
|
|
C threadIsRunning, threadIsComplete - Flags used to check for correct behaviour |
| 79 |
|
|
C of multi-threaded code. |
| 80 |
|
|
C threadIsRunning is used to check that the |
| 81 |
|
|
C threads we need are running. This catches the |
| 82 |
|
|
C situation where a program eedata file has nTthreads |
| 83 |
|
|
C greater than the setenv PARALLEL or NCPUS variable. |
| 84 |
|
|
C threadIsComplete is used to flag that a thread has |
| 85 |
|
|
C reached the end of the model. This is used as a check to |
| 86 |
|
|
C trap problems that might occur if one thread "escapes" |
| 87 |
|
|
C the main.F master loop. This should not happen |
| 88 |
|
|
C if the multi-threading compilation tools works right. |
| 89 |
|
|
C But (see for example KAP) this is not always the case! |
| 90 |
|
|
COMMON /EESUPP_L/ thError, threadIsRunning, threadIsComplete, |
| 91 |
|
|
& allMyEdgesAreSharedMemory, usingMPI, usingSyncMessages, |
| 92 |
jmc |
1.10 |
& notUsingXPeriodicity, notUsingYPeriodicity |
| 93 |
cnh |
1.1 |
LOGICAL thError(MAX_NO_THREADS) |
| 94 |
|
|
LOGICAL threadIsRunning(MAX_NO_THREADS) |
| 95 |
|
|
LOGICAL threadIsComplete(MAX_NO_THREADS) |
| 96 |
|
|
LOGICAL allMyEdgesAreSharedMemory(MAX_NO_THREADS) |
| 97 |
|
|
LOGICAL usingMPI |
| 98 |
|
|
LOGICAL usingSyncMessages |
| 99 |
|
|
LOGICAL notUsingXPeriodicity |
| 100 |
|
|
LOGICAL notUsingYPeriodicity |
| 101 |
jmc |
1.10 |
|
| 102 |
cnh |
1.1 |
C-- COMMON /EESUPP_I/ Parallel support integer globals |
| 103 |
cnh |
1.4 |
C pidW - Process ID of neighbor to West |
| 104 |
|
|
C pidE - ditto East |
| 105 |
|
|
C pidN - ditto North |
| 106 |
|
|
C pidS - ditto South |
| 107 |
cnh |
1.1 |
C Note: pid[XY] is not necessairily the UNIX |
| 108 |
|
|
C process id - it is just an identifying |
| 109 |
|
|
C number. |
| 110 |
cnh |
1.4 |
C myPid - My own process id |
| 111 |
|
|
C nProcs - Number of processes |
| 112 |
|
|
C westCommunicationMode - Mode of communication for each tile face |
| 113 |
|
|
C eastCommunicationMode |
| 114 |
|
|
C northCommunicationMode |
| 115 |
|
|
C southCommunicationMode |
| 116 |
|
|
C bi0 - Low cartesian tile index for this process |
| 117 |
|
|
C bj0 Note - In a tile distribution with holes bi0 and bj0 |
| 118 |
|
|
C are not useful. Neighboring tile indices must |
| 119 |
|
|
C be derived some other way. |
| 120 |
|
|
C tileNo - Tile identification number for my tile and |
| 121 |
|
|
C tileNo[WENS] my N,S,E,W neighbor tiles. |
| 122 |
|
|
C tilePid[WENS] - Process identification number for |
| 123 |
|
|
C my N,S,E,W neighbor tiles. |
| 124 |
|
|
C nTx, nTy - No. threads in X and Y. This assumes a simple |
| 125 |
|
|
C cartesian gridding of the threads which is not |
| 126 |
|
|
C required elsewhere but that makes it easier. |
| 127 |
cnh |
1.1 |
COMMON /EESUPP_I/ |
| 128 |
cnh |
1.4 |
& myPid, nProcs, pidW, pidE, pidN, pidS, |
| 129 |
|
|
& tileCommModeW, tileCommModeE, |
| 130 |
|
|
& tileCommModeN, tileCommModeS, |
| 131 |
|
|
& tileNo, tileNoW, tileNoE, tileNoS, tileNoN, |
| 132 |
|
|
& tilePidW, tilePidE, tilePidS, tilePidN, |
| 133 |
|
|
& tileBiW, tileBiE, tileBiS, tileBiN, |
| 134 |
|
|
& tileBjW, tileBjE, tileBjS, tileBjN, |
| 135 |
|
|
& tileTagSendW, tileTagSendE, tileTagSendS, tileTagSendN, |
| 136 |
|
|
& tileTagRecvW, tileTagRecvE, tileTagRecvS, tileTagRecvN |
| 137 |
|
|
INTEGER myPid |
| 138 |
|
|
INTEGER nProcs |
| 139 |
cnh |
1.1 |
INTEGER pidW |
| 140 |
|
|
INTEGER pidE |
| 141 |
cnh |
1.4 |
INTEGER pidN |
| 142 |
cnh |
1.1 |
INTEGER pidS |
| 143 |
cnh |
1.4 |
INTEGER tileCommModeW ( nSx, nSy ) |
| 144 |
|
|
INTEGER tileCommModeE ( nSx, nSy ) |
| 145 |
|
|
INTEGER tileCommModeN ( nSx, nSy ) |
| 146 |
|
|
INTEGER tileCommModeS ( nSx, nSy ) |
| 147 |
|
|
INTEGER tileNo( nSx, nSy ) |
| 148 |
|
|
INTEGER tileNoW( nSx, nSy ) |
| 149 |
|
|
INTEGER tileNoE( nSx, nSy ) |
| 150 |
|
|
INTEGER tileNoN( nSx, nSy ) |
| 151 |
|
|
INTEGER tileNoS( nSx, nSy ) |
| 152 |
|
|
INTEGER tilePidW( nSx, nSy ) |
| 153 |
|
|
INTEGER tilePidE( nSx, nSy ) |
| 154 |
|
|
INTEGER tilePidN( nSx, nSy ) |
| 155 |
|
|
INTEGER tilePidS( nSx, nSy ) |
| 156 |
|
|
INTEGER tileBiW( nSx, nSy ) |
| 157 |
|
|
INTEGER tileBiE( nSx, nSy ) |
| 158 |
|
|
INTEGER tileBiN( nSx, nSy ) |
| 159 |
|
|
INTEGER tileBiS( nSx, nSy ) |
| 160 |
|
|
INTEGER tileBjW( nSx, nSy ) |
| 161 |
|
|
INTEGER tileBjE( nSx, nSy ) |
| 162 |
|
|
INTEGER tileBjN( nSx, nSy ) |
| 163 |
|
|
INTEGER tileBjS( nSx, nSy ) |
| 164 |
|
|
INTEGER tileTagSendW( nSx, nSy ) |
| 165 |
|
|
INTEGER tileTagSendE( nSx, nSy ) |
| 166 |
|
|
INTEGER tileTagSendN( nSx, nSy ) |
| 167 |
|
|
INTEGER tileTagSendS( nSx, nSy ) |
| 168 |
|
|
INTEGER tileTagRecvW( nSx, nSy ) |
| 169 |
|
|
INTEGER tileTagRecvE( nSx, nSy ) |
| 170 |
|
|
INTEGER tileTagRecvN( nSx, nSy ) |
| 171 |
|
|
INTEGER tileTagRecvS( nSx, nSy ) |
| 172 |
cnh |
1.1 |
|
| 173 |
|
|
#ifdef ALLOW_USE_MPI |
| 174 |
|
|
C-- Include MPI standard Fortran header file |
| 175 |
|
|
#include "mpif.h" |
| 176 |
|
|
#define _mpiTRUE_ 1 |
| 177 |
|
|
#define _mpiFALSE_ 0 |
| 178 |
|
|
|
| 179 |
|
|
C-- COMMON /EESUPP_MPI_I/ MPI parallel support integer globals |
| 180 |
|
|
C mpiPidW - MPI process id for west neighbor. |
| 181 |
|
|
C mpiPidE - MPI process id for east neighbor. |
| 182 |
|
|
C mpiPidN - MPI process id for north neighbor. |
| 183 |
|
|
C mpiPidS - MPI process id for south neighbor. |
| 184 |
|
|
C mpiPidNW - MPI process id for northwest neighbor. |
| 185 |
|
|
C mpiPidNE - MPI process id for northeast neighbor. |
| 186 |
|
|
C mpiPidSW - MPI process id for southwest neighbor. |
| 187 |
|
|
C mpiPidSE - MPI process id for southeast neighbor. |
| 188 |
|
|
C mpiPidIO - MPI process to use for IO. |
| 189 |
|
|
C mpiNprocs - No. of MPI processes. |
| 190 |
|
|
C mpiMyId - MPI process id of me. |
| 191 |
|
|
C mpiComm - MPI communicator to use. |
| 192 |
|
|
C mpiPx - My MPI proc. grid X coord |
| 193 |
|
|
C mpiPy - My MPI proc. grid Y coord |
| 194 |
|
|
C mpiXGlobalLo - My bottom-left (south-west) x-coordinate in |
| 195 |
|
|
C global domain. |
| 196 |
|
|
C mpiYGlobalLo - My bottom-left (south-west) y-coordinate in |
| 197 |
|
|
C global domain. |
| 198 |
|
|
C mpiTypeXFaceBlock_xy_r4 - Primitives for communicating edge |
| 199 |
|
|
C mpiTypeXFaceBlock_xy_r8 of a block. |
| 200 |
|
|
C mpiTypeYFaceBlock_xy_r4 XFace is used in east-west transfer |
| 201 |
|
|
C mpiTypeYFaceBlock_xy_r8 YFace is used in nrth-south transfer |
| 202 |
|
|
C mpiTypeXFaceBlock_xyz_r4 xy is used in two-dimensional arrays |
| 203 |
|
|
C mpiTypeXFaceBlock_xyz_r8 xyz is used with three-dimensional arrays |
| 204 |
|
|
C mpiTypeYFaceBlock_xyz_r4 r4 is used for real*4 data |
| 205 |
|
|
C mpiTypeYFaceBlock_xyz_r8 r8 is used for real*8 data |
| 206 |
|
|
C mpiTypeXFaceThread_xy_r4 - Composites of the above primitives |
| 207 |
|
|
C mpiTypeXFaceThread_xy_r8 for communicating edges of all blocks |
| 208 |
|
|
C mpiTypeYFaceThread_xy_r4 owned by a thread. |
| 209 |
|
|
C mpiTypeYFaceThread_xy_r8 |
| 210 |
|
|
C mpiTypeXFaceThread_xyz_r4 |
| 211 |
|
|
C mpiTypeXFaceThread_xyz_r8 |
| 212 |
|
|
C mpiTypeYFaceThread_xyz_r4 |
| 213 |
|
|
C mpiTypeYFaceBlock_xyz_r8 |
| 214 |
|
|
C mpiTagE - Tags are needed to mark requests when MPI is running |
| 215 |
|
|
C mpiTagW between multithreaded processes or when the same process. |
| 216 |
|
|
C mpiTagS is a neighbor in more than one direction. The tags ensure that |
| 217 |
|
|
C mpiTagN a thread will get the message it is looking for. |
| 218 |
|
|
C mpiTagSW The scheme adopted is to tag messages according to |
| 219 |
|
|
C mpiTagSE the direction they are travelling. Thus a message |
| 220 |
|
|
C mpiTagNW travelling east is tagged mpiTagE. However, in a |
| 221 |
|
|
C mpiTagNE multi-threaded environemnt several messages could |
| 222 |
|
|
C be travelling east from the same process at the |
| 223 |
|
|
C same time. The tag is therefore modified to |
| 224 |
|
|
C be mpiTag[EWS...]*nThreads+myThid. This requires that |
| 225 |
|
|
C each thread also know the thread ids of its "neighbor" |
| 226 |
|
|
C threads. |
| 227 |
|
|
COMMON /EESUPP_MPI_I/ |
| 228 |
|
|
& mpiPidW, mpiPidE, mpiPidS, mpiPidN, |
| 229 |
|
|
& mpiPidSE, mpiPidSW, mpiPidNE, mpiPidNW, |
| 230 |
|
|
& mpiPidIo, mpiMyId, mpiNProcs, mpiComm, |
| 231 |
|
|
& mpiPx, mpiPy, mpiXGlobalLo, mpiYGlobalLo, |
| 232 |
|
|
& mpiTypeXFaceBlock_xy_r4, mpiTypeXFaceBlock_xy_r8, |
| 233 |
|
|
& mpiTypeYFaceBlock_xy_r4, mpiTypeYFaceBlock_xy_r8, |
| 234 |
|
|
& mpiTypeXFaceBlock_xyz_r4, mpiTypeXFaceBlock_xyz_r8, |
| 235 |
|
|
& mpiTypeYFaceBlock_xyz_r4, mpiTypeYFaceBlock_xyz_r8, |
| 236 |
|
|
& mpiTypeXFaceThread_xy_r4, mpiTypeXFaceThread_xy_r8, |
| 237 |
|
|
& mpiTypeYFaceThread_xy_r4, mpiTypeYFaceThread_xy_r8, |
| 238 |
|
|
& mpiTypeXFaceThread_xyz_r4, mpiTypeXFaceThread_xyz_r8, |
| 239 |
|
|
& mpiTypeYFaceThread_xyz_r4, mpiTypeYFaceThread_xyz_r8, |
| 240 |
|
|
& mpiTagE, mpiTagW, mpiTagN, mpiTagS, |
| 241 |
|
|
& mpiTagSE, mpiTagSW, mpiTagNW, mpiTagNE |
| 242 |
|
|
|
| 243 |
|
|
INTEGER mpiPidW |
| 244 |
|
|
INTEGER mpiPidE |
| 245 |
|
|
INTEGER mpiPidS |
| 246 |
|
|
INTEGER mpiPidN |
| 247 |
|
|
INTEGER mpiPidSW |
| 248 |
|
|
INTEGER mpiPidSE |
| 249 |
|
|
INTEGER mpiPidNW |
| 250 |
|
|
INTEGER mpiPidNE |
| 251 |
|
|
INTEGER mpiPidIO |
| 252 |
|
|
INTEGER mpiMyId |
| 253 |
|
|
INTEGER mpiNProcs |
| 254 |
|
|
INTEGER mpiComm |
| 255 |
|
|
INTEGER mpiPx |
| 256 |
|
|
INTEGER mpiPy |
| 257 |
|
|
INTEGER mpiXGlobalLo |
| 258 |
|
|
INTEGER mpiYGlobalLo |
| 259 |
|
|
INTEGER mpiTypeXFaceBlock_xy_r4 |
| 260 |
|
|
INTEGER mpiTypeXFaceBlock_xy_r8 |
| 261 |
|
|
INTEGER mpiTypeYFaceBlock_xy_r4 |
| 262 |
|
|
INTEGER mpiTypeYFaceBlock_xy_r8 |
| 263 |
|
|
INTEGER mpiTypeXFaceBlock_xyz_r4 |
| 264 |
|
|
INTEGER mpiTypeXFaceBlock_xyz_r8 |
| 265 |
|
|
INTEGER mpiTypeYFaceBlock_xyz_r4 |
| 266 |
|
|
INTEGER mpiTypeYFaceBlock_xyz_r8 |
| 267 |
|
|
INTEGER mpiTypeXFaceThread_xy_r4(MAX_NO_THREADS) |
| 268 |
|
|
INTEGER mpiTypeXFaceThread_xy_r8(MAX_NO_THREADS) |
| 269 |
|
|
INTEGER mpiTypeYFaceThread_xy_r4(MAX_NO_THREADS) |
| 270 |
|
|
INTEGER mpiTypeYFaceThread_xy_r8(MAX_NO_THREADS) |
| 271 |
|
|
INTEGER mpiTypeXFaceThread_xyz_r4(MAX_NO_THREADS) |
| 272 |
|
|
INTEGER mpiTypeXFaceThread_xyz_r8(MAX_NO_THREADS) |
| 273 |
|
|
INTEGER mpiTypeYFaceThread_xyz_r4(MAX_NO_THREADS) |
| 274 |
|
|
INTEGER mpiTypeYFaceThread_xyz_r8(MAX_NO_THREADS) |
| 275 |
|
|
INTEGER mpiTagNW |
| 276 |
|
|
INTEGER mpiTagNE |
| 277 |
|
|
INTEGER mpiTagSW |
| 278 |
|
|
INTEGER mpiTagSE |
| 279 |
|
|
INTEGER mpiTagW |
| 280 |
|
|
INTEGER mpiTagE |
| 281 |
|
|
INTEGER mpiTagN |
| 282 |
|
|
INTEGER mpiTagS |
| 283 |
adcroft |
1.5 |
|
| 284 |
jmc |
1.9 |
C-- COMMON /MPI_FULLMAP_I/ holds integer arrays of the full list of MPI process |
| 285 |
|
|
C mpi_myXGlobalLo :: List of all processors bottom-left X-index in global domain |
| 286 |
|
|
C mpi_myYGlobalLo :: List of all processors bottom-left Y-index in global domain |
| 287 |
|
|
C Note: needed for mpi gather/scatter routines & singleCpuIO. |
| 288 |
|
|
COMMON /MPI_FULLMAP_I/ |
| 289 |
|
|
& mpi_myXGlobalLo, mpi_myYGlobalLo |
| 290 |
|
|
INTEGER mpi_myXGlobalLo(nPx*nPy) |
| 291 |
|
|
INTEGER mpi_myYGlobalLo(nPx*nPy) |
| 292 |
|
|
|
| 293 |
adcroft |
1.5 |
C MPI communicator describing this model realization |
| 294 |
|
|
COMMON /MPI_COMMS/ |
| 295 |
|
|
& MPI_COMM_MODEL |
| 296 |
|
|
INTEGER MPI_COMM_MODEL |
| 297 |
jmc |
1.9 |
|
| 298 |
cnh |
1.1 |
#endif /* ALLOW_USE_MPI */ |
Parent Directory
| 
Revision Log
| 
Revision Graph