1 |
C $Header: /u/gcmpack/MITgcm/eesupp/inc/EXCH.h,v 1.6 2004/04/03 04:46:34 edhill Exp $ |
2 |
C $Name: $ |
3 |
C |
4 |
CBOP |
5 |
C !ROUTINE: EXCH.h |
6 |
C !INTERFACE: |
7 |
C include "EXCH.h" |
8 |
C |
9 |
C !DESCRIPTION: |
10 |
C *==========================================================* |
11 |
C | EXCH.h |
12 |
C *==========================================================* |
13 |
C | Support data structures for |
14 |
C | the MITgcm-UV "exchange routines" code. This data should |
15 |
C | be private to the execution environment routines. |
16 |
C *==========================================================* |
17 |
CEOP |
18 |
|
19 |
#ifndef _RL |
20 |
#define _RL Real*8 |
21 |
#endif |
22 |
|
23 |
#ifndef _RS |
24 |
#define _RS Real*4 |
25 |
#endif |
26 |
|
27 |
#ifndef _R8 |
28 |
#define _R8 Real*8 |
29 |
#endif |
30 |
|
31 |
#ifndef _R4 |
32 |
#define _R4 Real*4 |
33 |
#endif |
34 |
|
35 |
#ifndef _tileCommModeW |
36 |
#define _tileCommModeW(a,b) tileCommModeW(a,b) |
37 |
#endif |
38 |
|
39 |
#ifndef _tileCommModeE |
40 |
#define _tileCommModeE(a,b) tileCommModeE(a,b) |
41 |
#endif |
42 |
|
43 |
#ifndef _tileBiE |
44 |
#define _tileBiE(a,b) tileBiE(a,b) |
45 |
#endif |
46 |
|
47 |
#ifndef _tileBiW |
48 |
#define _tileBiW(a,b) tileBiW(a,b) |
49 |
#endif |
50 |
|
51 |
#ifndef _tileBjE |
52 |
#define _tileBjE(a,b) tileBjE(a,b) |
53 |
#endif |
54 |
|
55 |
#ifndef _tileBjW |
56 |
#define _tileBjW(a,b) tileBjW(a,b) |
57 |
#endif |
58 |
|
59 |
#ifndef _mpiPidW |
60 |
#define _mpiPidW(a,b) pidW(a,b) |
61 |
#endif |
62 |
|
63 |
#ifndef _mpiIdE |
64 |
#define _mpiIdE(a,b) pidE(a,b) |
65 |
#endif |
66 |
|
67 |
#ifndef _tileTagSendW |
68 |
#define _tileTagSendW(a,b) tileTagSendW(a,b) |
69 |
#endif |
70 |
|
71 |
#ifndef _tileTagSendE |
72 |
#define _tileTagSendE(a,b) tileTagSendE(a,b) |
73 |
#endif |
74 |
|
75 |
#ifndef _tileTagRecvW |
76 |
#define _tileTagRecvW(a,b) tileTagRecvW(a,b) |
77 |
#endif |
78 |
|
79 |
#ifndef _tileTagRecvE |
80 |
#define _tileTagRecvE(a,b) tileTagRecvE(a,b) |
81 |
#endif |
82 |
|
83 |
#ifndef _tileCommModeS |
84 |
#define _tileCommModeS(a,b) tileCommModeS(a,b) |
85 |
#endif |
86 |
|
87 |
#ifndef _tileCommModeN |
88 |
#define _tileCommModeN(a,b) tileCommModeN(a,b) |
89 |
#endif |
90 |
|
91 |
#ifndef _tileBiN |
92 |
#define _tileBiN(a,b) tileBiN(a,b) |
93 |
#endif |
94 |
|
95 |
#ifndef _tileBiS |
96 |
#define _tileBiS(a,b) tileBiS(a,b) |
97 |
#endif |
98 |
|
99 |
#ifndef _tileBjN |
100 |
#define _tileBjN(a,b) tileBjN(a,b) |
101 |
#endif |
102 |
|
103 |
#ifndef _tileBjS |
104 |
#define _tileBjS(a,b) tileBjS(a,b) |
105 |
#endif |
106 |
|
107 |
#ifndef _mpiPidS |
108 |
#define _mpiPidS(a,b) pidS(a,b) |
109 |
#endif |
110 |
|
111 |
#ifndef _mpiIdN |
112 |
#define _mpiIdN(a,b) pidN(a,b) |
113 |
#endif |
114 |
|
115 |
#ifndef _tileTagSendS |
116 |
#define _tileTagSendS(a,b) tileTagSendS(a,b) |
117 |
#endif |
118 |
|
119 |
#ifndef _tileTagSendN |
120 |
#define _tileTagSendN(a,b) tileTagSendN(a,b) |
121 |
#endif |
122 |
|
123 |
#ifndef _tileTagRecvS |
124 |
#define _tileTagRecvS(a,b) tileTagRecvS(a,b) |
125 |
#endif |
126 |
|
127 |
#ifndef _tileTagRecvN |
128 |
#define _tileTagRecvN(a,b) tileTagRecvN(a,b) |
129 |
#endif |
130 |
|
131 |
#ifndef _theSimulationMode |
132 |
#define _theSimulationMode theSimulationMode |
133 |
#endif |
134 |
|
135 |
#ifndef _EXCH_SPIN_LIMIT |
136 |
#define _EXCH_SPIN_LIMIT EXCH_SPIN_LIMIT |
137 |
#endif |
138 |
|
139 |
C MAX_OLX_EXCH - Maximum overlap region allowed in X |
140 |
C MAX_OLY_EXCH - Maximum overlap region allowed in Y |
141 |
C MAX_NR_EXCH - Maximum number of vertical levels allowed |
142 |
C NUMBER_OF_BUFFER_LEVELS - Number of levels of buffer allowed. |
143 |
C EXCH_SPIN_LIMIT - Error trapping threshold for deadlocked exchange |
144 |
INTEGER MAX_OLX_EXCH |
145 |
PARAMETER ( MAX_OLX_EXCH = MAX_OLX ) |
146 |
INTEGER MAX_OLY_EXCH |
147 |
PARAMETER ( MAX_OLY_EXCH = MAX_OLY ) |
148 |
INTEGER MAX_NR_EXCH |
149 |
PARAMETER ( MAX_NR_EXCH = nR + 1 ) |
150 |
INTEGER NUMBER_OF_BUFFER_LEVELS |
151 |
PARAMETER ( NUMBER_OF_BUFFER_LEVELS = 1 ) |
152 |
INTEGER EXCH_SPIN_LIMIT |
153 |
PARAMETER ( EXCH_SPIN_LIMIT = 100000000 ) |
154 |
|
155 |
C |
156 |
C L_BUFFER[XY] - Maximum size for exchange buffer in |
157 |
C L_WBUFFER west, |
158 |
C L_EBUFFER east, |
159 |
C L_SBUFFER south, |
160 |
C L_NBUFFER north. |
161 |
INTEGER L_BUFFERX |
162 |
PARAMETER ( L_BUFFERX = |
163 |
& (sNy+2*MAX_OLY_EXCH)*MAX_OLX_EXCH*MAX_NR_EXCH ) |
164 |
INTEGER L_BUFFERY |
165 |
PARAMETER ( L_BUFFERY = |
166 |
& (sNx+2*MAX_OLX_EXCH)*MAX_OLY_EXCH*MAX_NR_EXCH ) |
167 |
INTEGER L_WBUFFER |
168 |
INTEGER L_EBUFFER |
169 |
INTEGER L_SBUFFER |
170 |
INTEGER L_NBUFFER |
171 |
PARAMETER ( L_WBUFFER = L_BUFFERX, |
172 |
& L_EBUFFER = L_BUFFERX, |
173 |
& L_SBUFFER = L_BUFFERY, |
174 |
& L_NBUFFER = L_BUFFERY ) |
175 |
|
176 |
C-- COMMON / EXCH_L / LOGICAL number common arrays for exchanges |
177 |
C exchNeedsMemSync - TRUE if memory sync. required to ensure |
178 |
C memory consistency during exchange |
179 |
C exchUsesBarrier - TRUE if we use a call to BAR to do sync. |
180 |
C between processes. On some machines we wont |
181 |
C spin on the Ack setting ( the T90 ), |
182 |
C instead we will use s system barrier. |
183 |
C On the T90 the system barrier is very fast and |
184 |
C switches out the thread while it waits. On most |
185 |
C machines the system barrier is much too slow and if |
186 |
C we own the machine and have one thread per process |
187 |
C preemption is not a problem. |
188 |
C exchCollectStatistics - Turns exchange statistics collecting on and off. |
189 |
|
190 |
COMMON / EXCH_L / exchNeedsMemSync, exchUsesBarrier, |
191 |
& exchCollectStatistics |
192 |
LOGICAL exchNeedsMemSync |
193 |
LOGICAL exchUsesBarrier |
194 |
LOGICAL exchCollectStatistics |
195 |
|
196 |
C-- COMMON / EXCH_R / REAL number common arrays for exchanges |
197 |
C xxxxSendBuf - Buffer used for sending data to another tile. |
198 |
C xxxxRecvBuf - Buffer used for receiving data from another tile. |
199 |
COMMON / EXCH_R / |
200 |
& westSendBuf_RL, eastSendBuf_RL, |
201 |
& southSendBuf_RL, northSendBuf_RL, |
202 |
& westRecvBuf_RL, eastRecvBuf_RL, |
203 |
& southRecvBuf_RL, northRecvBuf_RL, |
204 |
& westSendBuf_RS, eastSendBuf_RS, |
205 |
& southSendBuf_RS, northSendBuf_RS, |
206 |
& westRecvBuf_RS, eastRecvBuf_RS, |
207 |
& southRecvBuf_RS, northRecvBuf_RS, |
208 |
& westSendBuf_R8, eastSendBuf_R8, |
209 |
& southSendBuf_R8, northSendBuf_R8, |
210 |
& westRecvBuf_R8, eastRecvBuf_R8, |
211 |
& southRecvBuf_R8, northRecvBuf_R8, |
212 |
& westSendBuf_R4, eastSendBuf_R4, |
213 |
& southSendBuf_R4, northSendBuf_R4, |
214 |
& westRecvBuf_R4, eastRecvBuf_R4, |
215 |
& southRecvBuf_R4, northRecvBuf_R4 |
216 |
_RL westSendBuf_RL( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
217 |
& nSx, nSy ) |
218 |
_RL eastSendBuf_RL( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
219 |
& nSx, nSy ) |
220 |
_RL southSendBuf_RL( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
221 |
& nSx, nSy ) |
222 |
_RL northSendBuf_RL( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
223 |
& nSx, nSy ) |
224 |
_RL westRecvBuf_RL( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
225 |
& nSx, nSy ) |
226 |
_RL eastRecvBuf_RL( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
227 |
& nSx, nSy ) |
228 |
_RL southRecvBuf_RL( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
229 |
& nSx, nSy ) |
230 |
_RL northRecvBuf_RL( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
231 |
& nSx, nSy ) |
232 |
_RS westSendBuf_RS( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
233 |
& nSx, nSy ) |
234 |
_RS eastSendBuf_RS( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
235 |
& nSx, nSy ) |
236 |
_RS southSendBuf_RS( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
237 |
& nSx, nSy ) |
238 |
_RS northSendBuf_RS( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
239 |
& nSx, nSy ) |
240 |
_RS westRecvBuf_RS( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
241 |
& nSx, nSy ) |
242 |
_RS eastRecvBuf_RS( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
243 |
& nSx, nSy ) |
244 |
_RS southRecvBuf_RS( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
245 |
& nSx, nSy ) |
246 |
_RS northRecvBuf_RS( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
247 |
& nSx, nSy ) |
248 |
_R8 westSendBuf_R8( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
249 |
& nSx, nSy ) |
250 |
_R8 eastSendBuf_R8( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
251 |
& nSx, nSy ) |
252 |
_R8 southSendBuf_R8( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
253 |
& nSx, nSy ) |
254 |
_R8 northSendBuf_R8( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
255 |
& nSx, nSy ) |
256 |
_R8 westRecvBuf_R8( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
257 |
& nSx, nSy ) |
258 |
_R8 eastRecvBuf_R8( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
259 |
& nSx, nSy ) |
260 |
_R8 southRecvBuf_R8( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
261 |
& nSx, nSy ) |
262 |
_R8 northRecvBuf_R8( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
263 |
& nSx, nSy ) |
264 |
_R4 westSendBuf_R4( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
265 |
& nSx, nSy ) |
266 |
_R4 eastSendBuf_R4( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
267 |
& nSx, nSy ) |
268 |
_R4 southSendBuf_R4( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
269 |
& nSx, nSy ) |
270 |
_R4 northSendBuf_R4( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
271 |
& nSx, nSy ) |
272 |
_R4 westRecvBuf_R4( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
273 |
& nSx, nSy ) |
274 |
_R4 eastRecvBuf_R4( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
275 |
& nSx, nSy ) |
276 |
_R4 southRecvBuf_R4( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
277 |
& nSx, nSy ) |
278 |
_R4 northRecvBuf_R4( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
279 |
& nSx, nSy ) |
280 |
|
281 |
C-- COMMON / EXCH_I / INTEGER common arrays for exchanges |
282 |
C xxxxSendAck - Flag indicating ready to send data. |
283 |
C xxxxRecvAck - Falg indicating receive data is ready. |
284 |
C exchBufferLevel - Current cyclic buffer level. |
285 |
C exchNReqsX, exchNReqsY - Pending message counts |
286 |
C exchReqIdX, exchReqIdY -Pending message identifiers |
287 |
C *Spin* - Exchange statistics holder |
288 |
C Count - No. spins for each thread |
289 |
C Max - Maximum spins for an exchange |
290 |
C Min - Minimum spins for an exchange |
291 |
COMMON / EXCH_I / |
292 |
& westSendAck, eastSendAck, southSendAck, northSendAck, |
293 |
& westRecvAck, eastRecvAck, southRecvAck, northRecvAck, |
294 |
& exchangeBufLevel, |
295 |
& exchNReqsX, exchNReqsY, exchReqIdX, exchReqIdY, |
296 |
& exchRecvXSpinCount, exchRecvXSpinMax, exchRecvXSpinMin, |
297 |
& exchRecvXExchCount, |
298 |
& exchRecvYSpinCount, exchRecvYSpinMax, exchRecvYSpinMin, |
299 |
& exchRecvYExchCount |
300 |
INTEGER westSendAck( NUMBER_OF_BUFFER_LEVELS, |
301 |
& nSx, nSy ) |
302 |
INTEGER eastSendAck( NUMBER_OF_BUFFER_LEVELS, |
303 |
& nSx, nSy ) |
304 |
INTEGER southSendAck( NUMBER_OF_BUFFER_LEVELS, |
305 |
& nSx, nSy ) |
306 |
INTEGER northSendAck( NUMBER_OF_BUFFER_LEVELS, |
307 |
& nSx, nSy ) |
308 |
INTEGER westRecvAck( NUMBER_OF_BUFFER_LEVELS, |
309 |
& nSx, nSy ) |
310 |
INTEGER eastRecvAck( NUMBER_OF_BUFFER_LEVELS, |
311 |
& nSx, nSy ) |
312 |
INTEGER southRecvAck( NUMBER_OF_BUFFER_LEVELS, |
313 |
& nSx, nSy ) |
314 |
INTEGER northRecvAck( NUMBER_OF_BUFFER_LEVELS, |
315 |
& nSx, nSy ) |
316 |
INTEGER exchangeBufLevel( cacheLineSize/4, nSx, nSy ) |
317 |
INTEGER exchNReqsX(cacheLineSize/4,nSx,nSy) |
318 |
INTEGER exchNReqsY(cacheLineSize/4,nSx,nSy) |
319 |
INTEGER exchReqIdX(2*nSx+2*nSy,cacheLineSize/4,nSx,nSy) |
320 |
INTEGER exchReqIdY(2*nSx+2*nSy,cacheLineSize/4,nSx,nSy) |
321 |
INTEGER exchRecvXSpinCount(cacheLineSize/4, nSx, nSy) |
322 |
INTEGER exchRecvXExchCount(cacheLineSize/4, nSx, nSy) |
323 |
INTEGER exchRecvXSpinMax (cacheLineSize/4, nSx, nSy) |
324 |
INTEGER exchRecvXSpinMin (cacheLineSize/4, nSx, nSy) |
325 |
INTEGER exchRecvYSpinCount(cacheLineSize/4, nSx, nSy) |
326 |
INTEGER exchRecvYExchCount(cacheLineSize/4, nSx, nSy) |
327 |
INTEGER exchRecvYSpinMax (cacheLineSize/4, nSx, nSy) |
328 |
INTEGER exchRecvYSpinMin (cacheLineSize/4, nSx, nSy) |
329 |
|