1 |
jmc |
1.7 |
C $Header: /u/gcmpack/MITgcm/eesupp/inc/EXCH.h,v 1.6 2004/04/03 04:46:34 edhill Exp $ |
2 |
cnh |
1.4 |
C $Name: $ |
3 |
cnh |
1.1 |
C |
4 |
cnh |
1.4 |
CBOP |
5 |
|
|
C !ROUTINE: EXCH.h |
6 |
jmc |
1.7 |
C !INTERFACE: |
7 |
cnh |
1.4 |
C include "EXCH.h" |
8 |
|
|
C |
9 |
|
|
C !DESCRIPTION: |
10 |
|
|
C *==========================================================* |
11 |
jmc |
1.7 |
C | EXCH.h |
12 |
cnh |
1.4 |
C *==========================================================* |
13 |
jmc |
1.7 |
C | Support data structures for |
14 |
|
|
C | the MITgcm-UV "exchange routines" code. This data should |
15 |
|
|
C | be private to the execution environment routines. |
16 |
cnh |
1.4 |
C *==========================================================* |
17 |
|
|
CEOP |
18 |
cnh |
1.1 |
|
19 |
|
|
#ifndef _RL |
20 |
|
|
#define _RL Real*8 |
21 |
|
|
#endif |
22 |
|
|
|
23 |
|
|
#ifndef _RS |
24 |
|
|
#define _RS Real*4 |
25 |
edhill |
1.6 |
#endif |
26 |
|
|
|
27 |
|
|
#ifndef _R8 |
28 |
|
|
#define _R8 Real*8 |
29 |
|
|
#endif |
30 |
|
|
|
31 |
|
|
#ifndef _R4 |
32 |
|
|
#define _R4 Real*4 |
33 |
cnh |
1.1 |
#endif |
34 |
|
|
|
35 |
|
|
#ifndef _tileCommModeW |
36 |
|
|
#define _tileCommModeW(a,b) tileCommModeW(a,b) |
37 |
|
|
#endif |
38 |
|
|
|
39 |
|
|
#ifndef _tileCommModeE |
40 |
|
|
#define _tileCommModeE(a,b) tileCommModeE(a,b) |
41 |
|
|
#endif |
42 |
|
|
|
43 |
|
|
#ifndef _tileBiE |
44 |
|
|
#define _tileBiE(a,b) tileBiE(a,b) |
45 |
|
|
#endif |
46 |
|
|
|
47 |
|
|
#ifndef _tileBiW |
48 |
|
|
#define _tileBiW(a,b) tileBiW(a,b) |
49 |
|
|
#endif |
50 |
|
|
|
51 |
|
|
#ifndef _tileBjE |
52 |
|
|
#define _tileBjE(a,b) tileBjE(a,b) |
53 |
|
|
#endif |
54 |
|
|
|
55 |
|
|
#ifndef _tileBjW |
56 |
|
|
#define _tileBjW(a,b) tileBjW(a,b) |
57 |
|
|
#endif |
58 |
|
|
|
59 |
|
|
#ifndef _mpiPidW |
60 |
|
|
#define _mpiPidW(a,b) pidW(a,b) |
61 |
|
|
#endif |
62 |
|
|
|
63 |
|
|
#ifndef _mpiIdE |
64 |
|
|
#define _mpiIdE(a,b) pidE(a,b) |
65 |
|
|
#endif |
66 |
|
|
|
67 |
|
|
#ifndef _tileTagSendW |
68 |
|
|
#define _tileTagSendW(a,b) tileTagSendW(a,b) |
69 |
|
|
#endif |
70 |
|
|
|
71 |
|
|
#ifndef _tileTagSendE |
72 |
|
|
#define _tileTagSendE(a,b) tileTagSendE(a,b) |
73 |
|
|
#endif |
74 |
|
|
|
75 |
|
|
#ifndef _tileTagRecvW |
76 |
|
|
#define _tileTagRecvW(a,b) tileTagRecvW(a,b) |
77 |
|
|
#endif |
78 |
|
|
|
79 |
|
|
#ifndef _tileTagRecvE |
80 |
|
|
#define _tileTagRecvE(a,b) tileTagRecvE(a,b) |
81 |
|
|
#endif |
82 |
|
|
|
83 |
|
|
#ifndef _tileCommModeS |
84 |
|
|
#define _tileCommModeS(a,b) tileCommModeS(a,b) |
85 |
|
|
#endif |
86 |
|
|
|
87 |
|
|
#ifndef _tileCommModeN |
88 |
|
|
#define _tileCommModeN(a,b) tileCommModeN(a,b) |
89 |
|
|
#endif |
90 |
|
|
|
91 |
|
|
#ifndef _tileBiN |
92 |
|
|
#define _tileBiN(a,b) tileBiN(a,b) |
93 |
|
|
#endif |
94 |
|
|
|
95 |
|
|
#ifndef _tileBiS |
96 |
|
|
#define _tileBiS(a,b) tileBiS(a,b) |
97 |
|
|
#endif |
98 |
|
|
|
99 |
|
|
#ifndef _tileBjN |
100 |
|
|
#define _tileBjN(a,b) tileBjN(a,b) |
101 |
|
|
#endif |
102 |
|
|
|
103 |
|
|
#ifndef _tileBjS |
104 |
|
|
#define _tileBjS(a,b) tileBjS(a,b) |
105 |
|
|
#endif |
106 |
|
|
|
107 |
|
|
#ifndef _mpiPidS |
108 |
|
|
#define _mpiPidS(a,b) pidS(a,b) |
109 |
|
|
#endif |
110 |
|
|
|
111 |
|
|
#ifndef _mpiIdN |
112 |
|
|
#define _mpiIdN(a,b) pidN(a,b) |
113 |
|
|
#endif |
114 |
|
|
|
115 |
|
|
#ifndef _tileTagSendS |
116 |
|
|
#define _tileTagSendS(a,b) tileTagSendS(a,b) |
117 |
|
|
#endif |
118 |
|
|
|
119 |
|
|
#ifndef _tileTagSendN |
120 |
|
|
#define _tileTagSendN(a,b) tileTagSendN(a,b) |
121 |
|
|
#endif |
122 |
|
|
|
123 |
|
|
#ifndef _tileTagRecvS |
124 |
|
|
#define _tileTagRecvS(a,b) tileTagRecvS(a,b) |
125 |
|
|
#endif |
126 |
|
|
|
127 |
|
|
#ifndef _tileTagRecvN |
128 |
|
|
#define _tileTagRecvN(a,b) tileTagRecvN(a,b) |
129 |
|
|
#endif |
130 |
|
|
|
131 |
|
|
#ifndef _theSimulationMode |
132 |
|
|
#define _theSimulationMode theSimulationMode |
133 |
|
|
#endif |
134 |
|
|
|
135 |
|
|
#ifndef _EXCH_SPIN_LIMIT |
136 |
|
|
#define _EXCH_SPIN_LIMIT EXCH_SPIN_LIMIT |
137 |
|
|
#endif |
138 |
|
|
|
139 |
|
|
C MAX_OLX_EXCH - Maximum overlap region allowed in X |
140 |
|
|
C MAX_OLY_EXCH - Maximum overlap region allowed in Y |
141 |
|
|
C MAX_NR_EXCH - Maximum number of vertical levels allowed |
142 |
|
|
C NUMBER_OF_BUFFER_LEVELS - Number of levels of buffer allowed. |
143 |
|
|
C EXCH_SPIN_LIMIT - Error trapping threshold for deadlocked exchange |
144 |
|
|
INTEGER MAX_OLX_EXCH |
145 |
|
|
PARAMETER ( MAX_OLX_EXCH = MAX_OLX ) |
146 |
|
|
INTEGER MAX_OLY_EXCH |
147 |
|
|
PARAMETER ( MAX_OLY_EXCH = MAX_OLY ) |
148 |
|
|
INTEGER MAX_NR_EXCH |
149 |
|
|
PARAMETER ( MAX_NR_EXCH = nR + 1 ) |
150 |
|
|
INTEGER NUMBER_OF_BUFFER_LEVELS |
151 |
adcroft |
1.2 |
PARAMETER ( NUMBER_OF_BUFFER_LEVELS = 1 ) |
152 |
cnh |
1.1 |
INTEGER EXCH_SPIN_LIMIT |
153 |
|
|
PARAMETER ( EXCH_SPIN_LIMIT = 100000000 ) |
154 |
|
|
|
155 |
|
|
C |
156 |
jmc |
1.7 |
C L_BUFFER[XY] - Maximum size for exchange buffer in |
157 |
cnh |
1.1 |
C L_WBUFFER west, |
158 |
|
|
C L_EBUFFER east, |
159 |
|
|
C L_SBUFFER south, |
160 |
|
|
C L_NBUFFER north. |
161 |
|
|
INTEGER L_BUFFERX |
162 |
jmc |
1.7 |
PARAMETER ( L_BUFFERX = |
163 |
cnh |
1.1 |
& (sNy+2*MAX_OLY_EXCH)*MAX_OLX_EXCH*MAX_NR_EXCH ) |
164 |
|
|
INTEGER L_BUFFERY |
165 |
jmc |
1.7 |
PARAMETER ( L_BUFFERY = |
166 |
cnh |
1.1 |
& (sNx+2*MAX_OLX_EXCH)*MAX_OLY_EXCH*MAX_NR_EXCH ) |
167 |
|
|
INTEGER L_WBUFFER |
168 |
|
|
INTEGER L_EBUFFER |
169 |
|
|
INTEGER L_SBUFFER |
170 |
|
|
INTEGER L_NBUFFER |
171 |
|
|
PARAMETER ( L_WBUFFER = L_BUFFERX, |
172 |
|
|
& L_EBUFFER = L_BUFFERX, |
173 |
|
|
& L_SBUFFER = L_BUFFERY, |
174 |
|
|
& L_NBUFFER = L_BUFFERY ) |
175 |
|
|
|
176 |
|
|
C-- COMMON / EXCH_L / LOGICAL number common arrays for exchanges |
177 |
|
|
C exchNeedsMemSync - TRUE if memory sync. required to ensure |
178 |
|
|
C memory consistency during exchange |
179 |
|
|
C exchUsesBarrier - TRUE if we use a call to BAR to do sync. |
180 |
jmc |
1.7 |
C between processes. On some machines we wont |
181 |
|
|
C spin on the Ack setting ( the T90 ), |
182 |
cnh |
1.1 |
C instead we will use s system barrier. |
183 |
jmc |
1.7 |
C On the T90 the system barrier is very fast and |
184 |
|
|
C switches out the thread while it waits. On most |
185 |
|
|
C machines the system barrier is much too slow and if |
186 |
|
|
C we own the machine and have one thread per process |
187 |
cnh |
1.1 |
C preemption is not a problem. |
188 |
|
|
C exchCollectStatistics - Turns exchange statistics collecting on and off. |
189 |
|
|
|
190 |
|
|
COMMON / EXCH_L / exchNeedsMemSync, exchUsesBarrier, |
191 |
|
|
& exchCollectStatistics |
192 |
|
|
LOGICAL exchNeedsMemSync |
193 |
|
|
LOGICAL exchUsesBarrier |
194 |
|
|
LOGICAL exchCollectStatistics |
195 |
|
|
|
196 |
|
|
C-- COMMON / EXCH_R / REAL number common arrays for exchanges |
197 |
|
|
C xxxxSendBuf - Buffer used for sending data to another tile. |
198 |
|
|
C xxxxRecvBuf - Buffer used for receiving data from another tile. |
199 |
|
|
COMMON / EXCH_R / |
200 |
jmc |
1.7 |
& westSendBuf_RL, eastSendBuf_RL, |
201 |
cnh |
1.1 |
& southSendBuf_RL, northSendBuf_RL, |
202 |
jmc |
1.7 |
& westRecvBuf_RL, eastRecvBuf_RL, |
203 |
cnh |
1.1 |
& southRecvBuf_RL, northRecvBuf_RL, |
204 |
jmc |
1.7 |
& westSendBuf_RS, eastSendBuf_RS, |
205 |
cnh |
1.1 |
& southSendBuf_RS, northSendBuf_RS, |
206 |
jmc |
1.7 |
& westRecvBuf_RS, eastRecvBuf_RS, |
207 |
cnh |
1.5 |
& southRecvBuf_RS, northRecvBuf_RS, |
208 |
jmc |
1.7 |
& westSendBuf_R8, eastSendBuf_R8, |
209 |
cnh |
1.5 |
& southSendBuf_R8, northSendBuf_R8, |
210 |
jmc |
1.7 |
& westRecvBuf_R8, eastRecvBuf_R8, |
211 |
cnh |
1.5 |
& southRecvBuf_R8, northRecvBuf_R8, |
212 |
jmc |
1.7 |
& westSendBuf_R4, eastSendBuf_R4, |
213 |
cnh |
1.5 |
& southSendBuf_R4, northSendBuf_R4, |
214 |
jmc |
1.7 |
& westRecvBuf_R4, eastRecvBuf_R4, |
215 |
cnh |
1.5 |
& southRecvBuf_R4, northRecvBuf_R4 |
216 |
jmc |
1.7 |
_RL westSendBuf_RL( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
217 |
cnh |
1.1 |
& nSx, nSy ) |
218 |
jmc |
1.7 |
_RL eastSendBuf_RL( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
219 |
cnh |
1.1 |
& nSx, nSy ) |
220 |
jmc |
1.7 |
_RL southSendBuf_RL( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
221 |
cnh |
1.1 |
& nSx, nSy ) |
222 |
jmc |
1.7 |
_RL northSendBuf_RL( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
223 |
cnh |
1.1 |
& nSx, nSy ) |
224 |
jmc |
1.7 |
_RL westRecvBuf_RL( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
225 |
cnh |
1.1 |
& nSx, nSy ) |
226 |
jmc |
1.7 |
_RL eastRecvBuf_RL( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
227 |
cnh |
1.1 |
& nSx, nSy ) |
228 |
jmc |
1.7 |
_RL southRecvBuf_RL( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
229 |
cnh |
1.1 |
& nSx, nSy ) |
230 |
jmc |
1.7 |
_RL northRecvBuf_RL( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
231 |
cnh |
1.1 |
& nSx, nSy ) |
232 |
jmc |
1.7 |
_RS westSendBuf_RS( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
233 |
cnh |
1.1 |
& nSx, nSy ) |
234 |
jmc |
1.7 |
_RS eastSendBuf_RS( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
235 |
cnh |
1.1 |
& nSx, nSy ) |
236 |
jmc |
1.7 |
_RS southSendBuf_RS( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
237 |
cnh |
1.1 |
& nSx, nSy ) |
238 |
jmc |
1.7 |
_RS northSendBuf_RS( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
239 |
cnh |
1.1 |
& nSx, nSy ) |
240 |
jmc |
1.7 |
_RS westRecvBuf_RS( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
241 |
cnh |
1.1 |
& nSx, nSy ) |
242 |
jmc |
1.7 |
_RS eastRecvBuf_RS( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
243 |
cnh |
1.1 |
& nSx, nSy ) |
244 |
jmc |
1.7 |
_RS southRecvBuf_RS( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
245 |
cnh |
1.1 |
& nSx, nSy ) |
246 |
jmc |
1.7 |
_RS northRecvBuf_RS( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
247 |
cnh |
1.5 |
& nSx, nSy ) |
248 |
jmc |
1.7 |
_R8 westSendBuf_R8( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
249 |
cnh |
1.5 |
& nSx, nSy ) |
250 |
jmc |
1.7 |
_R8 eastSendBuf_R8( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
251 |
cnh |
1.5 |
& nSx, nSy ) |
252 |
jmc |
1.7 |
_R8 southSendBuf_R8( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
253 |
cnh |
1.5 |
& nSx, nSy ) |
254 |
jmc |
1.7 |
_R8 northSendBuf_R8( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
255 |
cnh |
1.5 |
& nSx, nSy ) |
256 |
jmc |
1.7 |
_R8 westRecvBuf_R8( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
257 |
cnh |
1.5 |
& nSx, nSy ) |
258 |
jmc |
1.7 |
_R8 eastRecvBuf_R8( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
259 |
cnh |
1.5 |
& nSx, nSy ) |
260 |
jmc |
1.7 |
_R8 southRecvBuf_R8( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
261 |
cnh |
1.5 |
& nSx, nSy ) |
262 |
jmc |
1.7 |
_R8 northRecvBuf_R8( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
263 |
cnh |
1.5 |
& nSx, nSy ) |
264 |
jmc |
1.7 |
_R4 westSendBuf_R4( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
265 |
cnh |
1.5 |
& nSx, nSy ) |
266 |
jmc |
1.7 |
_R4 eastSendBuf_R4( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
267 |
cnh |
1.5 |
& nSx, nSy ) |
268 |
jmc |
1.7 |
_R4 southSendBuf_R4( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
269 |
cnh |
1.5 |
& nSx, nSy ) |
270 |
jmc |
1.7 |
_R4 northSendBuf_R4( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
271 |
cnh |
1.5 |
& nSx, nSy ) |
272 |
jmc |
1.7 |
_R4 westRecvBuf_R4( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS, |
273 |
cnh |
1.5 |
& nSx, nSy ) |
274 |
jmc |
1.7 |
_R4 eastRecvBuf_R4( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS, |
275 |
cnh |
1.5 |
& nSx, nSy ) |
276 |
jmc |
1.7 |
_R4 southRecvBuf_R4( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS, |
277 |
cnh |
1.5 |
& nSx, nSy ) |
278 |
jmc |
1.7 |
_R4 northRecvBuf_R4( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS, |
279 |
cnh |
1.1 |
& nSx, nSy ) |
280 |
|
|
|
281 |
|
|
C-- COMMON / EXCH_I / INTEGER common arrays for exchanges |
282 |
|
|
C xxxxSendAck - Flag indicating ready to send data. |
283 |
|
|
C xxxxRecvAck - Falg indicating receive data is ready. |
284 |
|
|
C exchBufferLevel - Current cyclic buffer level. |
285 |
|
|
C exchNReqsX, exchNReqsY - Pending message counts |
286 |
|
|
C exchReqIdX, exchReqIdY -Pending message identifiers |
287 |
|
|
C *Spin* - Exchange statistics holder |
288 |
|
|
C Count - No. spins for each thread |
289 |
|
|
C Max - Maximum spins for an exchange |
290 |
|
|
C Min - Minimum spins for an exchange |
291 |
|
|
COMMON / EXCH_I / |
292 |
|
|
& westSendAck, eastSendAck, southSendAck, northSendAck, |
293 |
|
|
& westRecvAck, eastRecvAck, southRecvAck, northRecvAck, |
294 |
|
|
& exchangeBufLevel, |
295 |
|
|
& exchNReqsX, exchNReqsY, exchReqIdX, exchReqIdY, |
296 |
|
|
& exchRecvXSpinCount, exchRecvXSpinMax, exchRecvXSpinMin, |
297 |
|
|
& exchRecvXExchCount, |
298 |
|
|
& exchRecvYSpinCount, exchRecvYSpinMax, exchRecvYSpinMin, |
299 |
|
|
& exchRecvYExchCount |
300 |
jmc |
1.7 |
INTEGER westSendAck( NUMBER_OF_BUFFER_LEVELS, |
301 |
cnh |
1.1 |
& nSx, nSy ) |
302 |
jmc |
1.7 |
INTEGER eastSendAck( NUMBER_OF_BUFFER_LEVELS, |
303 |
cnh |
1.1 |
& nSx, nSy ) |
304 |
jmc |
1.7 |
INTEGER southSendAck( NUMBER_OF_BUFFER_LEVELS, |
305 |
cnh |
1.1 |
& nSx, nSy ) |
306 |
jmc |
1.7 |
INTEGER northSendAck( NUMBER_OF_BUFFER_LEVELS, |
307 |
cnh |
1.1 |
& nSx, nSy ) |
308 |
jmc |
1.7 |
INTEGER westRecvAck( NUMBER_OF_BUFFER_LEVELS, |
309 |
cnh |
1.1 |
& nSx, nSy ) |
310 |
jmc |
1.7 |
INTEGER eastRecvAck( NUMBER_OF_BUFFER_LEVELS, |
311 |
cnh |
1.1 |
& nSx, nSy ) |
312 |
jmc |
1.7 |
INTEGER southRecvAck( NUMBER_OF_BUFFER_LEVELS, |
313 |
cnh |
1.1 |
& nSx, nSy ) |
314 |
jmc |
1.7 |
INTEGER northRecvAck( NUMBER_OF_BUFFER_LEVELS, |
315 |
cnh |
1.1 |
& nSx, nSy ) |
316 |
|
|
INTEGER exchangeBufLevel( cacheLineSize/4, nSx, nSy ) |
317 |
|
|
INTEGER exchNReqsX(cacheLineSize/4,nSx,nSy) |
318 |
|
|
INTEGER exchNReqsY(cacheLineSize/4,nSx,nSy) |
319 |
|
|
INTEGER exchReqIdX(2*nSx+2*nSy,cacheLineSize/4,nSx,nSy) |
320 |
|
|
INTEGER exchReqIdY(2*nSx+2*nSy,cacheLineSize/4,nSx,nSy) |
321 |
|
|
INTEGER exchRecvXSpinCount(cacheLineSize/4, nSx, nSy) |
322 |
|
|
INTEGER exchRecvXExchCount(cacheLineSize/4, nSx, nSy) |
323 |
|
|
INTEGER exchRecvXSpinMax (cacheLineSize/4, nSx, nSy) |
324 |
|
|
INTEGER exchRecvXSpinMin (cacheLineSize/4, nSx, nSy) |
325 |
|
|
INTEGER exchRecvYSpinCount(cacheLineSize/4, nSx, nSy) |
326 |
|
|
INTEGER exchRecvYExchCount(cacheLineSize/4, nSx, nSy) |
327 |
|
|
INTEGER exchRecvYSpinMax (cacheLineSize/4, nSx, nSy) |
328 |
|
|
INTEGER exchRecvYSpinMin (cacheLineSize/4, nSx, nSy) |
329 |
|
|
|