1 |
C $Id$ |
2 |
SUBROUTINE EXCH_XY_R8( arr ) |
3 |
|
4 |
C == Global variables == |
5 |
#include "SIZE.h" |
6 |
#include "EEPARAMS.h" |
7 |
#include "EXCH.h" |
8 |
|
9 |
#ifdef ALLOW_MPI |
10 |
#include "mpif.h" |
11 |
#include "MPI_INFO.h" |
12 |
#endif |
13 |
|
14 |
#include "JAM_INFO.h" |
15 |
|
16 |
C == Routine arguments == |
17 |
Real arr(1-OLx:sNx+OLx,1-OLy:sNy+OLy) |
18 |
|
19 |
C == Local variables == |
20 |
INTEGER I, J |
21 |
INTEGER northProc, southProc |
22 |
#ifdef DECOMP2D |
23 |
INTEGER eastProc, westProc |
24 |
#endif |
25 |
INTEGER farProc1, farProc2 |
26 |
INTEGER toPid, fromPid |
27 |
INTEGER rc |
28 |
|
29 |
INTEGER myFourWayRank, exchangePhase |
30 |
|
31 |
#ifdef ALLOW_MPI |
32 |
INTEGER mpiReq(8) |
33 |
INTEGER mpiStat(MPI_STATUS_SIZE,8) |
34 |
INTEGER mpiStatus(MPI_STATUS_SIZE) |
35 |
#endif |
36 |
|
37 |
#ifndef DECOMP2D |
38 |
C East-west halo update (without corners) |
39 |
DO J=1,sNy |
40 |
DO I=1,OLx |
41 |
arr(1-I ,J) = arr(sNx-I+1,J) |
42 |
arr(sNx+I,J) = arr(1+I-1 ,J) |
43 |
ENDDO |
44 |
ENDDO |
45 |
#endif |
46 |
|
47 |
C Phase 1 pairing |
48 |
C | 0 | ---> | 1 | |
49 |
C | 0 | <--- | 1 | |
50 |
|
51 |
C | 2 | ---> | 3 | |
52 |
C | 2 | <--- | 3 | |
53 |
|
54 |
C | 4 | ---> | 5 | |
55 |
C | 4 | <--- | 5 | |
56 |
|
57 |
C etc ... |
58 |
C |
59 |
|
60 |
#ifdef USE_MPI_EXCH |
61 |
#ifdef DECOMP2D |
62 |
C East-West exchanges |
63 |
#ifdef USE_SNDRCV |
64 |
C Send West, receive from East |
65 |
CALL MPI_Sendrecv(arr(1,1), 1, ewslice, mpi_westId, 100, |
66 |
$ arr(sNx+1,1), 1, ewslice, mpi_eastId, 100, comm_use, |
67 |
$ mpiStatus, rc) |
68 |
C Send East, receive from West |
69 |
CALL MPI_Sendrecv(arr(sNx-OLx+1,1), 1, ewslice, mpi_eastId, 200, |
70 |
$ arr(1-OLx,1), 1, ewslice, mpi_westId, 200, comm_use, |
71 |
$ mpiStatus,rc) |
72 |
|
73 |
C North-South exchanges |
74 |
|
75 |
C Send South, receive from North |
76 |
CALL MPI_Sendrecv(arr(1,1), 1, nsslice, mpi_southId, 300, |
77 |
$ arr(1,sNy+1), 1, nsslice, mpi_northId, 300, comm_use, |
78 |
$ mpiStatus, rc) |
79 |
C Send North, receive from South |
80 |
CALL MPI_Sendrecv(arr(1,sNy-OLy+1), 1, nsslice, mpi_northId, 400, |
81 |
$ arr(1,1-OLy), 1, nsslice, mpi_southId, 400, comm_use, |
82 |
$ mpiStatus,rc) |
83 |
#else |
84 |
C Send West, receive from East |
85 |
CALL MPI_Isend(arr(1,1), 1, ewslice, mpi_westId, 100, |
86 |
$ comm_use, mpiReq(1), rc) |
87 |
CALL MPI_Irecv(arr(sNx+1,1), 1, ewslice, mpi_eastId, 100, |
88 |
$ comm_use, mpiReq(2), rc) |
89 |
C Send East, receive from West |
90 |
CALL MPI_Isend(arr(sNx-OLx+1,1), 1, ewslice, mpi_eastId, 200, |
91 |
$ comm_use, mpiReq(3), rc) |
92 |
CALL MPI_Irecv(arr(1-OLx,1), 1, ewslice, mpi_westId, 200, |
93 |
$ comm_use, mpiReq(4),rc) |
94 |
|
95 |
C North-South exchanges |
96 |
|
97 |
C Send South, receive from North |
98 |
CALL MPI_Isend(arr(1,1), 1, nsslice, mpi_southId, 300, |
99 |
$ comm_use, mpiReq(5), rc) |
100 |
CALL MPI_Irecv(arr(1,sNy+1), 1, nsslice, mpi_northId, 300, |
101 |
$ comm_use, mpiReq(6), rc) |
102 |
C Send North, receive from South |
103 |
CALL MPI_Isend(arr(1,sNy-OLy+1), 1, nsslice, mpi_northId, 400, |
104 |
$ comm_use, mpiReq(7), rc) |
105 |
CALL MPI_Irecv(arr(1,1-OLy), 1, nsslice, mpi_southId, 400, |
106 |
$ comm_use, mpiReq(8),rc) |
107 |
|
108 |
CALL MPI_Waitall(8, mpiReq, mpiStat, rc) |
109 |
|
110 |
#endif |
111 |
|
112 |
#else |
113 |
C North-south halo update (without corners) |
114 |
C Put my edges into a buffers |
115 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
116 |
DO I=1,sNx |
117 |
exchBuf1(I) = arr(I,sNy) |
118 |
exchBuf2(I) = arr(I,1 ) |
119 |
ENDDO |
120 |
ELSE |
121 |
DO I=1,sNx |
122 |
exchBuf1(I) = arr(I,1 ) |
123 |
exchBuf2(I) = arr(I,sNy) |
124 |
ENDDO |
125 |
ENDIF |
126 |
|
127 |
C Exchange the buffers |
128 |
northProc = mpi_northId |
129 |
southProc = mpi_southId |
130 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
131 |
farProc1 = northProc |
132 |
farProc2 = southProc |
133 |
ELSE |
134 |
farProc1 = southProc |
135 |
farProc2 = northProc |
136 |
ENDIF |
137 |
C Even-odd pairs |
138 |
IF ( farProc1 .NE. myProcId ) THEN |
139 |
CALL MPI_Sendrecv_replace(exchBuf1,sNx,_MPI_TYPE_REAL, |
140 |
& farProc1,0, |
141 |
& farProc1,MPI_ANY_TAG, |
142 |
& comm_use,mpiStatus, |
143 |
& rc) |
144 |
ENDIF |
145 |
C Odd-even pairs |
146 |
IF ( farProc2 .NE. myProcId ) THEN |
147 |
CALL MPI_Sendrecv_replace(exchBuf2,sNx,_MPI_TYPE_REAL, |
148 |
& farProc2,0, |
149 |
& farProc2,MPI_ANY_TAG, |
150 |
& comm_use,mpiStatus, |
151 |
& rc) |
152 |
ENDIF |
153 |
#endif |
154 |
#endif |
155 |
|
156 |
#ifdef USE_JAM_EXCH |
157 |
|
158 |
myFourWayRank = MOD(myProcId,4) |
159 |
|
160 |
northProc = jam_northId |
161 |
southProc = jam_southId |
162 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
163 |
C sendBuf1 = &arr(I,sNy ) |
164 |
C recvBuf1 = &arr(I,sNy+1) |
165 |
C sendBuf2 = &arr(I,1 ) |
166 |
C recvBuf2 = &arr(I,0 ) |
167 |
farProc1 = northProc |
168 |
farProc2 = southProc |
169 |
IF ( farProc1 .NE. myProcId ) THEN |
170 |
CALL JAM_EXCHANGE(farProc1,arr(I,sNy),arr(I,sNy+1),sNx*8 |
171 |
$ ,jam_exchKey) |
172 |
jam_exchKey = jam_exchKey+1 |
173 |
ENDIF |
174 |
|
175 |
10 CONTINUE |
176 |
CALL JAM_EXCHANGE_TEST( exchangePhase ) |
177 |
IF ( myFourWayRank .EQ. 0 ) THEN |
178 |
IF ( exchangePhase .EQ. 0 ) GOTO 11 |
179 |
ELSE |
180 |
IF ( exchangePhase .EQ. 1 ) GOTO 11 |
181 |
ENDIF |
182 |
GOTO 10 |
183 |
11 CONTINUE |
184 |
|
185 |
IF ( farProc2 .NE. myProcId ) THEN |
186 |
CALL JAM_EXCHANGE(farProc2,arr(I,1),arr(I,0),sNx*8,jam_exchKey) |
187 |
jam_exchKey = jam_exchKey+1 |
188 |
ENDIF |
189 |
|
190 |
CALL JAM_EXCHANGE_MARK |
191 |
|
192 |
ELSE |
193 |
C sendBuf1 = &arr(I,1 ) |
194 |
C recvBuf1 = &arr(I,0 ) |
195 |
C sendBuf2 = &arr(I,sNy ) |
196 |
C recvBuf2 = &arr(I,sNy+1) |
197 |
farProc1 = southProc |
198 |
farProc2 = northProc |
199 |
IF ( farProc1 .NE. myProcId ) THEN |
200 |
CALL JAM_EXCHANGE(farProc1,arr(I,1),arr(I,0),sNx*8,jam_exchKey) |
201 |
jam_exchKey = jam_exchKey+1 |
202 |
ENDIF |
203 |
|
204 |
20 CONTINUE |
205 |
CALL JAM_EXCHANGE_TEST( exchangePhase ) |
206 |
IF ( myFourWayRank .EQ. 3 ) THEN |
207 |
IF ( exchangePhase .EQ. 0 ) GOTO 21 |
208 |
ELSE |
209 |
IF ( exchangePhase .EQ. 1 ) GOTO 21 |
210 |
ENDIF |
211 |
GOTO 20 |
212 |
21 CONTINUE |
213 |
|
214 |
IF ( farProc2 .NE. myProcId ) THEN |
215 |
CALL JAM_EXCHANGE(farProc2,arr(I,sNy),arr(I,sNy+1),sNx*8 |
216 |
$ ,jam_exchKey) |
217 |
jam_exchKey = jam_exchKey+1 |
218 |
ENDIF |
219 |
|
220 |
CALL JAM_EXCHANGE_MARK |
221 |
|
222 |
ENDIF |
223 |
#endif |
224 |
|
225 |
#if defined(USE_MPI_EXCH) && !defined(DECOMP2D) |
226 |
C Fill overlap regions from the buffers |
227 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
228 |
DO I=1,sNx |
229 |
arr(I,sNy+1) = exchBuf1(I) |
230 |
arr(I,0 ) = exchBuf2(I) |
231 |
ENDDO |
232 |
ELSE |
233 |
DO I=1,sNx |
234 |
arr(I,sNy+1) = exchBuf2(I) |
235 |
arr(I,0 ) = exchBuf1(I) |
236 |
ENDDO |
237 |
ENDIF |
238 |
#endif |
239 |
|
240 |
IF ( numberOfProcs .EQ. 1 ) THEN |
241 |
DO I=1,sNx |
242 |
arr(I,sNy+1) = arr(I,1 ) |
243 |
arr(I,0 ) = arr(I,sNy) |
244 |
ENDDO |
245 |
ENDIF |
246 |
|
247 |
RETURN |
248 |
END |