1 |
SUBROUTINE EXCH_XY_R8( arr ) |
2 |
|
3 |
C == Global variables == |
4 |
#include "SIZE.h" |
5 |
#include "EEPARAMS.h" |
6 |
#include "EXCH.h" |
7 |
|
8 |
#ifdef ALLOW_MPI |
9 |
#include "mpif.h" |
10 |
#endif |
11 |
|
12 |
#include "MPI_INFO.h" |
13 |
#include "JAM_INFO.h" |
14 |
|
15 |
C == Routine arguments == |
16 |
Real*8 arr(1-OLx:sNx+OLx,1-OLy:sNy+OLy) |
17 |
|
18 |
C == Local variables == |
19 |
INTEGER I, J |
20 |
INTEGER northProc, southProc |
21 |
INTEGER farProc1, farProc2 |
22 |
INTEGER toPid, fromPid |
23 |
INTEGER rc |
24 |
|
25 |
INTEGER myFourWayRank, exchangePhase |
26 |
|
27 |
#ifdef ALLOW_MPI |
28 |
INTEGER mpiStatus(MPI_STATUS_SIZE) |
29 |
#endif |
30 |
|
31 |
C East-west halo update (without corners) |
32 |
DO J=1,sNy |
33 |
DO I=1,OLx |
34 |
arr(1-I ,J) = arr(sNx-I+1,J) |
35 |
arr(sNx+I,J) = arr(1+I-1 ,J) |
36 |
ENDDO |
37 |
ENDDO |
38 |
|
39 |
C Phase 1 pairing |
40 |
C | 0 | ---> | 1 | |
41 |
C | 0 | <--- | 1 | |
42 |
|
43 |
C | 2 | ---> | 3 | |
44 |
C | 2 | <--- | 3 | |
45 |
|
46 |
C | 4 | ---> | 5 | |
47 |
C | 4 | <--- | 5 | |
48 |
|
49 |
C etc ... |
50 |
C |
51 |
|
52 |
#ifdef USE_MPI_EXCH |
53 |
C North-south halo update (without corners) |
54 |
C Put my edges into a buffers |
55 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
56 |
DO I=1,sNx |
57 |
exchBuf1(I) = arr(I,sNy) |
58 |
exchBuf2(I) = arr(I,1 ) |
59 |
ENDDO |
60 |
ELSE |
61 |
DO I=1,sNx |
62 |
exchBuf1(I) = arr(I,1 ) |
63 |
exchBuf2(I) = arr(I,sNy) |
64 |
ENDDO |
65 |
ENDIF |
66 |
|
67 |
C Exchange the buffers |
68 |
northProc = mpi_northId |
69 |
southProc = mpi_southId |
70 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
71 |
farProc1 = northProc |
72 |
farProc2 = southProc |
73 |
ELSE |
74 |
farProc1 = southProc |
75 |
farProc2 = northProc |
76 |
ENDIF |
77 |
C Even-odd pairs |
78 |
IF ( farProc1 .NE. myProcId ) THEN |
79 |
CALL MPI_Sendrecv_replace(exchBuf1,sNx,MPI_DOUBLE_PRECISION, |
80 |
& farProc1,0, |
81 |
& farProc1,MPI_ANY_TAG, |
82 |
& MPI_COMM_WORLD,mpiStatus, |
83 |
& rc) |
84 |
ENDIF |
85 |
C Odd-even pairs |
86 |
IF ( farProc2 .NE. myProcId ) THEN |
87 |
CALL MPI_Sendrecv_replace(exchBuf2,sNx,MPI_DOUBLE_PRECISION, |
88 |
& farProc2,0, |
89 |
& farProc2,MPI_ANY_TAG, |
90 |
& MPI_COMM_WORLD,mpiStatus, |
91 |
& rc) |
92 |
ENDIF |
93 |
#endif |
94 |
|
95 |
#ifdef USE_JAM_EXCH |
96 |
|
97 |
myFourWayRank = MOD(myProcId,4) |
98 |
|
99 |
northProc = jam_northId |
100 |
southProc = jam_southId |
101 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
102 |
C sendBuf1 = &arr(I,sNy ) |
103 |
C recvBuf1 = &arr(I,sNy+1) |
104 |
C sendBuf2 = &arr(I,1 ) |
105 |
C recvBuf2 = &arr(I,0 ) |
106 |
farProc1 = northProc |
107 |
farProc2 = southProc |
108 |
IF ( farProc1 .NE. myProcId ) THEN |
109 |
CALL JAM_EXCHANGE(farProc1,arr(I,sNy),arr(I,sNy+1),sNx*8,jam_exchKey) |
110 |
jam_exchKey = jam_exchKey+1 |
111 |
ENDIF |
112 |
|
113 |
10 CONTINUE |
114 |
CALL JAM_EXCHANGE_TEST( exchangePhase ) |
115 |
IF ( myFourWayRank .EQ. 0 ) THEN |
116 |
IF ( exchangePhase .EQ. 0 ) GOTO 11 |
117 |
ELSE |
118 |
IF ( exchangePhase .EQ. 1 ) GOTO 11 |
119 |
ENDIF |
120 |
GOTO 10 |
121 |
11 CONTINUE |
122 |
|
123 |
IF ( farProc2 .NE. myProcId ) THEN |
124 |
CALL JAM_EXCHANGE(farProc2,arr(I,1),arr(I,0),sNx*8,jam_exchKey) |
125 |
jam_exchKey = jam_exchKey+1 |
126 |
ENDIF |
127 |
|
128 |
CALL JAM_EXCHANGE_MARK |
129 |
|
130 |
ELSE |
131 |
C sendBuf1 = &arr(I,1 ) |
132 |
C recvBuf1 = &arr(I,0 ) |
133 |
C sendBuf2 = &arr(I,sNy ) |
134 |
C recvBuf2 = &arr(I,sNy+1) |
135 |
farProc1 = southProc |
136 |
farProc2 = northProc |
137 |
IF ( farProc1 .NE. myProcId ) THEN |
138 |
CALL JAM_EXCHANGE(farProc1,arr(I,1),arr(I,0),sNx*8,jam_exchKey) |
139 |
jam_exchKey = jam_exchKey+1 |
140 |
ENDIF |
141 |
|
142 |
20 CONTINUE |
143 |
CALL JAM_EXCHANGE_TEST( exchangePhase ) |
144 |
IF ( myFourWayRank .EQ. 3 ) THEN |
145 |
IF ( exchangePhase .EQ. 0 ) GOTO 21 |
146 |
ELSE |
147 |
IF ( exchangePhase .EQ. 1 ) GOTO 21 |
148 |
ENDIF |
149 |
GOTO 20 |
150 |
21 CONTINUE |
151 |
|
152 |
IF ( farProc2 .NE. myProcId ) THEN |
153 |
CALL JAM_EXCHANGE(farProc2,arr(I,sNy),arr(I,sNy+1),sNx*8,jam_exchKey) |
154 |
jam_exchKey = jam_exchKey+1 |
155 |
ENDIF |
156 |
|
157 |
CALL JAM_EXCHANGE_MARK |
158 |
|
159 |
ENDIF |
160 |
#endif |
161 |
|
162 |
#ifdef USE_MPI_EXCH |
163 |
C Fill overlap regions from the buffers |
164 |
IF ( MOD(myProcId,2) .EQ. 0 ) THEN |
165 |
DO I=1,sNx |
166 |
arr(I,sNy+1) = exchBuf1(I) |
167 |
arr(I,0 ) = exchBuf2(I) |
168 |
ENDDO |
169 |
ELSE |
170 |
DO I=1,sNx |
171 |
arr(I,sNy+1) = exchBuf2(I) |
172 |
arr(I,0 ) = exchBuf1(I) |
173 |
ENDDO |
174 |
ENDIF |
175 |
#endif |
176 |
|
177 |
IF ( numberOfProcs .EQ. 1 ) THEN |
178 |
DO I=1,sNx |
179 |
arr(I,sNy+1) = arr(I,1 ) |
180 |
arr(I,0 ) = arr(I,sNy) |
181 |
ENDDO |
182 |
ENDIF |
183 |
|
184 |
RETURN |
185 |
END |