| 76 |
INTEGER eastCommMode |
INTEGER eastCommMode |
| 77 |
INTEGER spinCount |
INTEGER spinCount |
| 78 |
#ifdef ALLOW_USE_MPI |
#ifdef ALLOW_USE_MPI |
| 79 |
INTEGER theProc, theTag, theType, theSize |
INTEGER theProc, theTag, theType, theSize, pReqI |
| 80 |
INTEGER mpiStatus(MPI_STATUS_SIZE,4), mpiRc |
INTEGER mpiStatus(MPI_STATUS_SIZE,4), mpiRc |
| 81 |
#endif |
#endif |
| 82 |
CEOP |
CEOP |
| 83 |
|
|
| 84 |
|
INTEGER myBxLoSave(MAX_NO_THREADS) |
| 85 |
|
INTEGER myBxHiSave(MAX_NO_THREADS) |
| 86 |
|
INTEGER myByLoSave(MAX_NO_THREADS) |
| 87 |
|
INTEGER myByHiSave(MAX_NO_THREADS) |
| 88 |
|
LOGICAL doingSingleThreadedComms |
| 89 |
|
|
| 90 |
|
doingSingleThreadedComms = .FALSE. |
| 91 |
|
#ifdef ALLOW_USE_MPI |
| 92 |
|
#ifndef ALWAYS_USE_MPI |
| 93 |
|
IF ( usingMPI ) THEN |
| 94 |
|
#endif |
| 95 |
|
C Set default behavior to have MPI comms done by a single thread. |
| 96 |
|
C Most MPI implementations don't support concurrent comms from |
| 97 |
|
C several threads. |
| 98 |
|
IF ( nThreads .GT. 1 ) THEN |
| 99 |
|
_BARRIER |
| 100 |
|
_BEGIN_MASTER( myThid ) |
| 101 |
|
DO I=1,nThreads |
| 102 |
|
myBxLoSave(I) = myBxLo(I) |
| 103 |
|
myBxHiSave(I) = myBxHi(I) |
| 104 |
|
myByLoSave(I) = myByLo(I) |
| 105 |
|
myByHiSave(I) = myByHi(I) |
| 106 |
|
ENDDO |
| 107 |
|
C Comment out loop below and myB[xy][Lo|Hi](1) settings below |
| 108 |
|
C if you want to get multi-threaded MPI comms. |
| 109 |
|
DO I=1,nThreads |
| 110 |
|
myBxLo(I) = 0 |
| 111 |
|
myBxHi(I) = -1 |
| 112 |
|
myByLo(I) = 0 |
| 113 |
|
myByHi(I) = -1 |
| 114 |
|
ENDDO |
| 115 |
|
myBxLo(1) = 1 |
| 116 |
|
myBxHi(1) = nSx |
| 117 |
|
myByLo(1) = 1 |
| 118 |
|
myByHi(1) = nSy |
| 119 |
|
doingSingleThreadedComms = .TRUE. |
| 120 |
|
_END_MASTER( myThid ) |
| 121 |
|
_BARRIER |
| 122 |
|
ENDIF |
| 123 |
|
#ifndef ALWAYS_USE_MPI |
| 124 |
|
ENDIF |
| 125 |
|
#endif |
| 126 |
|
#endif |
| 127 |
|
|
| 128 |
C-- Under a "put" scenario we |
C-- Under a "put" scenario we |
| 129 |
C-- i. set completetion signal for buffer we put into. |
C-- i. set completetion signal for buffer we put into. |
| 154 |
theTag = _tileTagRecvW(bi,bj) |
theTag = _tileTagRecvW(bi,bj) |
| 155 |
theType = _MPI_TYPE_RX |
theType = _MPI_TYPE_RX |
| 156 |
theSize = sNy*exchWidthX*myNz |
theSize = sNy*exchWidthX*myNz |
| 157 |
|
# ifndef ALLOW_AUTODIFF_OPENAD |
| 158 |
CALL MPI_Recv( westRecvBuf_RX(1,eBl,bi,bj), theSize, theType, |
CALL MPI_Recv( westRecvBuf_RX(1,eBl,bi,bj), theSize, theType, |
| 159 |
& theProc, theTag, MPI_COMM_MODEL, |
& theProc, theTag, MPI_COMM_MODEL, |
| 160 |
& mpiStatus, mpiRc ) |
& mpiStatus, mpiRc ) |
| 161 |
|
# else |
| 162 |
|
pReqI=exchNReqsX(1,bi,bj)+1 |
| 163 |
|
CALL ampi_recv_RX( |
| 164 |
|
& westRecvBuf_RX(1,eBl,bi,bj) , |
| 165 |
|
& theSize , |
| 166 |
|
& theType , |
| 167 |
|
& theProc , |
| 168 |
|
& theTag , |
| 169 |
|
& MPI_COMM_MODEL , |
| 170 |
|
& exchReqIdX(pReqI,1,bi,bj), |
| 171 |
|
& exchNReqsX(1,bi,bj), |
| 172 |
|
& mpiStatus , |
| 173 |
|
& mpiRc ) |
| 174 |
|
# endif /* ALLOW_AUTODIFF_OPENAD */ |
| 175 |
#ifndef ALWAYS_USE_MPI |
#ifndef ALWAYS_USE_MPI |
| 176 |
ENDIF |
ENDIF |
| 177 |
#endif |
#endif |
| 186 |
theTag = _tileTagRecvE(bi,bj) |
theTag = _tileTagRecvE(bi,bj) |
| 187 |
theType = _MPI_TYPE_RX |
theType = _MPI_TYPE_RX |
| 188 |
theSize = sNy*exchWidthX*myNz |
theSize = sNy*exchWidthX*myNz |
| 189 |
|
# ifndef ALLOW_AUTODIFF_OPENAD |
| 190 |
CALL MPI_Recv( eastRecvBuf_RX(1,eBl,bi,bj), theSize, theType, |
CALL MPI_Recv( eastRecvBuf_RX(1,eBl,bi,bj), theSize, theType, |
| 191 |
& theProc, theTag, MPI_COMM_MODEL, |
& theProc, theTag, MPI_COMM_MODEL, |
| 192 |
& mpiStatus, mpiRc ) |
& mpiStatus, mpiRc ) |
| 193 |
|
# else |
| 194 |
|
pReqI=exchNReqsX(1,bi,bj)+1 |
| 195 |
|
CALL ampi_recv_RX( |
| 196 |
|
& eastRecvBuf_RX(1,eBl,bi,bj) , |
| 197 |
|
& theSize , |
| 198 |
|
& theType , |
| 199 |
|
& theProc , |
| 200 |
|
& theTag , |
| 201 |
|
& MPI_COMM_MODEL , |
| 202 |
|
& exchReqIdX(pReqI,1,bi,bj), |
| 203 |
|
& exchNReqsX(1,bi,bj), |
| 204 |
|
& mpiStatus , |
| 205 |
|
& mpiRc ) |
| 206 |
|
# endif /* ALLOW_AUTODIFF_OPENAD */ |
| 207 |
#ifndef ALWAYS_USE_MPI |
#ifndef ALWAYS_USE_MPI |
| 208 |
ENDIF |
ENDIF |
| 209 |
#endif |
#endif |
| 225 |
ebL = exchangeBufLevel(1,bi,bj) |
ebL = exchangeBufLevel(1,bi,bj) |
| 226 |
westCommMode = _tileCommModeW(bi,bj) |
westCommMode = _tileCommModeW(bi,bj) |
| 227 |
eastCommMode = _tileCommModeE(bi,bj) |
eastCommMode = _tileCommModeE(bi,bj) |
| 228 |
|
# ifndef ALLOW_AUTODIFF_OPENAD |
| 229 |
10 CONTINUE |
10 CONTINUE |
| 230 |
CALL FOOL_THE_COMPILER( spinCount ) |
CALL FOOL_THE_COMPILER( spinCount ) |
| 231 |
spinCount = spinCount+1 |
spinCount = spinCount+1 |
| 233 |
C WRITE(*,*) ' eBl = ', ebl |
C WRITE(*,*) ' eBl = ', ebl |
| 234 |
C STOP ' S/R EXCH_RECV_GET_X: spinCount .GT. _EXCH_SPIN_LIMIT' |
C STOP ' S/R EXCH_RECV_GET_X: spinCount .GT. _EXCH_SPIN_LIMIT' |
| 235 |
C ENDIF |
C ENDIF |
| 236 |
IF ( westRecvAck(eBl,bi,bj) .EQ. 0. ) GOTO 10 |
IF ( westRecvAck(eBl,bi,bj) .EQ. 0 ) GOTO 10 |
| 237 |
IF ( eastRecvAck(eBl,bi,bj) .EQ. 0. ) GOTO 10 |
IF ( eastRecvAck(eBl,bi,bj) .EQ. 0 ) GOTO 10 |
| 238 |
|
# else |
| 239 |
|
do while ((westRecvAck(eBl,bi,bj) .EQ. 0. |
| 240 |
|
& .or. |
| 241 |
|
& eastRecvAck(eBl,bi,bj) .EQ. 0. )) |
| 242 |
|
CALL FOOL_THE_COMPILER( spinCount ) |
| 243 |
|
spinCount = spinCount+1 |
| 244 |
|
end do |
| 245 |
|
# endif /* ALLOW_AUTODIFF_OPENAD */ |
| 246 |
C Clear outstanding requests |
C Clear outstanding requests |
| 247 |
westRecvAck(eBl,bi,bj) = 0. |
westRecvAck(eBl,bi,bj) = 0 |
| 248 |
eastRecvAck(eBl,bi,bj) = 0. |
eastRecvAck(eBl,bi,bj) = 0 |
| 249 |
|
|
| 250 |
IF ( exchNReqsX(1,bi,bj) .GT. 0 ) THEN |
IF ( exchNReqsX(1,bi,bj) .GT. 0 ) THEN |
| 251 |
#ifdef ALLOW_USE_MPI |
#ifdef ALLOW_USE_MPI |
| 252 |
#ifndef ALWAYS_USE_MPI |
#ifndef ALWAYS_USE_MPI |
| 253 |
IF ( usingMPI ) THEN |
IF ( usingMPI ) THEN |
| 254 |
#endif |
#endif |
| 255 |
|
# ifndef ALLOW_AUTODIFF_OPENAD |
| 256 |
CALL MPI_Waitall( exchNReqsX(1,bi,bj), exchReqIdX(1,1,bi,bj), |
CALL MPI_Waitall( exchNReqsX(1,bi,bj), exchReqIdX(1,1,bi,bj), |
| 257 |
& mpiStatus, mpiRC ) |
& mpiStatus, mpiRC ) |
| 258 |
|
# else |
| 259 |
|
CALL ampi_waitall( |
| 260 |
|
& exchNReqsX(1,bi,bj), |
| 261 |
|
& exchReqIdX(1,1,bi,bj), |
| 262 |
|
& mpiStatus, |
| 263 |
|
& mpiRC ) |
| 264 |
|
# endif /* ALLOW_AUTODIFF_OPENAD */ |
| 265 |
#ifndef ALWAYS_USE_MPI |
#ifndef ALWAYS_USE_MPI |
| 266 |
ENDIF |
ENDIF |
| 267 |
#endif |
#endif |
| 408 |
ENDDO |
ENDDO |
| 409 |
ENDDO |
ENDDO |
| 410 |
|
|
| 411 |
|
_BARRIER |
| 412 |
|
IF ( doingSingleThreadedComms ) THEN |
| 413 |
|
C Restore saved settings that were stored to allow |
| 414 |
|
C single thred comms. |
| 415 |
|
_BEGIN_MASTER(myThid) |
| 416 |
|
DO I=1,nThreads |
| 417 |
|
myBxLo(I) = myBxLoSave(I) |
| 418 |
|
myBxHi(I) = myBxHiSave(I) |
| 419 |
|
myByLo(I) = myByLoSave(I) |
| 420 |
|
myByHi(I) = myByHiSave(I) |
| 421 |
|
ENDDO |
| 422 |
|
_END_MASTER(myThid) |
| 423 |
|
ENDIF |
| 424 |
|
_BARRIER |
| 425 |
|
|
| 426 |
RETURN |
RETURN |
| 427 |
END |
END |