| 1 |
C $Header: /u/gcmpack/MITgcm_contrib/dgoldberg/code_cg3d_petsc/cg3d_petsc_numerate.F,v 1.2 2016/07/02 17:48:26 dgoldberg Exp $ |
| 2 |
C $Name: $ |
| 3 |
|
| 4 |
#include "CPP_OPTIONS.h" |
| 5 |
|
| 6 |
C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----| |
| 7 |
|
| 8 |
CBOP |
| 9 |
SUBROUTINE CG3D_PETSC_NUMERATE (myThid) |
| 10 |
|
| 11 |
C /============================================================\ |
| 12 |
C | SUBROUTINE | |
| 13 |
C | o | |
| 14 |
C |============================================================| |
| 15 |
C | | |
| 16 |
C \============================================================/ |
| 17 |
IMPLICIT NONE |
| 18 |
|
| 19 |
C === Global variables === |
| 20 |
#include "SIZE.h" |
| 21 |
#include "GRID.h" |
| 22 |
!#include "SURFACE.h" |
| 23 |
#include "EEPARAMS.h" |
| 24 |
#include "PARAMS.h" |
| 25 |
!#ifdef ALLOW_PETSC |
| 26 |
!#include "CG3D_PETSC.h" |
| 27 |
! UNCOMMENT IF V3.0 |
| 28 |
!#include "finclude/petscvec.h" |
| 29 |
!#include "finclude/petscmat.h" |
| 30 |
!#include "finclude/petscksp.h" |
| 31 |
!#include "finclude/petscpc.h" |
| 32 |
!#endif |
| 33 |
#include "CG3D.h" |
| 34 |
#ifdef ALLOW_USE_MPI |
| 35 |
#include "EESUPPORT.h" |
| 36 |
#endif |
| 37 |
|
| 38 |
INTEGER myThid |
| 39 |
|
| 40 |
#ifdef ALLOW_NONHYDROSTATIC |
| 41 |
|
| 42 |
INTEGER i, j, bi, bj, k, l |
| 43 |
CHARACTER*(MAX_LEN_MBUF) msgBuf |
| 44 |
#ifdef ALLOW_USE_MPI |
| 45 |
integer mpiRC, mpiMyWid |
| 46 |
#endif |
| 47 |
#ifdef ALLOW_PETSC |
| 48 |
_RS DoFCountColor (MAX_CG3D_PETSC_CPUINVERT) |
| 49 |
_RS DoFCountColorCumSum (MAX_CG3D_PETSC_CPUINVERT) |
| 50 |
! _RS DoFCount |
| 51 |
integer color, rank |
| 52 |
integer cg3d_dofs_proc_loc (0:nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1) |
| 53 |
integer cg3d_dofs_cum_sum (0:nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1) |
| 54 |
integer dofPerRank |
| 55 |
|
| 56 |
#ifdef ALLOW_USE_MPI |
| 57 |
CALL MPI_COMM_RANK( MPI_COMM_WORLD, mpiMyWId, mpiRC ) |
| 58 |
#endif |
| 59 |
|
| 60 |
dofPerRank = Nr/cg3d_petsc_cpuInVert |
| 61 |
|
| 62 |
IF (cg3d_petsc_cpuInVert.eq.1) THEN |
| 63 |
cg3d_color_rank(1) = mpimywid |
| 64 |
ELSE |
| 65 |
! NEEDS TO CHANGE, SUCH THAT |
| 66 |
! CG3D_COLOR_RANK(i) = rank corresponding to ith vertical tile |
| 67 |
! and then pushed to all other processors IN VERTICAL COLUMN ONLY |
| 68 |
cg3d_color_rank (1) = mpimywid |
| 69 |
ENDIF |
| 70 |
|
| 71 |
DO bj = myByLo(myThid), myByHi(myThid) |
| 72 |
DO bi = myBxLo(myThid), myBxHi(myThid) |
| 73 |
DO j=1,sNy |
| 74 |
DO i=1,sNx |
| 75 |
DO k=1,Nr |
| 76 |
cg3d_petsc_dofs (i,j,k,bi,bj) = -2.0 |
| 77 |
cg3d_petsc_color (i,j,k,bi,bj) = 1 + ((k-1) / dofPerRank) |
| 78 |
ENDDO |
| 79 |
ENDDO |
| 80 |
ENDDO |
| 81 |
ENDDO |
| 82 |
ENDDO |
| 83 |
|
| 84 |
DO k=1, MAX_CG3D_PETSC_CPUINVERT |
| 85 |
IF (k.le.cg3d_petsc_cpuInVert) THEN |
| 86 |
DoFCountColor (k) = -1.0 |
| 87 |
ENDIF |
| 88 |
ENDDO |
| 89 |
!DoFCount = -1.0 |
| 90 |
DO bj = myByLo(myThid), myByHi(myThid) |
| 91 |
DO bi = myBxLo(myThid), myBxHi(myThid) |
| 92 |
DO j=1,sNy |
| 93 |
DO i=1,sNx |
| 94 |
DO k=1,Nr |
| 95 |
|
| 96 |
|
| 97 |
C DOFS ARE NUMBERED AS FOLLOWS ON PROCESSOR DOMAIN: |
| 98 |
C grid is stepped through in order bj, bi, j, i |
| 99 |
C 1) if hfacc0(i,j,k,bi,bj)>0, the counter is updated by 1; |
| 100 |
C cg3d_petsc_dofs is assigned the counter; |
| 101 |
C o/w cg3d_petsc_dofs is assigned -1 |
| 102 |
C NOTE THESE NUMBERING ARRAYS ARE USED TO CONSTRUCT PETSC VECTORS AND MATRIX |
| 103 |
|
| 104 |
color = cg3d_petsc_color (i,j,k,bi,bj) |
| 105 |
|
| 106 |
if (_hFacC(i,j,k,bi,bj).gt.0.0) THEN |
| 107 |
DoFCountColor(color) = DoFCountColor(color) + 1.0 |
| 108 |
cg3d_petsc_dofs (i,j,k,bi,bj) = DoFCountColor(color) |
| 109 |
else |
| 110 |
cg3d_petsc_dofs (i,j,k,bi,bj) = -1.0 |
| 111 |
endif |
| 112 |
|
| 113 |
ENDDO |
| 114 |
ENDDO |
| 115 |
ENDDO |
| 116 |
ENDDO |
| 117 |
ENDDO |
| 118 |
|
| 119 |
! SUM UP ALL DOF COUNTS WITHIN A COLUMN -- NOT GLOBAL YET |
| 120 |
! SAME FOR ALL VERTICAL TILES IN A COLUMN |
| 121 |
! DoFCountColorCumSum (1) = 0.0 |
| 122 |
! DO k=2, MAX_CG3D_PETSC_CPUINVERT |
| 123 |
! IF (k.le.cg3d_petsc_cpuInVert) THEN |
| 124 |
! DoFCountColorCumSum (k) = DoFCountColorCumSum (k-1) + |
| 125 |
! & DoFCountColor (k-1) |
| 126 |
! ENDIF |
| 127 |
! ENDDO |
| 128 |
|
| 129 |
|
| 130 |
#ifdef ALLOW_USE_MPI |
| 131 |
|
| 132 |
DO i=0,nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1 |
| 133 |
# IF (i.le.cg3d_petsc_cpuInVert) THEN |
| 134 |
cg3d_dofs_proc_loc (i) = 0 |
| 135 |
# ENDIF |
| 136 |
ENDDO |
| 137 |
|
| 138 |
DO l=1,MAX_CG3D_PETSC_CPUINVERT |
| 139 |
IF (l.le.cg3d_petsc_cpuInVert) THEN |
| 140 |
IF (cg3d_color_rank(l).eq.mpiMyWid) THEN |
| 141 |
cg3d_dofs_proc_loc (mpiMyWId) = INT(DoFCountColor(l))+1 |
| 142 |
ENDIF |
| 143 |
ENDIF |
| 144 |
ENDDO |
| 145 |
|
| 146 |
CALL MPI_Allreduce(cg3d_dofs_proc_loc,cg3d_dofs_process, |
| 147 |
& nPx*nPy*MAX_CG3D_PETSC_CPUINVERT,MPI_INTEGER, |
| 148 |
& MPI_SUM,MPI_COMM_MODEL,mpiRC) |
| 149 |
|
| 150 |
cg3d_dofs_cum_sum(0) = 0 |
| 151 |
|
| 152 |
DO i=1,nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1 |
| 153 |
if (i.ge.1) then |
| 154 |
! IF (i.le.cg3d_petsc_cpuInVert) THEN |
| 155 |
cg3d_dofs_cum_sum(i) = cg3d_dofs_cum_sum(i-1)+ |
| 156 |
& cg3d_dofs_process(i-1) |
| 157 |
! ENDIF |
| 158 |
endif |
| 159 |
ENDDO |
| 160 |
|
| 161 |
#else /* ALLOW_USE_MPI */ |
| 162 |
|
| 163 |
cg3d_dofs_process (0) = INT(DoFCountColor(1))+1 |
| 164 |
cg3d_dofs_cum_sum (0) = INT(DoFCountColor(1))+1 |
| 165 |
|
| 166 |
#endif /* ALLOW_USE_MPI */ |
| 167 |
|
| 168 |
DO bj = myByLo(myThid), myByHi(myThid) |
| 169 |
DO bi = myBxLo(myThid), myBxHi(myThid) |
| 170 |
DO j=1,sNy |
| 171 |
DO i=1,sNx |
| 172 |
DO k=1,Nr |
| 173 |
IF (cg3d_petsc_dofs(i,j,k,bi,bj).ge.0 ) THEN |
| 174 |
color = cg3d_petsc_color (i,j,k,bi,bj) |
| 175 |
rank = cg3d_color_rank (color) |
| 176 |
cg3d_petsc_dofs(i,j,k,bi,bj) = |
| 177 |
& cg3d_petsc_dofs(i,j,k,bi,bj) + |
| 178 |
& cg3d_dofs_cum_sum(rank) |
| 179 |
ENDIF |
| 180 |
ENDDO |
| 181 |
ENDDO |
| 182 |
ENDDO |
| 183 |
ENDDO |
| 184 |
ENDDO |
| 185 |
|
| 186 |
_EXCH_XYZ_RS(cg3d_petsc_dofs,myThid) |
| 187 |
|
| 188 |
call write_fld_xyz_rs ('petscDofs1','',cg3d_petsc_dofs,0,mythid) |
| 189 |
call write_fld_xyz_rs ('petscColor1','',cg3d_petsc_color,0,mythid) |
| 190 |
print *, "GOT HERE CG3D RANK COLOR", cg3d_dofs_proc_loc |
| 191 |
|
| 192 |
#endif /* ALLOW_PETSC */ |
| 193 |
|
| 194 |
|
| 195 |
#endif |
| 196 |
RETURN |
| 197 |
END |
Parent Directory
| 
Revision Log
| 
Revision Graph