model/src/the_model_main.F

C $Header: /u/gcmpack/models/MITgcmUV/model/src/the_model_main.F,v 1.21 1998/10/28 03:11:38 cnh Exp $

#include "CPP_OPTIONS.h"

      SUBROUTINE THE_MODEL_MAIN(myThid)
C     /==========================================================\
C     | SUBROUTINE THE_MODEL_MAIN                                |
C     | o Master controlling routine for model using the MITgcm  |
C     |   UV parallel wrapper.                                   |
C     |==========================================================|
C     | THE_MODEL_MAIN is invoked by the MITgcm UV parallel      |
C     | wrapper with a single integer argument "myThid". This    |
C     | variable identifies the thread number of an instance of  |
C     | THE_MODEL_MAIN. Each instance of THE_MODEL_MAIN works    |
C     | on a particular region of the models domain and          |
C     | synchronises with other instances as necessary. The      |
C     | routine has to "understand" the MITgcm parallel          |
C     | environment and the numerical algorithm. Editing this    |
C     | routine is best done with some knowledge of both aspects.|
C     | Notes                                                    |
C     | =====                                                    |
C     | C*P* comments indicating place holders for which code is |
C     |      presently being developed.                          |
C     \==========================================================/
C
C     Call Tree
C     =========
C     
C      main ( eesupp )
C       |
C       .
C       .
C       . Generic environment initialisation ( see eesupp/src and
C       .                                      eesupp/inc )      
C       . multiple threads and/or processes are created in here
C       .
C       .
C       .
C       |
C       |-THE_MODEL_MAIN - Begin specific model. One instance
C       |  |               of this codes exists for each thread
C       |  |               and/or instance. Each instance manages
C       |  |               a specifc set of tiles.               
C       |  |
C       |  |--INITIALISE
C       |  |   o Set initial conditions and model configuration
C       |  |     Topography, hydrography, timestep, grid, etc..
C       |  | 
C  ==>  |  | ** Time stepping loop starts here **
C  |    |  |
C /|\   |  |
C  |    |  |--LOAD_EXTERNAL_DATA
C /|\   |  |   o Load and/or set time dependent forcing fields
C  |    |  |
C /|\   |  |--DYNAMICS
C  |    |  |   o Evaluate "forward" terms
C /|\   |  |
C  |    |  |--DO_THE_MODEL_IO
C /|\   |  |   o Write model state
C  |    |  |
C /|\   |  |--SOLVE_FOR_PRESSURE
C  |    |  |   o Find pressure field to keep flow non-divergent
C /|\   |  |
C  |    |  |--DO_GTERM_BLOCKING_EXCHANGES
C /|\   |  |   o Update overlap regions
C  |    |  |
C /|\   |  |--WRITE_CHECKPOINT
C  |    |  |   o Write restart file(s)
C /|\   |  |
C  |    |  |
C  |<== |  | ** Time stepping loop finishes here **
C       |  |
C       |  |--WRITE_STATE
C       |  |--WRITE_CHECKPOINT
C       |
C       .
C       .
C       . Generic environment termination ( see eesupp/src and
C       .                                      eesupp/inc )      
C       .
C       .

C     == Global variables ===
#include "SIZE.h"
#include "EEPARAMS.h"
#include "PARAMS.h"
#include "CG2D.h"
#include "DYNVARS.h"
#include "AVER.h"

C     == Routine arguments ==
C     myThid - Thread number for this instance of the routine.
      INTEGER myThid  

C     == Local variables ==
C     Note: Under the multi-threaded model myCurrentIter and 
C           myCurrentTime are local variables passed around as routine 
C           arguments. Although this is fiddly it saves the need to 
C           impose additional synchronisation points when they are 
C           updated.
C     myCurrentIter - Iteration counter for this thread
C     myCurrentTime - Time counter for this thread
C     I             - Loop counter
      INTEGER I, myCurrentIter
      REAL    myCurrentTime

C--   This timer encompasses the whole code
      CALL TIMER_START('ALL',myThid)

      CALL TIMER_START('SPIN-UP',myThid)

C--   Set model initial conditions 
      CALL TIMER_START('INITIALISE          [SPIN-UP]',myThid)
      CALL INITIALISE( myThid )
      myCurrentTime = startTime
      myCurrentIter = nIter0
      CALL TIMER_STOP ('INITIALISE          [SPIN-UP]',myThid)

C--   Dump for start state
      CALL TIMER_START('I/O (WRITE)         [SPIN-UP]',myThid)
      CALL WRITE_STATE( .TRUE., myCurrentTime, myCurrentIter, myThid )
      CALL TIMER_STOP ('I/O (WRITE)         [SPIN-UP]',myThid)

      CALL TIMER_STOP ('SPIN-UP',myThid)

C--   Begin time stepping loop
      CALL TIMER_START('MAIN LOOP',myThid)
      DO I=1, nTimeSteps

C--    Load forcing/external data fields
       CALL TIMER_START('I/O (READ)         [MAIN LOOP]',myThid)
       CALL LOAD_EXTERNAL_FIELDS( myCurrentTime, myCurrentIter, myThid )
       CALL TIMER_STOP ('I/O (READ)         [MAIN LOOP]',myThid)

#ifdef INCLUDE_SHAPIRO_FILTER_CODE
C--    Step forward all tiles, filter and exchange.
       CALL TIMER_START('SHAP_FILT          [MAIN LOOP]',myThid)
       CALL SHAP_FILT( myCurrentTime, myCurrentIter, myThid )
       CALL TIMER_STOP ('SHAP_FILT          [MAIN LOOP]',myThid)
#endif


C--    Step forward fields and calculate time tendency terms
       CALL TIMER_START('DYNAMICS           [MAIN LOOP]',myThid)
       CALL DYNAMICS( myCurrentTime, myCurrentIter, myThid )
       CALL TIMER_STOP ('DYNAMICS           [MAIN LOOP]',myThid)

C--    Do time averages
#ifdef INCLUDE_DIAGNOSTICS_INTERFACE_CODE
       CALL TIMER_START('I/O (WRITE)        [MAIN LOOP]',myThid)
       IF (taveFreq.GT.0.) THEN
        CALL WRITE_TIME_AVERAGES( myCurrentTime, myCurrentIter, myThid )
       ENDIF
       CALL TIMER_STOP ('I/O (WRITE)        [MAIN LOOP]',myThid)
#endif

C--    Do IO if needed.
C      Note:
C      =====
C      At this point model arrays hold U,V,T,S  at "time-level" N 
C      and cg2d_x at "time-level" N-1/2 where N = I+timeLevBase-1. 
C      By convention this is taken to be the model "state". 
       CALL TIMER_START('I/O (WRITE)        [MAIN LOOP]',myThid)
       CALL DO_THE_MODEL_IO( myCurrentTime, myCurrentIter, myThid )
       CALL TIMER_STOP ('I/O (WRITE)        [MAIN LOOP]',myThid)

C--    Solve elliptic equation(s).
C      Two-dimensional only for conventional hydrostatic or 
C      three-dimensional for non-hydrostatic and/or IGW scheme.
       CALL TIMER_START('SOLVE_FOR_PRESSURE [MAIN LOOP]',myThid)
       CALL SOLVE_FOR_PRESSURE( myThid )
       CALL TIMER_STOP ('SOLVE_FOR_PRESSURE [MAIN LOOP]',myThid)

C--    Do "blocking" sends and receives for tendency "overlap" terms
       CALL TIMER_START('BLOCKING_EXCHANGES [MAIN LOOP]',myThid)
       CALL DO_GTERM_BLOCKING_EXCHANGES( myThid )
       CALL TIMER_STOP ('BLOCKING_EXCHANGES [MAIN LOOP]',myThid)

       myCurrentIter = myCurrentIter + 1
       myCurrentTime = myCurrentTime + deltaTClock 

C--    Save state for restarts
C      Note:
C      =====
C      Because of the ordering of the timestepping code and
C      tendency term code at end of loop model arrays hold
C      U,V,T,S  at "time-level" N but gu, gv, gs, gt, guNM1,... 
C      at "time-level" N+1/2 (guNM1 at "time-level" N+1/2 is 
C      gu at "time-level" N-1/2) and cg2d_x at "time-level" N+1/2.
C       where N = I+timeLevBase-1
C      Thus a checkpoint contains U.0000000000, GU.0000000001 and 
C      cg2d_x.0000000001 in the indexing scheme used for the model 
C      "state" files. This example is referred to as a checkpoint 
C      at time level 1 
       CALL TIMER_START('I/O (WRITE)        [MAIN LOOP]',myThid)
       CALL 
     & WRITE_CHECKPOINT( .FALSE., myCurrentTime, myCurrentIter, myThid )
       CALL TIMER_STOP ('I/O (WRITE)        [MAIN LOOP]',myThid)

      ENDDO
      CALL TIMER_STOP ('MAIN LOOP',myThid)
      CALL TIMER_START('SPIN-DOWN',myThid)

C--   Final checkpoint (incase the in-loop checkpoint was missed)
      CALL TIMER_START('I/O (WRITE)         [SPIN-DOWN]',myThid)
      CALL 
     & WRITE_CHECKPOINT( .TRUE., myCurrentTime, myCurrentIter, myThid )
      CALL TIMER_STOP ('I/O (WRITE)         [SPIN-DOWN]',myThid)

C--   Step-forward U/V/Theta/Salt for purposes of final I/O dump
      CALL TIMER_START('DYNAMICS            [SPIN-DOWN]',myThid)
      CALL DYNAMICS( myCurrentTime, myCurrentIter, myThid )
      CALL TIMER_STOP ('DYNAMICS            [SPIN-DOWN]',myThid)

C--   Do time averages
#ifdef ALLOW_DIAGNOSTICS
      IF (taveFreq.GT.0.) THEN
       CALL TIMER_START('I/O (WRITE)        [SPIN-DOWN]',myThid)
       CALL WRITE_TIME_AVERAGES( myCurrentTime, myCurrentIter, myThid )
       CALL TIMER_STOP ('I/O (WRITE)        [SPIN-DOWN]',myThid)
      ENDIF
#endif

C--   Dump for end state
      CALL TIMER_START('I/O (WRITE)         [SPIN-DOWN]',myThid)
      CALL WRITE_STATE( .FALSE., myCurrentTime, myCurrentIter, myThid )
      CALL TIMER_STOP ('I/O (WRITE)         [SPIN-DOWN]',myThid)

      CALL TIMER_STOP ('SPIN-DOWN',myThid)
      CALL TIMER_STOP ('ALL',myThid)

C--   Write timer statistics
      IF ( myThid .EQ. 1 ) THEN
       CALL TIMER_PRINTALL( myThid )
       CALL COMM_STATS
      ENDIF

      RETURN
      END
1	cnh	1.22	C $Header: /u/gcmpack/models/MITgcmUV/model/src/the_model_main.F,v 1.21 1998/10/28 03:11:38 cnh Exp $
2	cnh	1.1
3	adcroft	1.18	#include "CPP_OPTIONS.h"
4	cnh	1.1
5			SUBROUTINE THE_MODEL_MAIN(myThid)
6			C /==========================================================\
7			C \| SUBROUTINE THE_MODEL_MAIN \|
8			C \| o Master controlling routine for model using the MITgcm \|
9			C \| UV parallel wrapper. \|
10			C \|==========================================================\|
11			C \| THE_MODEL_MAIN is invoked by the MITgcm UV parallel \|
12			C \| wrapper with a single integer argument "myThid". This \|
13			C \| variable identifies the thread number of an instance of \|
14			C \| THE_MODEL_MAIN. Each instance of THE_MODEL_MAIN works \|
15			C \| on a particular region of the models domain and \|
16			C \| synchronises with other instances as necessary. The \|
17			C \| routine has to "understand" the MITgcm parallel \|
18			C \| environment and the numerical algorithm. Editing this \|
19			C \| routine is best done with some knowledge of both aspects.\|
20			C \| Notes \|
21			C \| ===== \|
22			C \| CP comments indicating place holders for which code is \|
23			C \| presently being developed. \|
24			C \==========================================================/
25	cnh	1.10	C
26			C Call Tree
27			C =========
28			C
29			C main ( eesupp )
30			C \|
31			C .
32			C .
33			C . Generic environment initialisation ( see eesupp/src and
34			C . eesupp/inc )
35			C . multiple threads and/or processes are created in here
36			C .
37			C .
38			C .
39			C \|
40			C \|-THE_MODEL_MAIN - Begin specific model. One instance
41			C \| \| of this codes exists for each thread
42			C \| \| and/or instance. Each instance manages
43			C \| \| a specifc set of tiles.
44			C \| \|
45			C \| \|--INITIALISE
46			C \| \| o Set initial conditions and model configuration
47			C \| \| Topography, hydrography, timestep, grid, etc..
48			C \| \|
49			C ==> \| \| Time stepping loop starts here
50			C \| \| \|
51			C /\|\ \| \|
52			C \| \| \|--LOAD_EXTERNAL_DATA
53			C /\|\ \| \| o Load and/or set time dependent forcing fields
54			C \| \| \|
55			C /\|\ \| \|--DYNAMICS
56			C \| \| \| o Evaluate "forward" terms
57			C /\|\ \| \|
58			C \| \| \|--DO_THE_MODEL_IO
59			C /\|\ \| \| o Write model state
60			C \| \| \|
61			C /\|\ \| \|--SOLVE_FOR_PRESSURE
62			C \| \| \| o Find pressure field to keep flow non-divergent
63			C /\|\ \| \|
64			C \| \| \|--DO_GTERM_BLOCKING_EXCHANGES
65			C /\|\ \| \| o Update overlap regions
66			C \| \| \|
67			C /\|\ \| \|--WRITE_CHECKPOINT
68			C \| \| \| o Write restart file(s)
69			C /\|\ \| \|
70			C \| \| \|
71			C \|<== \| \| Time stepping loop finishes here
72			C \| \|
73			C \| \|--WRITE_STATE
74	adcroft	1.17	C \| \|--WRITE_CHECKPOINT
75	cnh	1.10	C \|
76			C .
77			C .
78			C . Generic environment termination ( see eesupp/src and
79			C . eesupp/inc )
80			C .
81			C .
82	cnh	1.1
83			C == Global variables ===
84			#include "SIZE.h"
85			#include "EEPARAMS.h"
86			#include "PARAMS.h"
87			#include "CG2D.h"
88			#include "DYNVARS.h"
89	adcroft	1.14	#include "AVER.h"
90	cnh	1.1
91			C == Routine arguments ==
92			C myThid - Thread number for this instance of the routine.
93	cnh	1.6	INTEGER myThid
94	cnh	1.1
95			C == Local variables ==
96	cnh	1.7	C Note: Under the multi-threaded model myCurrentIter and
97			C myCurrentTime are local variables passed around as routine
98			C arguments. Although this is fiddly it saves the need to
99			C impose additional synchronisation points when they are
100			C updated.
101	cnh	1.1	C myCurrentIter - Iteration counter for this thread
102			C myCurrentTime - Time counter for this thread
103			C I - Loop counter
104			INTEGER I, myCurrentIter
105			REAL myCurrentTime
106
107	adcroft	1.19	C-- This timer encompasses the whole code
108			CALL TIMER_START('ALL',myThid)
109
110			CALL TIMER_START('SPIN-UP',myThid)
111
112	cnh	1.1	C-- Set model initial conditions
113	adcroft	1.19	CALL TIMER_START('INITIALISE [SPIN-UP]',myThid)
114	cnh	1.1	CALL INITIALISE( myThid )
115			myCurrentTime = startTime
116			myCurrentIter = nIter0
117	adcroft	1.19	CALL TIMER_STOP ('INITIALISE [SPIN-UP]',myThid)
118	cnh	1.1
119	adcroft	1.15	C-- Dump for start state
120	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [SPIN-UP]',myThid)
121	adcroft	1.15	CALL WRITE_STATE( .TRUE., myCurrentTime, myCurrentIter, myThid )
122	adcroft	1.19	CALL TIMER_STOP ('I/O (WRITE) [SPIN-UP]',myThid)
123
124			CALL TIMER_STOP ('SPIN-UP',myThid)
125	adcroft	1.15
126	cnh	1.1	C-- Begin time stepping loop
127	adcroft	1.19	CALL TIMER_START('MAIN LOOP',myThid)
128	cnh	1.1	DO I=1, nTimeSteps
129	cnh	1.4
130	cnh	1.11	C-- Load forcing/external data fields
131	adcroft	1.19	CALL TIMER_START('I/O (READ) [MAIN LOOP]',myThid)
132	cnh	1.11	CALL LOAD_EXTERNAL_FIELDS( myCurrentTime, myCurrentIter, myThid )
133	adcroft	1.19	CALL TIMER_STOP ('I/O (READ) [MAIN LOOP]',myThid)
134	cnh	1.1
135	cnh	1.22	#ifdef INCLUDE_SHAPIRO_FILTER_CODE
136			C-- Step forward all tiles, filter and exchange.
137			CALL TIMER_START('SHAP_FILT [MAIN LOOP]',myThid)
138			CALL SHAP_FILT( myCurrentTime, myCurrentIter, myThid )
139			CALL TIMER_STOP ('SHAP_FILT [MAIN LOOP]',myThid)
140			#endif
141
142
143	cnh	1.1	C-- Step forward fields and calculate time tendency terms
144	adcroft	1.19	CALL TIMER_START('DYNAMICS [MAIN LOOP]',myThid)
145	cnh	1.8	CALL DYNAMICS( myCurrentTime, myCurrentIter, myThid )
146	adcroft	1.19	CALL TIMER_STOP ('DYNAMICS [MAIN LOOP]',myThid)
147	cnh	1.1
148	adcroft	1.14	C-- Do time averages
149	cnh	1.22	#ifdef INCLUDE_DIAGNOSTICS_INTERFACE_CODE
150	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [MAIN LOOP]',myThid)
151	adcroft	1.14	IF (taveFreq.GT.0.) THEN
152	adcroft	1.17	CALL WRITE_TIME_AVERAGES( myCurrentTime, myCurrentIter, myThid )
153	adcroft	1.14	ENDIF
154	adcroft	1.19	CALL TIMER_STOP ('I/O (WRITE) [MAIN LOOP]',myThid)
155	adcroft	1.17	#endif
156	adcroft	1.14
157	cnh	1.7	C-- Do IO if needed.
158			C Note:
159			C =====
160			C At this point model arrays hold U,V,T,S at "time-level" N
161			C and cg2d_x at "time-level" N-1/2 where N = I+timeLevBase-1.
162			C By convention this is taken to be the model "state".
163	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [MAIN LOOP]',myThid)
164	cnh	1.7	CALL DO_THE_MODEL_IO( myCurrentTime, myCurrentIter, myThid )
165	adcroft	1.19	CALL TIMER_STOP ('I/O (WRITE) [MAIN LOOP]',myThid)
166	cnh	1.7
167	cnh	1.1	C-- Solve elliptic equation(s).
168	cnh	1.7	C Two-dimensional only for conventional hydrostatic or
169			C three-dimensional for non-hydrostatic and/or IGW scheme.
170	adcroft	1.19	CALL TIMER_START('SOLVE_FOR_PRESSURE [MAIN LOOP]',myThid)
171	cnh	1.1	CALL SOLVE_FOR_PRESSURE( myThid )
172	adcroft	1.19	CALL TIMER_STOP ('SOLVE_FOR_PRESSURE [MAIN LOOP]',myThid)
173	cnh	1.1
174			C-- Do "blocking" sends and receives for tendency "overlap" terms
175	adcroft	1.19	CALL TIMER_START('BLOCKING_EXCHANGES [MAIN LOOP]',myThid)
176	cnh	1.1	CALL DO_GTERM_BLOCKING_EXCHANGES( myThid )
177	adcroft	1.19	CALL TIMER_STOP ('BLOCKING_EXCHANGES [MAIN LOOP]',myThid)
178	cnh	1.1
179			myCurrentIter = myCurrentIter + 1
180	cnh	1.7	myCurrentTime = myCurrentTime + deltaTClock
181	cnh	1.1
182			C-- Save state for restarts
183	cnh	1.7	C Note:
184			C =====
185	adcroft	1.12	C Because of the ordering of the timestepping code and
186	cnh	1.7	C tendency term code at end of loop model arrays hold
187			C U,V,T,S at "time-level" N but gu, gv, gs, gt, guNM1,...
188			C at "time-level" N+1/2 (guNM1 at "time-level" N+1/2 is
189			C gu at "time-level" N-1/2) and cg2d_x at "time-level" N+1/2.
190			C where N = I+timeLevBase-1
191			C Thus a checkpoint contains U.0000000000, GU.0000000001 and
192			C cg2d_x.0000000001 in the indexing scheme used for the model
193			C "state" files. This example is referred to as a checkpoint
194			C at time level 1
195	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [MAIN LOOP]',myThid)
196	cnh	1.21	CALL
197			& WRITE_CHECKPOINT( .FALSE., myCurrentTime, myCurrentIter, myThid )
198	adcroft	1.19	CALL TIMER_STOP ('I/O (WRITE) [MAIN LOOP]',myThid)
199	cnh	1.1
200			ENDDO
201	adcroft	1.19	CALL TIMER_STOP ('MAIN LOOP',myThid)
202			CALL TIMER_START('SPIN-DOWN',myThid)
203	adcroft	1.13
204	adcroft	1.12	C-- Final checkpoint (incase the in-loop checkpoint was missed)
205	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [SPIN-DOWN]',myThid)
206	cnh	1.21	CALL
207			& WRITE_CHECKPOINT( .TRUE., myCurrentTime, myCurrentIter, myThid )
208	adcroft	1.19	CALL TIMER_STOP ('I/O (WRITE) [SPIN-DOWN]',myThid)
209	adcroft	1.12
210			C-- Step-forward U/V/Theta/Salt for purposes of final I/O dump
211	adcroft	1.19	CALL TIMER_START('DYNAMICS [SPIN-DOWN]',myThid)
212	adcroft	1.12	CALL DYNAMICS( myCurrentTime, myCurrentIter, myThid )
213	adcroft	1.19	CALL TIMER_STOP ('DYNAMICS [SPIN-DOWN]',myThid)
214	adcroft	1.14
215			C-- Do time averages
216	adcroft	1.17	#ifdef ALLOW_DIAGNOSTICS
217	adcroft	1.14	IF (taveFreq.GT.0.) THEN
218	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [SPIN-DOWN]',myThid)
219			CALL WRITE_TIME_AVERAGES( myCurrentTime, myCurrentIter, myThid )
220			CALL TIMER_STOP ('I/O (WRITE) [SPIN-DOWN]',myThid)
221	adcroft	1.14	ENDIF
222	adcroft	1.17	#endif
223	adcroft	1.12
224			C-- Dump for end state
225	adcroft	1.19	CALL TIMER_START('I/O (WRITE) [SPIN-DOWN]',myThid)
226	adcroft	1.15	CALL WRITE_STATE( .FALSE., myCurrentTime, myCurrentIter, myThid )
227	adcroft	1.19	CALL TIMER_STOP ('I/O (WRITE) [SPIN-DOWN]',myThid)
228
229			CALL TIMER_STOP ('SPIN-DOWN',myThid)
230			CALL TIMER_STOP ('ALL',myThid)
231	cnh	1.7
232	adcroft	1.19	C-- Write timer statistics
233	cnh	1.20	IF ( myThid .EQ. 1 ) THEN
234			CALL TIMER_PRINTALL( myThid )
235			CALL COMM_STATS
236			ENDIF
237	cnh	1.1
238			RETURN
239			END