model/src/the_main_loop.F

C $Header: /u/gcmpack/MITgcm/model/src/the_main_loop.F,v 1.21.2.4 2002/11/07 16:56:31 heimbach Exp $

#include "CPP_OPTIONS.h"

CBOP
C     !ROUTINE: THE_MAIN_LOOP
C     !INTERFACE:
      SUBROUTINE THE_MAIN_LOOP( mytime, myiter, mythid )

C     !DESCRIPTION: \bv
C     *================================================================*
C     | SUBROUTINE the_main_loop
C     | o Run the ocean model and evaluate the specified cost function.
C     *================================================================*
C     |
C     | THE_MAIN_LOOP is the toplevel routine for the Tangent Linear and
C     | Adjoint Model Compiler (TAMC). For this purpose the initialization
C     | of the model was split into two parts. Those parameters that do
C     | not depend on a specific model run are set in INITIALISE_FIXED,
C     | whereas those that do depend on the specific realization are
C     | initialized in INITIALISE_VARIA. 
C     | This routine is to be used in conjuction with the MITgcmuv 
C     | checkpoint 37.
C     *================================================================*
C     \ev

C     !USES:
      IMPLICIT NONE
C     == Global variables ==
#include "SIZE.h"
#include "EEPARAMS.h"
#include "PARAMS.h"
#ifdef ALLOW_AUTODIFF_TAMC
# include "tamc.h"
# include "ctrl.h"
# include "ctrl_dummy.h"
# include "cost.h"
# include "DYNVARS.h"
# include "FFIELDS.h"
# include "GAD.h"
# ifdef ALLOW_PASSIVE_TRACER
#  include "TR1.h"
# endif
# ifdef ALLOW_NONHYDROSTATIC
#  include "CG3D.h"
# endif
# ifdef EXACT_CONSERV
#  include "SURFACE.h"
# endif
#endif

C     !INPUT/OUTPUT PARAMETERS:
C     == Routine arguments ==
C     note: under the multi-threaded model myiter and 
C           mytime are local variables passed around as routine 
C           arguments. Although this is fiddly it saves the need to 
C           impose additional synchronisation points when they are 
C           updated.
C     myIter - iteration counter for this thread
C     myTime - time counter for this thread
C     myThid - thread number for this instance of the routine.
      INTEGER myThid 
      INTEGER myIter
      _RL     myTime

C     !LOCAL VARIABLES:
C     == Local variables ==
      integer iloop
#ifdef ALLOW_TAMC_CHECKPOINTING
      integer ilev_1
      integer ilev_2
      integer ilev_3
      integer max_lev2
      integer max_lev3
#endif
CEOP

#ifdef ALLOW_AUTODIFF_TAMC
c--   Initialize storage for the cost function evaluation.
CADJ  INIT dummytape = common, 1
c--   Initialize storage for the outermost loop.
CADJ  INIT tapelev3 = USER
CADJ  INIT tapelev_ini_bibj_k   = USER
#ifdef ALLOW_TAMC_CHECKPOINTING
       nIter0 = INT( startTime/deltaTClock )
       ikey_dynamics = 1
#endif
      CALL TIMER_START('ADJOINT SPIN-UP', mythid)
#endif

C--   Set initial conditions (variable arrays)
      CALL TIMER_START('INITIALISE_VARIA    [THE_MAIN_LOOP]', mythid)
      CALL INITIALISE_VARIA( mythid )
      CALL TIMER_STOP ('INITIALISE_VARIA    [THE_MAIN_LOOP]', mythid)

#ifndef ALLOW_AUTODIFF_TAMC
c--   Dump for start state.
      CALL TIMER_START('WRITE_STATE         [THE_MAIN_LOOP]', mythid)
      CALL WRITE_STATE( mytime, myiter, mythid )
      CALL TIMER_STOP ('WRITE_STATE         [THE_MAIN_LOOP]', mythid)
#endif

#ifndef EXCLUDE_MONITOR
C--   Check status of solution (statistics, cfl, etc...)
      CALL TIMER_START('MONITOR             [THE_MAIN_LOOP]', mythid)
      CALL MONITOR( myIter, myTime, myThid )
      CALL TIMER_STOP ('MONITOR             [THE_MAIN_LOOP]', mythid)
#endif /* EXCLUDE_MONITOR */

#if (defined (ALLOW_ADJOINT_RUN) || defined (ALLOW_TANGENTLINEAR_RUN))
c--   Add control vector for forcing and parameter fields
      CALL CTRL_MAP_FORCING (mythid)
#endif

#ifdef ALLOW_AUTODIFF_TAMC
      CALL TIMER_STOP ('ADJOINT SPIN-UP', mythid)
      _BARRIER
#endif

c--   Do the model integration.
      CALL TIMER_START('MAIN LOOP           [THE_MAIN_LOOP]', mythid)

c     >>>>>>>>>>>>>>>>>>>>>>>>>>>   LOOP   <<<<<<<<<<<<<<<<<<<<<<<<<<<<
c     >>>>>>>>>>>>>>>>>>>>>>>>>>>  STARTS  <<<<<<<<<<<<<<<<<<<<<<<<<<<<

#ifdef ALLOW_AUTODIFF_TAMC
#ifdef ALLOW_TAMC_CHECKPOINTING
c--   Implement a three level checkpointing. For a two level
c--   checkpointing delete the middle loop; for n levels (n > 3)
c--   insert more loops.

c--   Check the choice of the checkpointing parameters in relation
c--   to nTimeSteps: (nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps)
      if (nchklev_1*nchklev_2*nchklev_3 .lt. nTimeSteps) then
        print*
        print*, ' the_main_loop: TAMC checkpointing parameters'
        print*, '            nchklev_1*nchklev_2*nchklev_3 = ',
     &       nchklev_1*nchklev_2*nchklev_3
        print*, '            are not consistent with nTimeSteps = ',
     &       nTimeSteps
        stop    ' ... stopped in the_main_loop.'
      endif
      max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1
      max_lev2=nTimeSteps/nchklev_1+1

c**************************************
#ifdef ALLOW_DIVIDED_ADJOINT
CADJ loop = divided
#endif
c**************************************

      do ilev_3 = 1,nchklev_3
         if(ilev_3.le.max_lev3) then
c**************************************
#include "checkpoint_lev3_directives.h"
c**************************************

c--     Initialise storage for the middle loop.
CADJ    INIT tapelev2 = USER

        do ilev_2 = 1,nchklev_2
         if(ilev_2.le.max_lev2) then
c**************************************
#include "checkpoint_lev2_directives.h"
c**************************************

c--       Initialize storage for the innermost loop.
c--       Always check common block sizes for the checkpointing!
CADJ INIT comlev1        = COMMON,nchklev_1
CADJ INIT comlev1_bibj   = COMMON,nchklev_1*nsx*nsy*nthreads_chkpt
CADJ INIT comlev1_bibj_k = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt
c--
#ifdef ALLOW_KPP
CADJ INIT comlev1_kpp    = COMMON,nchklev_1*nsx*nsy
#endif /* ALLOW_KPP */
c--
#ifdef ALLOW_GMREDI
CADJ INIT comlev1_gmredi_k_gad
CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
#endif /* ALLOW_GMREDI */
c--
#ifndef DISABLE_MULTIDIM_ADVECTION
CADJ INIT comlev1_bibj_k_gad
CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
CADJ INIT comlev1_bibj_k_gad_pass
CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass*maxcube
#endif /* DISABLE_MULTIDIM_ADVECTION */
c--
#ifdef ALLOW_BULKFORMULAE
CADJ INIT comlev1_exf_1
CADJ &     = COMMON,nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt
CADJ INIT comlev1_exf_2
CADJ &     = COMMON,niter_bulk*nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt
#endif /* ALLOW_BULKFORMULAE */

          do ilev_1 = 1,nchklev_1

c--         The if-statement below introduces a some flexibility in the
c--         choice of the 3-tupel ( nchklev_1, nchklev_2, nchklev_3 ).
c--
c--         Requirement: nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps .

            iloop = (ilev_3 - 1)*nchklev_2*nchklev_1 +
     &              (ilev_2 - 1)*nchklev_1           + ilev_1

            if ( iloop .le. nTimeSteps ) then

#else /* ALLOW_TAMC_CHECKPOINTING  undefined */
c--   Initialise storage for reference trajectory without TAMC check-
c--   pointing.
CADJ INIT history        = USER
CADJ INIT comlev1_bibj   = COMMON,nchklev_0*nsx*nsy*nthreads_chkpt
CADJ INIT comlev1_bibj_k = COMMON,nchklev_0*nsx*nsy*nr*nthreads_chkpt
CADJ INIT comlev1_kpp    = COMMON,nchklev_0*nsx*nsy

C--   RG replace 2 by max of num_v_smooth_Ri
CADJ INIT comlev1_kpp_sm = COMMON,nchklev_0*nsx*nsy*2

c--   Check the choice of the checkpointing parameters in relation
c--   to nTimeSteps: (nchklev_0 .ge. nTimeSteps)
      if (nchklev_0 .lt. nTimeSteps) then
        print*
        print*, ' the_main_loop: TAMC checkpointing parameter ',
     &       'nchklev_0 = ',       nchklev_0
        print*, '                 not consistent with nTimeSteps = ', 
     &       nTimeSteps
        stop    ' ... stopped in the_main_loop.'
      endif

      DO iloop = 1, nTimeSteps

#endif /* ALLOW_TAMC_CHECKPOINTING */

#else /* ALLOW_AUTODIFF_TAMC  undefined */

c--   Start the main loop of adjoint_Objfunc. Automatic differentiation
c--   NOT enabled.
      DO iloop = 1, nTimeSteps

#endif /* ALLOW_AUTODIFF_TAMC */

c--     >>> Loop body start <<<

#ifdef ALLOW_TAMC_CHECKPOINTING
        nIter0 = INT( startTime/deltaTClock )
        ikey_dynamics = ilev_1
#endif


        CALL TIMER_START('FORWARD_STEP        [THE_MAIN_LOOP]',mythid)
        CALL FORWARD_STEP( iloop, mytime, myiter, mythid )
        CALL TIMER_STOP ('FORWARD_STEP        [THE_MAIN_LOOP]',mythid)

#ifdef ALLOW_COST
C--     compare model with data and compute cost function
C--     this is done after exchanges to allow interpolation
        CALL TIMER_START('COST_TILE           [THE_MAIN_LOOP]',myThid)
        CALL COST_TILE  ( myThid )
        CALL TIMER_STOP ('COST_TILE           [THE_MAIN_LOOP]',myThid)
#endif

c--     >>> Loop body end <<<

#ifdef ALLOW_AUTODIFF_TAMC
#ifdef ALLOW_TAMC_CHECKPOINTING
            endif
          enddo
          endif
        enddo
        endif
      enddo
#else
      enddo
#endif

#else
      enddo
#endif

#ifdef ALLOW_COST
c--   Sum all cost function contributions.
      call TIMER_START('COST_FINAL         [ADJOINT SPIN-DOWN]', mythid)
      call COST_FINAL ( mythid )
      call TIMER_STOP ('COST_FINAL         [ADJOINT SPIN-DOWN]', mythid)
#endif

      _BARRIER
      CALL TIMER_STOP ('MAIN LOOP           [THE_MAIN_LOOP]', mythid)

      END
1	heimbach	1.28	C $Header: /u/gcmpack/MITgcm/model/src/the_main_loop.F,v 1.21.2.4 2002/11/07 16:56:31 heimbach Exp $
2	adcroft	1.1
3			#include "CPP_OPTIONS.h"
4
5	cnh	1.18	CBOP
6			C !ROUTINE: THE_MAIN_LOOP
7			C !INTERFACE:
8			SUBROUTINE THE_MAIN_LOOP( mytime, myiter, mythid )
9
10			C !DESCRIPTION: \bv
11			C ================================================================
12			C \| SUBROUTINE the_main_loop
13			C \| o Run the ocean model and evaluate the specified cost function.
14			C ================================================================
15			C \|
16			C \| THE_MAIN_LOOP is the toplevel routine for the Tangent Linear and
17			C \| Adjoint Model Compiler (TAMC). For this purpose the initialization
18			C \| of the model was split into two parts. Those parameters that do
19			C \| not depend on a specific model run are set in INITIALISE_FIXED,
20			C \| whereas those that do depend on the specific realization are
21			C \| initialized in INITIALISE_VARIA.
22			C \| This routine is to be used in conjuction with the MITgcmuv
23			C \| checkpoint 37.
24			C ================================================================
25			C \ev
26
27			C !USES:
28			IMPLICIT NONE
29			C == Global variables ==
30	adcroft	1.1	#include "SIZE.h"
31			#include "EEPARAMS.h"
32			#include "PARAMS.h"
33	heimbach	1.6	#ifdef ALLOW_AUTODIFF_TAMC
34	heimbach	1.16	# include "tamc.h"
35			# include "ctrl.h"
36			# include "ctrl_dummy.h"
37			# include "cost.h"
38			# include "DYNVARS.h"
39			# include "FFIELDS.h"
40	adcroft	1.21	# include "GAD.h"
41	heimbach	1.16	# ifdef ALLOW_PASSIVE_TRACER
42			# include "TR1.h"
43			# endif
44			# ifdef ALLOW_NONHYDROSTATIC
45			# include "CG3D.h"
46			# endif
47	heimbach	1.22	# ifdef EXACT_CONSERV
48			# include "SURFACE.h"
49			# endif
50	heimbach	1.6	#endif
51
52	cnh	1.18	C !INPUT/OUTPUT PARAMETERS:
53			C == Routine arguments ==
54			C note: under the multi-threaded model myiter and
55			C mytime are local variables passed around as routine
56			C arguments. Although this is fiddly it saves the need to
57			C impose additional synchronisation points when they are
58			C updated.
59			C myIter - iteration counter for this thread
60			C myTime - time counter for this thread
61			C myThid - thread number for this instance of the routine.
62			INTEGER myThid
63			INTEGER myIter
64			_RL myTime
65	heimbach	1.6
66	cnh	1.18	C !LOCAL VARIABLES:
67			C == Local variables ==
68	heimbach	1.6	integer iloop
69			#ifdef ALLOW_TAMC_CHECKPOINTING
70			integer ilev_1
71			integer ilev_2
72			integer ilev_3
73			integer max_lev2
74			integer max_lev3
75			#endif
76	cnh	1.18	CEOP
77	adcroft	1.1
78	heimbach	1.6	#ifdef ALLOW_AUTODIFF_TAMC
79			c-- Initialize storage for the cost function evaluation.
80			CADJ INIT dummytape = common, 1
81			c-- Initialize storage for the outermost loop.
82			CADJ INIT tapelev3 = USER
83	heimbach	1.27	CADJ INIT tapelev_ini_bibj_k = USER
84	heimbach	1.7	#ifdef ALLOW_TAMC_CHECKPOINTING
85	heimbach	1.13	nIter0 = INT( startTime/deltaTClock )
86	heimbach	1.7	ikey_dynamics = 1
87			#endif
88	adcroft	1.10	CALL TIMER_START('ADJOINT SPIN-UP', mythid)
89	heimbach	1.6	#endif
90
91	adcroft	1.1	C-- Set initial conditions (variable arrays)
92	adcroft	1.10	CALL TIMER_START('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid)
93	heimbach	1.6	CALL INITIALISE_VARIA( mythid )
94	adcroft	1.10	CALL TIMER_STOP ('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid)
95	heimbach	1.6
96			#ifndef ALLOW_AUTODIFF_TAMC
97			c-- Dump for start state.
98	adcroft	1.10	CALL TIMER_START('WRITE_STATE [THE_MAIN_LOOP]', mythid)
99	heimbach	1.6	CALL WRITE_STATE( mytime, myiter, mythid )
100	adcroft	1.10	CALL TIMER_STOP ('WRITE_STATE [THE_MAIN_LOOP]', mythid)
101	heimbach	1.6	#endif
102	adcroft	1.9
103			#ifndef EXCLUDE_MONITOR
104			C-- Check status of solution (statistics, cfl, etc...)
105	adcroft	1.10	CALL TIMER_START('MONITOR [THE_MAIN_LOOP]', mythid)
106	adcroft	1.9	CALL MONITOR( myIter, myTime, myThid )
107	adcroft	1.10	CALL TIMER_STOP ('MONITOR [THE_MAIN_LOOP]', mythid)
108	adcroft	1.9	#endif /* EXCLUDE_MONITOR */
109	heimbach	1.6
110	heimbach	1.25	#if (defined (ALLOW_ADJOINT_RUN) \|\| defined (ALLOW_TANGENTLINEAR_RUN))
111	heimbach	1.6	c-- Add control vector for forcing and parameter fields
112			CALL CTRL_MAP_FORCING (mythid)
113			#endif
114
115	adcroft	1.10	#ifdef ALLOW_AUTODIFF_TAMC
116	heimbach	1.6	CALL TIMER_STOP ('ADJOINT SPIN-UP', mythid)
117			_BARRIER
118	adcroft	1.10	#endif
119	heimbach	1.6
120			c-- Do the model integration.
121	adcroft	1.10	CALL TIMER_START('MAIN LOOP [THE_MAIN_LOOP]', mythid)
122	heimbach	1.6
123			c >>>>>>>>>>>>>>>>>>>>>>>>>>> LOOP <<<<<<<<<<<<<<<<<<<<<<<<<<<<
124			c >>>>>>>>>>>>>>>>>>>>>>>>>>> STARTS <<<<<<<<<<<<<<<<<<<<<<<<<<<<
125
126			#ifdef ALLOW_AUTODIFF_TAMC
127			#ifdef ALLOW_TAMC_CHECKPOINTING
128			c-- Implement a three level checkpointing. For a two level
129			c-- checkpointing delete the middle loop; for n levels (n > 3)
130			c-- insert more loops.
131
132			c-- Check the choice of the checkpointing parameters in relation
133			c-- to nTimeSteps: (nchklev_1nchklev_2nchklev_3 .ge. nTimeSteps)
134			if (nchklev_1nchklev_2nchklev_3 .lt. nTimeSteps) then
135			print*
136			print*, ' the_main_loop: TAMC checkpointing parameters'
137			print, ' nchklev_1nchklev_2*nchklev_3 = ',
138			& nchklev_1nchklev_2nchklev_3
139			print*, ' are not consistent with nTimeSteps = ',
140			& nTimeSteps
141			stop ' ... stopped in the_main_loop.'
142			endif
143			max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1
144			max_lev2=nTimeSteps/nchklev_1+1
145	heimbach	1.26
146			c**************************************
147			#ifdef ALLOW_DIVIDED_ADJOINT
148			CADJ loop = divided
149			#endif
150			c**************************************
151	heimbach	1.6
152			do ilev_3 = 1,nchklev_3
153			if(ilev_3.le.max_lev3) then
154	heimbach	1.24	c**************************************
155			#include "checkpoint_lev3_directives.h"
156			c**************************************
157	heimbach	1.6
158			c-- Initialise storage for the middle loop.
159			CADJ INIT tapelev2 = USER
160
161			do ilev_2 = 1,nchklev_2
162			if(ilev_2.le.max_lev2) then
163	heimbach	1.24	c**************************************
164			#include "checkpoint_lev2_directives.h"
165			c**************************************
166	heimbach	1.6
167			c-- Initialize storage for the innermost loop.
168			c-- Always check common block sizes for the checkpointing!
169			CADJ INIT comlev1 = COMMON,nchklev_1
170			CADJ INIT comlev1_bibj = COMMON,nchklev_1nsxnsy*nthreads_chkpt
171			CADJ INIT comlev1_bibj_k = COMMON,nchklev_1nsxnsynrnthreads_chkpt
172	heimbach	1.28	c--
173			#ifdef ALLOW_KPP
174	heimbach	1.6	CADJ INIT comlev1_kpp = COMMON,nchklev_1nsxnsy
175	heimbach	1.28	#endif /* ALLOW_KPP */
176			c--
177			#ifdef ALLOW_GMREDI
178			CADJ INIT comlev1_gmredi_k_gad
179			CADJ & = COMMON,nchklev_1nsxnsynrnthreads_chkpt*maxpass
180			#endif /* ALLOW_GMREDI */
181			c--
182	adcroft	1.21	#ifndef DISABLE_MULTIDIM_ADVECTION
183	heimbach	1.28	CADJ INIT comlev1_bibj_k_gad
184	heimbach	1.20	CADJ & = COMMON,nchklev_1nsxnsynrnthreads_chkpt*maxpass
185	heimbach	1.28	CADJ INIT comlev1_bibj_k_gad_pass
186			CADJ & = COMMON,nchklev_1nsxnsynrnthreads_chkptmaxpassmaxcube
187	adcroft	1.21	#endif /* DISABLE_MULTIDIM_ADVECTION */
188	heimbach	1.28	c--
189	heimbach	1.20	#ifdef ALLOW_BULKFORMULAE
190			CADJ INIT comlev1_exf_1
191			CADJ & = COMMON,nchklev_1snxnsxsnynsy*nthreads_chkpt
192			CADJ INIT comlev1_exf_2
193			CADJ & = COMMON,niter_bulknchklev_1snxnsxsnynsynthreads_chkpt
194	heimbach	1.28	#endif /* ALLOW_BULKFORMULAE */
195	heimbach	1.6
196			do ilev_1 = 1,nchklev_1
197
198			c-- The if-statement below introduces a some flexibility in the
199			c-- choice of the 3-tupel ( nchklev_1, nchklev_2, nchklev_3 ).
200			c--
201			c-- Requirement: nchklev_1nchklev_2nchklev_3 .ge. nTimeSteps .
202
203			iloop = (ilev_3 - 1)nchklev_2nchklev_1 +
204			& (ilev_2 - 1)*nchklev_1 + ilev_1
205
206			if ( iloop .le. nTimeSteps ) then
207
208			#else /* ALLOW_TAMC_CHECKPOINTING undefined */
209			c-- Initialise storage for reference trajectory without TAMC check-
210			c-- pointing.
211			CADJ INIT history = USER
212			CADJ INIT comlev1_bibj = COMMON,nchklev_0nsxnsy*nthreads_chkpt
213			CADJ INIT comlev1_bibj_k = COMMON,nchklev_0nsxnsynrnthreads_chkpt
214			CADJ INIT comlev1_kpp = COMMON,nchklev_0nsxnsy
215
216			C-- RG replace 2 by max of num_v_smooth_Ri
217			CADJ INIT comlev1_kpp_sm = COMMON,nchklev_0nsxnsy*2
218
219			c-- Check the choice of the checkpointing parameters in relation
220			c-- to nTimeSteps: (nchklev_0 .ge. nTimeSteps)
221			if (nchklev_0 .lt. nTimeSteps) then
222			print*
223			print*, ' the_main_loop: TAMC checkpointing parameter ',
224	adcroft	1.19	& 'nchklev_0 = ', nchklev_0
225	heimbach	1.6	print*, ' not consistent with nTimeSteps = ',
226			& nTimeSteps
227			stop ' ... stopped in the_main_loop.'
228			endif
229
230	adcroft	1.10	DO iloop = 1, nTimeSteps
231	heimbach	1.6
232			#endif /* ALLOW_TAMC_CHECKPOINTING */
233
234			#else /* ALLOW_AUTODIFF_TAMC undefined */
235
236			c-- Start the main loop of adjoint_Objfunc. Automatic differentiation
237			c-- NOT enabled.
238	adcroft	1.10	DO iloop = 1, nTimeSteps
239	heimbach	1.6
240			#endif /* ALLOW_AUTODIFF_TAMC */
241
242	adcroft	1.10	c-- >>> Loop body start <<<
243	heimbach	1.6
244			#ifdef ALLOW_TAMC_CHECKPOINTING
245	heimbach	1.14	nIter0 = INT( startTime/deltaTClock )
246	heimbach	1.13	ikey_dynamics = ilev_1
247	heimbach	1.6	#endif
248	heimbach	1.14
249
250	heimbach	1.13	CALL TIMER_START('FORWARD_STEP [THE_MAIN_LOOP]',mythid)
251			CALL FORWARD_STEP( iloop, mytime, myiter, mythid )
252			CALL TIMER_STOP ('FORWARD_STEP [THE_MAIN_LOOP]',mythid)
253	adcroft	1.8
254	heimbach	1.12	#ifdef ALLOW_COST
255	heimbach	1.13	C-- compare model with data and compute cost function
256			C-- this is done after exchanges to allow interpolation
257			CALL TIMER_START('COST_TILE [THE_MAIN_LOOP]',myThid)
258			CALL COST_TILE ( myThid )
259			CALL TIMER_STOP ('COST_TILE [THE_MAIN_LOOP]',myThid)
260	heimbach	1.12	#endif
261
262	heimbach	1.13	c-- >>> Loop body end <<<
263	heimbach	1.6
264			#ifdef ALLOW_AUTODIFF_TAMC
265			#ifdef ALLOW_TAMC_CHECKPOINTING
266			endif
267			enddo
268			endif
269			enddo
270			endif
271			enddo
272			#else
273			enddo
274			#endif
275
276			#else
277			enddo
278			#endif
279
280	heimbach	1.12	#ifdef ALLOW_COST
281			c-- Sum all cost function contributions.
282			call TIMER_START('COST_FINAL [ADJOINT SPIN-DOWN]', mythid)
283			call COST_FINAL ( mythid )
284			call TIMER_STOP ('COST_FINAL [ADJOINT SPIN-DOWN]', mythid)
285			#endif
286
287	heimbach	1.6	_BARRIER
288	adcroft	1.10	CALL TIMER_STOP ('MAIN LOOP [THE_MAIN_LOOP]', mythid)
289	adcroft	1.1
290	adcroft	1.10	END