model/src/the_main_loop.F

C $Header: /u/gcmpack/MITgcm/model/src/the_main_loop.F,v 1.28 2002/11/12 20:42:24 heimbach Exp $

#include "CPP_OPTIONS.h"

CBOP
C     !ROUTINE: THE_MAIN_LOOP
C     !INTERFACE:
      SUBROUTINE THE_MAIN_LOOP( mytime, myiter, mythid )

C     !DESCRIPTION: \bv
C     *================================================================*
C     | SUBROUTINE the_main_loop
C     | o Run the ocean model and evaluate the specified cost function.
C     *================================================================*
C     |
C     | THE_MAIN_LOOP is the toplevel routine for the Tangent Linear and
C     | Adjoint Model Compiler (TAMC). For this purpose the initialization
C     | of the model was split into two parts. Those parameters that do
C     | not depend on a specific model run are set in INITIALISE_FIXED,
C     | whereas those that do depend on the specific realization are
C     | initialized in INITIALISE_VARIA. 
C     | This routine is to be used in conjuction with the MITgcmuv 
C     | checkpoint 37.
C     *================================================================*
C     \ev

C     !USES:
      IMPLICIT NONE
C     == Global variables ==
#include "SIZE.h"
#include "EEPARAMS.h"
#include "PARAMS.h"
#ifdef ALLOW_AUTODIFF_TAMC
# include "tamc.h"
# include "ctrl.h"
# include "ctrl_dummy.h"
# include "cost.h"
# include "DYNVARS.h"
# include "FFIELDS.h"
# include "EOS.h"
# include "GAD.h"
# ifdef ALLOW_PASSIVE_TRACER
#  include "TR1.h"
# endif
# ifdef ALLOW_NONHYDROSTATIC
#  include "CG3D.h"
# endif
# ifdef EXACT_CONSERV
#  include "SURFACE.h"
# endif
#endif

C     !INPUT/OUTPUT PARAMETERS:
C     == Routine arguments ==
C     note: under the multi-threaded model myiter and 
C           mytime are local variables passed around as routine 
C           arguments. Although this is fiddly it saves the need to 
C           impose additional synchronisation points when they are 
C           updated.
C     myIter - iteration counter for this thread
C     myTime - time counter for this thread
C     myThid - thread number for this instance of the routine.
      INTEGER myThid 
      INTEGER myIter
      _RL     myTime

C     !LOCAL VARIABLES:
C     == Local variables ==
      integer iloop
#ifdef ALLOW_TAMC_CHECKPOINTING
      integer ilev_1
      integer ilev_2
      integer ilev_3
      integer max_lev2
      integer max_lev3
#endif
CEOP

#ifdef ALLOW_AUTODIFF_TAMC
c--   Initialize storage for the cost function evaluation.
CADJ  INIT dummytape = common, 1
c--   Initialize storage for the outermost loop.
CADJ  INIT tapelev3 = USER
CADJ  INIT tapelev_ini_bibj_k   = USER
#ifdef ALLOW_TAMC_CHECKPOINTING
       nIter0 = INT( startTime/deltaTClock )
       ikey_dynamics = 1
#endif
      CALL TIMER_START('ADJOINT SPIN-UP', mythid)
#endif

C--   Set initial conditions (variable arrays)
      CALL TIMER_START('INITIALISE_VARIA    [THE_MAIN_LOOP]', mythid)
      CALL INITIALISE_VARIA( mythid )
      CALL TIMER_STOP ('INITIALISE_VARIA    [THE_MAIN_LOOP]', mythid)

#ifndef ALLOW_AUTODIFF_TAMC
c--   Dump for start state.
      CALL TIMER_START('WRITE_STATE         [THE_MAIN_LOOP]', mythid)
      CALL WRITE_STATE( mytime, myiter, mythid )
      CALL TIMER_STOP ('WRITE_STATE         [THE_MAIN_LOOP]', mythid)
#endif

#ifndef EXCLUDE_MONITOR
C--   Check status of solution (statistics, cfl, etc...)
      CALL TIMER_START('MONITOR             [THE_MAIN_LOOP]', mythid)
      CALL MONITOR( myIter, myTime, myThid )
      CALL TIMER_STOP ('MONITOR             [THE_MAIN_LOOP]', mythid)
#endif /* EXCLUDE_MONITOR */

#if (defined (ALLOW_ADJOINT_RUN) || defined (ALLOW_TANGENTLINEAR_RUN))
c--   Add control vector for forcing and parameter fields
      CALL CTRL_MAP_FORCING (mythid)
#endif

#ifdef ALLOW_AUTODIFF_TAMC
      CALL TIMER_STOP ('ADJOINT SPIN-UP', mythid)
      _BARRIER
#endif

c--   Do the model integration.
      CALL TIMER_START('MAIN LOOP           [THE_MAIN_LOOP]', mythid)

c     >>>>>>>>>>>>>>>>>>>>>>>>>>>   LOOP   <<<<<<<<<<<<<<<<<<<<<<<<<<<<
c     >>>>>>>>>>>>>>>>>>>>>>>>>>>  STARTS  <<<<<<<<<<<<<<<<<<<<<<<<<<<<

#ifdef ALLOW_AUTODIFF_TAMC
#ifdef ALLOW_TAMC_CHECKPOINTING
c--   Implement a three level checkpointing. For a two level
c--   checkpointing delete the middle loop; for n levels (n > 3)
c--   insert more loops.

c--   Check the choice of the checkpointing parameters in relation
c--   to nTimeSteps: (nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps)
      if (nchklev_1*nchklev_2*nchklev_3 .lt. nTimeSteps) then
        print*
        print*, ' the_main_loop: TAMC checkpointing parameters'
        print*, '            nchklev_1*nchklev_2*nchklev_3 = ',
     &       nchklev_1*nchklev_2*nchklev_3
        print*, '            are not consistent with nTimeSteps = ',
     &       nTimeSteps
        stop    ' ... stopped in the_main_loop.'
      endif
      max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1
      max_lev2=nTimeSteps/nchklev_1+1

c**************************************
#ifdef ALLOW_DIVIDED_ADJOINT
CADJ loop = divided
#endif
c**************************************

      do ilev_3 = 1,nchklev_3
         if(ilev_3.le.max_lev3) then
c**************************************
#include "checkpoint_lev3_directives.h"
c**************************************

c--     Initialise storage for the middle loop.
CADJ    INIT tapelev2 = USER

        do ilev_2 = 1,nchklev_2
         if(ilev_2.le.max_lev2) then
c**************************************
#include "checkpoint_lev2_directives.h"
c**************************************

c--       Initialize storage for the innermost loop.
c--       Always check common block sizes for the checkpointing!
CADJ INIT comlev1        = COMMON,nchklev_1
CADJ INIT comlev1_bibj   = COMMON,nchklev_1*nsx*nsy*nthreads_chkpt
CADJ INIT comlev1_bibj_k = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt
c--
#ifdef ALLOW_KPP
CADJ INIT comlev1_kpp    = COMMON,nchklev_1*nsx*nsy
#endif /* ALLOW_KPP */
c--
#ifdef ALLOW_GMREDI
CADJ INIT comlev1_gmredi_k_gad
CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
#endif /* ALLOW_GMREDI */
c--
#ifndef DISABLE_MULTIDIM_ADVECTION
CADJ INIT comlev1_bibj_k_gad
CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
CADJ INIT comlev1_bibj_k_gad_pass
CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass*maxcube
#endif /* DISABLE_MULTIDIM_ADVECTION */
c--
#ifdef ALLOW_BULKFORMULAE
CADJ INIT comlev1_exf_1
CADJ &     = COMMON,nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt
CADJ INIT comlev1_exf_2
CADJ &     = COMMON,niter_bulk*nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt
#endif /* ALLOW_BULKFORMULAE */

          do ilev_1 = 1,nchklev_1

c--         The if-statement below introduces a some flexibility in the
c--         choice of the 3-tupel ( nchklev_1, nchklev_2, nchklev_3 ).
c--
c--         Requirement: nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps .

            iloop = (ilev_3 - 1)*nchklev_2*nchklev_1 +
     &              (ilev_2 - 1)*nchklev_1           + ilev_1

            if ( iloop .le. nTimeSteps ) then

#else /* ALLOW_TAMC_CHECKPOINTING  undefined */
c--   Initialise storage for reference trajectory without TAMC check-
c--   pointing.
CADJ INIT history        = USER
CADJ INIT comlev1_bibj   = COMMON,nchklev_0*nsx*nsy*nthreads_chkpt
CADJ INIT comlev1_bibj_k = COMMON,nchklev_0*nsx*nsy*nr*nthreads_chkpt
CADJ INIT comlev1_kpp    = COMMON,nchklev_0*nsx*nsy

C--   RG replace 2 by max of num_v_smooth_Ri
CADJ INIT comlev1_kpp_sm = COMMON,nchklev_0*nsx*nsy*2

c--   Check the choice of the checkpointing parameters in relation
c--   to nTimeSteps: (nchklev_0 .ge. nTimeSteps)
      if (nchklev_0 .lt. nTimeSteps) then
        print*
        print*, ' the_main_loop: TAMC checkpointing parameter ',
     &       'nchklev_0 = ',       nchklev_0
        print*, '                 not consistent with nTimeSteps = ', 
     &       nTimeSteps
        stop    ' ... stopped in the_main_loop.'
      endif

      DO iloop = 1, nTimeSteps

#endif /* ALLOW_TAMC_CHECKPOINTING */

#else /* ALLOW_AUTODIFF_TAMC  undefined */

c--   Start the main loop of adjoint_Objfunc. Automatic differentiation
c--   NOT enabled.
      DO iloop = 1, nTimeSteps

#endif /* ALLOW_AUTODIFF_TAMC */

c--     >>> Loop body start <<<

#ifdef ALLOW_TAMC_CHECKPOINTING
        nIter0 = INT( startTime/deltaTClock )
        ikey_dynamics = ilev_1
#endif

        CALL TIMER_START('FORWARD_STEP        [THE_MAIN_LOOP]',mythid)
        CALL FORWARD_STEP( iloop, mytime, myiter, mythid )
        CALL TIMER_STOP ('FORWARD_STEP        [THE_MAIN_LOOP]',mythid)

#ifdef ALLOW_COST
C--     compare model with data and compute cost function
C--     this is done after exchanges to allow interpolation
        CALL TIMER_START('COST_TILE           [THE_MAIN_LOOP]',myThid)
        CALL COST_TILE  ( myThid )
        CALL TIMER_STOP ('COST_TILE           [THE_MAIN_LOOP]',myThid)
#endif

c--     >>> Loop body end <<<

#ifdef ALLOW_AUTODIFF_TAMC
#ifdef ALLOW_TAMC_CHECKPOINTING
            endif
          enddo
          endif
        enddo
        endif
      enddo
#else
      enddo
#endif

#else
      enddo
#endif

#ifdef ALLOW_COST
c--   Sum all cost function contributions.
      call TIMER_START('COST_FINAL         [ADJOINT SPIN-DOWN]', mythid)
      call COST_FINAL ( mythid )
      call TIMER_STOP ('COST_FINAL         [ADJOINT SPIN-DOWN]', mythid)
#endif

      _BARRIER
      CALL TIMER_STOP ('MAIN LOOP           [THE_MAIN_LOOP]', mythid)

      END
1	heimbach	1.29	C $Header: /u/gcmpack/MITgcm/model/src/the_main_loop.F,v 1.28 2002/11/12 20:42:24 heimbach Exp $
2	adcroft	1.1
3			#include "CPP_OPTIONS.h"
4
5	cnh	1.18	CBOP
6			C !ROUTINE: THE_MAIN_LOOP
7			C !INTERFACE:
8			SUBROUTINE THE_MAIN_LOOP( mytime, myiter, mythid )
9
10			C !DESCRIPTION: \bv
11			C ================================================================
12			C \| SUBROUTINE the_main_loop
13			C \| o Run the ocean model and evaluate the specified cost function.
14			C ================================================================
15			C \|
16			C \| THE_MAIN_LOOP is the toplevel routine for the Tangent Linear and
17			C \| Adjoint Model Compiler (TAMC). For this purpose the initialization
18			C \| of the model was split into two parts. Those parameters that do
19			C \| not depend on a specific model run are set in INITIALISE_FIXED,
20			C \| whereas those that do depend on the specific realization are
21			C \| initialized in INITIALISE_VARIA.
22			C \| This routine is to be used in conjuction with the MITgcmuv
23			C \| checkpoint 37.
24			C ================================================================
25			C \ev
26
27			C !USES:
28			IMPLICIT NONE
29			C == Global variables ==
30	adcroft	1.1	#include "SIZE.h"
31			#include "EEPARAMS.h"
32			#include "PARAMS.h"
33	heimbach	1.6	#ifdef ALLOW_AUTODIFF_TAMC
34	heimbach	1.16	# include "tamc.h"
35			# include "ctrl.h"
36			# include "ctrl_dummy.h"
37			# include "cost.h"
38			# include "DYNVARS.h"
39			# include "FFIELDS.h"
40	heimbach	1.29	# include "EOS.h"
41	adcroft	1.21	# include "GAD.h"
42	heimbach	1.16	# ifdef ALLOW_PASSIVE_TRACER
43			# include "TR1.h"
44			# endif
45			# ifdef ALLOW_NONHYDROSTATIC
46			# include "CG3D.h"
47			# endif
48	heimbach	1.22	# ifdef EXACT_CONSERV
49			# include "SURFACE.h"
50			# endif
51	heimbach	1.6	#endif
52
53	cnh	1.18	C !INPUT/OUTPUT PARAMETERS:
54			C == Routine arguments ==
55			C note: under the multi-threaded model myiter and
56			C mytime are local variables passed around as routine
57			C arguments. Although this is fiddly it saves the need to
58			C impose additional synchronisation points when they are
59			C updated.
60			C myIter - iteration counter for this thread
61			C myTime - time counter for this thread
62			C myThid - thread number for this instance of the routine.
63			INTEGER myThid
64			INTEGER myIter
65			_RL myTime
66	heimbach	1.6
67	cnh	1.18	C !LOCAL VARIABLES:
68			C == Local variables ==
69	heimbach	1.6	integer iloop
70			#ifdef ALLOW_TAMC_CHECKPOINTING
71			integer ilev_1
72			integer ilev_2
73			integer ilev_3
74			integer max_lev2
75			integer max_lev3
76			#endif
77	cnh	1.18	CEOP
78	adcroft	1.1
79	heimbach	1.6	#ifdef ALLOW_AUTODIFF_TAMC
80			c-- Initialize storage for the cost function evaluation.
81			CADJ INIT dummytape = common, 1
82			c-- Initialize storage for the outermost loop.
83			CADJ INIT tapelev3 = USER
84	heimbach	1.27	CADJ INIT tapelev_ini_bibj_k = USER
85	heimbach	1.7	#ifdef ALLOW_TAMC_CHECKPOINTING
86	heimbach	1.13	nIter0 = INT( startTime/deltaTClock )
87	heimbach	1.7	ikey_dynamics = 1
88			#endif
89	adcroft	1.10	CALL TIMER_START('ADJOINT SPIN-UP', mythid)
90	heimbach	1.6	#endif
91
92	adcroft	1.1	C-- Set initial conditions (variable arrays)
93	adcroft	1.10	CALL TIMER_START('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid)
94	heimbach	1.6	CALL INITIALISE_VARIA( mythid )
95	adcroft	1.10	CALL TIMER_STOP ('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid)
96	heimbach	1.6
97			#ifndef ALLOW_AUTODIFF_TAMC
98			c-- Dump for start state.
99	adcroft	1.10	CALL TIMER_START('WRITE_STATE [THE_MAIN_LOOP]', mythid)
100	heimbach	1.6	CALL WRITE_STATE( mytime, myiter, mythid )
101	adcroft	1.10	CALL TIMER_STOP ('WRITE_STATE [THE_MAIN_LOOP]', mythid)
102	heimbach	1.6	#endif
103	adcroft	1.9
104			#ifndef EXCLUDE_MONITOR
105			C-- Check status of solution (statistics, cfl, etc...)
106	adcroft	1.10	CALL TIMER_START('MONITOR [THE_MAIN_LOOP]', mythid)
107	adcroft	1.9	CALL MONITOR( myIter, myTime, myThid )
108	adcroft	1.10	CALL TIMER_STOP ('MONITOR [THE_MAIN_LOOP]', mythid)
109	adcroft	1.9	#endif /* EXCLUDE_MONITOR */
110	heimbach	1.6
111	heimbach	1.25	#if (defined (ALLOW_ADJOINT_RUN) \|\| defined (ALLOW_TANGENTLINEAR_RUN))
112	heimbach	1.6	c-- Add control vector for forcing and parameter fields
113			CALL CTRL_MAP_FORCING (mythid)
114			#endif
115
116	adcroft	1.10	#ifdef ALLOW_AUTODIFF_TAMC
117	heimbach	1.6	CALL TIMER_STOP ('ADJOINT SPIN-UP', mythid)
118			_BARRIER
119	adcroft	1.10	#endif
120	heimbach	1.6
121			c-- Do the model integration.
122	adcroft	1.10	CALL TIMER_START('MAIN LOOP [THE_MAIN_LOOP]', mythid)
123	heimbach	1.6
124			c >>>>>>>>>>>>>>>>>>>>>>>>>>> LOOP <<<<<<<<<<<<<<<<<<<<<<<<<<<<
125			c >>>>>>>>>>>>>>>>>>>>>>>>>>> STARTS <<<<<<<<<<<<<<<<<<<<<<<<<<<<
126
127			#ifdef ALLOW_AUTODIFF_TAMC
128			#ifdef ALLOW_TAMC_CHECKPOINTING
129			c-- Implement a three level checkpointing. For a two level
130			c-- checkpointing delete the middle loop; for n levels (n > 3)
131			c-- insert more loops.
132
133			c-- Check the choice of the checkpointing parameters in relation
134			c-- to nTimeSteps: (nchklev_1nchklev_2nchklev_3 .ge. nTimeSteps)
135			if (nchklev_1nchklev_2nchklev_3 .lt. nTimeSteps) then
136			print*
137			print*, ' the_main_loop: TAMC checkpointing parameters'
138			print, ' nchklev_1nchklev_2*nchklev_3 = ',
139			& nchklev_1nchklev_2nchklev_3
140			print*, ' are not consistent with nTimeSteps = ',
141			& nTimeSteps
142			stop ' ... stopped in the_main_loop.'
143			endif
144			max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1
145			max_lev2=nTimeSteps/nchklev_1+1
146	heimbach	1.26
147			c**************************************
148			#ifdef ALLOW_DIVIDED_ADJOINT
149			CADJ loop = divided
150			#endif
151			c**************************************
152	heimbach	1.6
153			do ilev_3 = 1,nchklev_3
154			if(ilev_3.le.max_lev3) then
155	heimbach	1.24	c**************************************
156			#include "checkpoint_lev3_directives.h"
157			c**************************************
158	heimbach	1.6
159			c-- Initialise storage for the middle loop.
160			CADJ INIT tapelev2 = USER
161
162			do ilev_2 = 1,nchklev_2
163			if(ilev_2.le.max_lev2) then
164	heimbach	1.24	c**************************************
165			#include "checkpoint_lev2_directives.h"
166			c**************************************
167	heimbach	1.6
168			c-- Initialize storage for the innermost loop.
169			c-- Always check common block sizes for the checkpointing!
170			CADJ INIT comlev1 = COMMON,nchklev_1
171			CADJ INIT comlev1_bibj = COMMON,nchklev_1nsxnsy*nthreads_chkpt
172			CADJ INIT comlev1_bibj_k = COMMON,nchklev_1nsxnsynrnthreads_chkpt
173	heimbach	1.28	c--
174			#ifdef ALLOW_KPP
175	heimbach	1.6	CADJ INIT comlev1_kpp = COMMON,nchklev_1nsxnsy
176	heimbach	1.28	#endif /* ALLOW_KPP */
177			c--
178			#ifdef ALLOW_GMREDI
179			CADJ INIT comlev1_gmredi_k_gad
180			CADJ & = COMMON,nchklev_1nsxnsynrnthreads_chkpt*maxpass
181			#endif /* ALLOW_GMREDI */
182			c--
183	adcroft	1.21	#ifndef DISABLE_MULTIDIM_ADVECTION
184	heimbach	1.28	CADJ INIT comlev1_bibj_k_gad
185	heimbach	1.20	CADJ & = COMMON,nchklev_1nsxnsynrnthreads_chkpt*maxpass
186	heimbach	1.28	CADJ INIT comlev1_bibj_k_gad_pass
187			CADJ & = COMMON,nchklev_1nsxnsynrnthreads_chkptmaxpassmaxcube
188	adcroft	1.21	#endif /* DISABLE_MULTIDIM_ADVECTION */
189	heimbach	1.28	c--
190	heimbach	1.20	#ifdef ALLOW_BULKFORMULAE
191			CADJ INIT comlev1_exf_1
192			CADJ & = COMMON,nchklev_1snxnsxsnynsy*nthreads_chkpt
193			CADJ INIT comlev1_exf_2
194			CADJ & = COMMON,niter_bulknchklev_1snxnsxsnynsynthreads_chkpt
195	heimbach	1.28	#endif /* ALLOW_BULKFORMULAE */
196	heimbach	1.6
197			do ilev_1 = 1,nchklev_1
198
199			c-- The if-statement below introduces a some flexibility in the
200			c-- choice of the 3-tupel ( nchklev_1, nchklev_2, nchklev_3 ).
201			c--
202			c-- Requirement: nchklev_1nchklev_2nchklev_3 .ge. nTimeSteps .
203
204			iloop = (ilev_3 - 1)nchklev_2nchklev_1 +
205			& (ilev_2 - 1)*nchklev_1 + ilev_1
206
207			if ( iloop .le. nTimeSteps ) then
208
209			#else /* ALLOW_TAMC_CHECKPOINTING undefined */
210			c-- Initialise storage for reference trajectory without TAMC check-
211			c-- pointing.
212			CADJ INIT history = USER
213			CADJ INIT comlev1_bibj = COMMON,nchklev_0nsxnsy*nthreads_chkpt
214			CADJ INIT comlev1_bibj_k = COMMON,nchklev_0nsxnsynrnthreads_chkpt
215			CADJ INIT comlev1_kpp = COMMON,nchklev_0nsxnsy
216
217			C-- RG replace 2 by max of num_v_smooth_Ri
218			CADJ INIT comlev1_kpp_sm = COMMON,nchklev_0nsxnsy*2
219
220			c-- Check the choice of the checkpointing parameters in relation
221			c-- to nTimeSteps: (nchklev_0 .ge. nTimeSteps)
222			if (nchklev_0 .lt. nTimeSteps) then
223			print*
224			print*, ' the_main_loop: TAMC checkpointing parameter ',
225	adcroft	1.19	& 'nchklev_0 = ', nchklev_0
226	heimbach	1.6	print*, ' not consistent with nTimeSteps = ',
227			& nTimeSteps
228			stop ' ... stopped in the_main_loop.'
229			endif
230
231	adcroft	1.10	DO iloop = 1, nTimeSteps
232	heimbach	1.6
233			#endif /* ALLOW_TAMC_CHECKPOINTING */
234
235			#else /* ALLOW_AUTODIFF_TAMC undefined */
236
237			c-- Start the main loop of adjoint_Objfunc. Automatic differentiation
238			c-- NOT enabled.
239	adcroft	1.10	DO iloop = 1, nTimeSteps
240	heimbach	1.6
241			#endif /* ALLOW_AUTODIFF_TAMC */
242
243	adcroft	1.10	c-- >>> Loop body start <<<
244	heimbach	1.6
245			#ifdef ALLOW_TAMC_CHECKPOINTING
246	heimbach	1.14	nIter0 = INT( startTime/deltaTClock )
247	heimbach	1.13	ikey_dynamics = ilev_1
248	heimbach	1.6	#endif
249	heimbach	1.14
250	heimbach	1.13	CALL TIMER_START('FORWARD_STEP [THE_MAIN_LOOP]',mythid)
251			CALL FORWARD_STEP( iloop, mytime, myiter, mythid )
252			CALL TIMER_STOP ('FORWARD_STEP [THE_MAIN_LOOP]',mythid)
253	adcroft	1.8
254	heimbach	1.12	#ifdef ALLOW_COST
255	heimbach	1.13	C-- compare model with data and compute cost function
256			C-- this is done after exchanges to allow interpolation
257			CALL TIMER_START('COST_TILE [THE_MAIN_LOOP]',myThid)
258			CALL COST_TILE ( myThid )
259			CALL TIMER_STOP ('COST_TILE [THE_MAIN_LOOP]',myThid)
260	heimbach	1.12	#endif
261
262	heimbach	1.13	c-- >>> Loop body end <<<
263	heimbach	1.6
264			#ifdef ALLOW_AUTODIFF_TAMC
265			#ifdef ALLOW_TAMC_CHECKPOINTING
266			endif
267			enddo
268			endif
269			enddo
270			endif
271			enddo
272			#else
273			enddo
274			#endif
275
276			#else
277			enddo
278			#endif
279
280	heimbach	1.12	#ifdef ALLOW_COST
281			c-- Sum all cost function contributions.
282			call TIMER_START('COST_FINAL [ADJOINT SPIN-DOWN]', mythid)
283			call COST_FINAL ( mythid )
284			call TIMER_STOP ('COST_FINAL [ADJOINT SPIN-DOWN]', mythid)
285			#endif
286
287	heimbach	1.6	_BARRIER
288	adcroft	1.10	CALL TIMER_STOP ('MAIN LOOP [THE_MAIN_LOOP]', mythid)
289	adcroft	1.1
290	adcroft	1.10	END