/[MITgcm]/MITgcm/pkg/ecco/the_main_loop.F
ViewVC logotype

Contents of /MITgcm/pkg/ecco/the_main_loop.F

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.35 - (show annotations) (download)
Fri May 5 19:00:28 2006 UTC (18 years, 2 months ago) by ce107
Branch: MAIN
Changes since 1.34: +72 -7 lines
Updates to support PCL performance counters, fix real*4 bug for PAPIS
and enhance PAPI counter support (including IPC per timestep output)

1 C $Header: /u/gcmpack/MITgcm/pkg/ecco/the_main_loop.F,v 1.34 2006/04/28 22:53:14 heimbach Exp $
2 C $Name: $
3
4 #include "PACKAGES_CONFIG.h"
5 #include "CPP_OPTIONS.h"
6
7 #ifdef ALLOW_OBCS
8 # include "OBCS_OPTIONS.h"
9 #endif
10 #ifdef ALLOW_SEAICE
11 # include "SEAICE_OPTIONS.h"
12 #endif
13 #ifdef ALLOW_GMREDI
14 # include "GMREDI_OPTIONS.h"
15 #endif
16
17 subroutine the_main_loop( myTime, myIter, mythid )
18
19 c ==================================================================
20 c SUBROUTINE the_main_loop
21 c ==================================================================
22 c
23 c o Run the ocean model and evaluate the specified cost function.
24 c
25 c *the_main_loop* is the top-level routine for the Tangent Linear and
26 c Adjoint Model Compiler (TAMC). For this purpose, the initialization
27 c of the model was split into two parts. Those parameters that do
28 c not depend on a specific model run are set in *initialise_fixed*,
29 c whereas those that do depend on the specific realization are
30 c initialized in *initialise_varia*. In order to do a so called
31 c checkpointing during the adjoint calculation and to account for the
32 c typical data involved in oceanographic applications a call tree
33 c that is divided into yearly, monthly, daily, and step parts can
34 c be used.
35 c
36 c This routine is to be used in conjuction with the MITgcmuv release
37 c checkpoint 24.
38 c
39 c started: Christian Eckert eckert@mit.edu 30-Jun-1999
40 c
41 c changed: Christian Eckert eckert@mit.edu 14-Jul-1999
42 c
43 c - The call to mapping was moved to initialise_varia,
44 c since this routine has to be called before
45 c ini_predictor.
46 c
47 c Christian Eckert eckert@mit.edu 11-Feb-2000
48 c
49 c - Restructured the code in order to create a package
50 c for the MITgcmUV.
51 c
52 c Patrick Heimbach heimbach@mit.edu 3-Jun-2000
53 c - corrected computation of ikey_dynamics and
54 c added computation of ikey_dynamics for the case
55 c undef ALLOW_TAMC_CHECKPOINTING
56 c
57 c Patrick Heimbach heimbach@mit.edu 6-Jun-2000
58 c - corrected initialisation of comlev1 common blocks
59 c
60 c Dimitris Menemenlis menemenlis@jpl.nasa.gov 26-Feb-2003
61 c - modifications for pkg/seaice
62 c
63 c ==================================================================
64 c SUBROUTINE the_main_loop
65 c ==================================================================
66
67 implicit none
68
69 c == global variables ==
70
71 #include "SIZE.h"
72 #include "EEPARAMS.h"
73 #include "PARAMS.h"
74
75 c**************************************
76 #ifdef ALLOW_AUTODIFF_TAMC
77
78 c These includes are needed for
79 c AD-checkpointing.
80 c They provide the fields to be stored.
81
82 # include "GRID.h"
83 # include "DYNVARS.h"
84 # include "FFIELDS.h"
85 # include "EOS.h"
86
87 # ifdef ALLOW_GENERIC_ADVDIFF
88 # include "GAD.h"
89 # endif
90 # ifdef ALLOW_CD_CODE
91 # include "CD_CODE_VARS.h"
92 # endif
93 # ifdef ALLOW_PTRACERS
94 # include "PTRACERS_SIZE.h"
95 # include "PTRACERS.h"
96 # endif
97 # ifdef ALLOW_NONHYDROSTATIC
98 # include "CG3D.h"
99 # endif
100 # if (defined (EXACT_CONSERV) || defined (NONLIN_FRSURF))
101 # include "SURFACE.h"
102 # endif
103 # ifdef ALLOW_OBCS
104 # include "OBCS.h"
105 # endif
106 # ifdef ALLOW_EXF
107 # include "exf_fields.h"
108 # include "exf_clim_fields.h"
109 # ifdef ALLOW_BULKFORMULAE
110 # include "exf_constants.h"
111 # endif
112 # endif /* ALLOW_EXF */
113 # ifdef ALLOW_SEAICE
114 # include "SEAICE.h"
115 # endif
116 # ifdef ALLOW_KPP
117 # include "KPP.h"
118 # endif
119 # ifdef ALLOW_GMREDI
120 # include "GMREDI.h"
121 # endif
122 # ifdef ALLOW_DIVIDED_ADJOINT_MPI
123 # include "mpif.h"
124 # endif
125
126 # include "tamc.h"
127 # include "ctrl.h"
128 # include "ctrl_dummy.h"
129 # include "cost.h"
130 # include "ecco_cost.h"
131
132 #endif /* ALLOW_AUTODIFF_TAMC */
133 c**************************************
134
135 c == routine arguments ==
136 c note: under the multi-threaded model myiter and
137 c mytime are local variables passed around as routine
138 c arguments. Although this is fiddly it saves the need to
139 c impose additional synchronisation points when they are
140 c updated.
141 c myiter - iteration counter for this thread
142 c mytime - time counter for this thread
143 c mythid - thread number for this instance of the routine.
144 integer mythid
145 integer myiter
146 _RL mytime
147
148 c == local variables ==
149
150 integer bi,bj
151 integer iloop
152 integer mydate(4)
153 #ifdef ALLOW_SNAPSHOTS
154 character yprefix*3
155 #endif
156
157 #ifdef ALLOW_TAMC_CHECKPOINTING
158 integer ilev_1
159 integer ilev_2
160 integer ilev_3
161 integer ilev_4
162 integer max_lev2
163 integer max_lev3
164 integer max_lev4
165 #endif
166
167 #if defined(TIME_PER_TIMESTEP) || defined(USE_PAPI_FLOPS) || defined(USE_PCL_FLOPS)
168 CHARACTER*(MAX_LEN_MBUF) msgBuf
169 #ifdef TIME_PER_TIMESTEP
170 CCE107 common block for per timestep timing
171 C !TIMING VARIABLES
172 C == Timing variables ==
173 REAL*8 utnew, utold, stnew, stold, wtnew, wtold
174 DATA utnew, utold, stnew, stold, wtnew, wtold /6*0.0D0/
175 #endif
176 #ifdef USE_PAPI_FLOPS
177 CCE107 common block for PAPI summary performance
178 #include <fpapi.h>
179 INTEGER*8 flpops, instr
180 DATA flpops, instr /2*0/
181 INTEGER check
182 REAL*4 real_time, proc_time, mflops, ipc
183 DATA real_time, proc_time, mflops, ipc /4*0.0E0/
184 #else
185 #ifdef USE_PCL_FLOPS
186 CCE107 common block for PCL summary performance
187 #include <pclh.f>
188 INTEGER pcl_counter_list(5), flags, nevents, res, ipcl
189 INTEGER*8 i_result(5), descr
190 REAL*8 fp_result(5)
191 COMMON /pclvars/ i_result, descr, fp_result, pcl_counter_list,
192 $ flags, nevents
193 INTEGER nmaxevents
194 PARAMETER (nmaxevents = 61)
195 CHARACTER*22 pcl_counter_name(0:nmaxevents-1)
196 COMMON /pclnames/ pcl_counter_name
197 #endif
198 #endif
199 #endif
200 c-- == end of interface ==
201
202 #ifndef DISABLE_DEBUGMODE
203 IF ( debugLevel .GE. debLevB )
204 & CALL DEBUG_ENTER('THE_MAIN_LOOP',myThid)
205 #endif
206
207 #ifdef ALLOW_AUTODIFF_TAMC
208 c-- Initialize storage for the initialisations.
209 CADJ INIT tapelev_ini_bibj_k = USER
210 CADJ INIT tapelev_init = USER
211 c
212 #if (defined (AUTODIFF_2_LEVEL_CHECKPOINT))
213 CADJ INIT tapelev2 = USER
214 #elif (defined (AUTODIFF_4_LEVEL_CHECKPOINT))
215 CADJ INIT tapelev4 = USER
216 #else
217 CADJ INIT tapelev3 = USER
218 #endif
219 c
220 # ifdef ALLOW_DIVIDED_ADJOINT
221 CADJ INIT onetape = user
222 cphCADJ INIT onetape = common, 1
223 cph We want to avoid common blocks except in the inner loop.
224 cph Reason: the active write and consecutive read may occur
225 cph in separate model executions for which the info
226 cph in common blocks are lost.
227 cph Thus, we can only store real values (no integers)
228 cph because we only have active file handling to real available.
229 # endif
230 # ifdef ALLOW_TAMC_CHECKPOINTING
231 ikey_dynamics = 1
232 # endif
233 #endif /* ALLOW_AUTODIFF_TAMC */
234
235 CALL TIMER_START('ECCO SPIN-UP', mythid)
236
237 c-- Get the current date.
238 call CAL_TIMESTAMP( myiter, mytime, mydate, mythid )
239
240 #ifdef ALLOW_AUTODIFF_TAMC
241 # ifdef NONLIN_FRSURF
242 CADJ STORE hFacC = tapelev_init, key = 1
243 # endif
244 #endif
245
246 C-- Set initial conditions (variable arrays)
247 #ifndef DISABLE_DEBUGMODE
248 IF ( debugLevel .GE. debLevB )
249 & CALL DEBUG_CALL('INITIALISE_VARIA',myThid)
250 #endif
251 CALL TIMER_START('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid)
252 CALL INITIALISE_VARIA( mythid )
253 CALL TIMER_STOP ('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid)
254
255 #ifdef ALLOW_MONITOR
256 #ifdef ALLOW_DEBUG
257 IF (debugMode) CALL DEBUG_CALL('MONITOR',myThid)
258 #endif
259 C-- Check status of solution (statistics, cfl, etc...)
260 CALL TIMER_START('MONITOR [THE_MAIN_LOOP]', mythid)
261 CALL MONITOR( myIter, myTime, myThid )
262 CALL TIMER_STOP ('MONITOR [THE_MAIN_LOOP]', mythid)
263 #endif /* ALLOW_MONITOR */
264
265 C-- Do IO if needed (Dump for start state).
266 #ifdef ALLOW_DEBUG
267 IF (debugMode) CALL DEBUG_CALL('DO_THE_MODEL_IO',myThid)
268 #endif
269
270 #ifdef ALLOW_OFFLINE
271 CALL TIMER_START('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid)
272 CALL OFFLINE_MODEL_IO( myTime, myIter, myThid )
273 CALL TIMER_STOP ('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid)
274 #else
275 CALL TIMER_START('DO_THE_MODEL_IO [THE_MAIN_LOOP]', mythid)
276 CALL DO_THE_MODEL_IO( myTime, myIter, mythid )
277 CALL TIMER_STOP ('DO_THE_MODEL_IO [THE_MAIN_LOOP]', mythid)
278 #endif
279
280 call timer_stop ('ECCO SPIN-UP', mythid)
281 _BARRIER
282
283 #ifdef TIME_PER_TIMESTEP
284 CCE107 Initial call for timers
285 _BEGIN_MASTER( myThid )
286 CALL TIMER_GET_TIME( utold, stold, wtold )
287 _END_MASTER( myThid )
288 #endif
289 #ifdef USE_PAPI_FLOPS
290 CCE107 Initial call for PAPI
291 _BEGIN_MASTER( myThid )
292 #ifdef USE_FLIPS
293 call PAPIF_flips(real_time, proc_time, flpops, mflops, check)
294 #else
295 call PAPIF_flops(real_time, proc_time, flpops, mflops, check)
296 #endif
297 WRITE(msgBuf,'(A34,F10.6)')
298 $ 'Mflop/s before timestepping:', mflops
299 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
300 #ifdef PAPI_VERSION
301 call PAPIF_ipc(real_time, proc_time, instr, ipc, check)
302 WRITE(msgBuf,'(A34,F10.6)')
303 $ 'IPC before timestepping:', ipc
304 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
305 #endif
306 _END_MASTER( myThid )
307 #else
308 #ifdef USE_PCL_FLOPS
309 CCE107 Initial call for PCL
310 _BEGIN_MASTER( myThid )
311 res = PCLstop(descr, i_result, fp_result, nevents)
312 do ipcl = 1, nevents
313 WRITE(msgBuf,'(A22,A26,F10.6)'),
314 $ pcl_counter_name(pcl_counter_list(ipcl)),
315 $ 'before timestepping:', fp_result(ipcl)
316 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
317 enddo
318 res = PCLstart(descr, pcl_counter_list, nevents, flags)
319 _END_MASTER( myThid )
320 #endif
321 #endif
322
323 c-- Do the model integration.
324 call timer_start('ECCO MAIN LOOP',mythid)
325
326 c >>>>>>>>>>>>>>>>>>>>>>>>>>> LOOP <<<<<<<<<<<<<<<<<<<<<<<<<<<<
327 c >>>>>>>>>>>>>>>>>>>>>>>>>>> STARTS <<<<<<<<<<<<<<<<<<<<<<<<<<<<
328
329 #ifdef ALLOW_AUTODIFF_TAMC
330 #ifdef ALLOW_TAMC_CHECKPOINTING
331
332 max_lev4=nTimeSteps/(nchklev_1*nchklev_2*nchklev_3)+1
333 max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1
334 max_lev2=nTimeSteps/nchklev_1+1
335
336 c**************************************
337 #ifdef ALLOW_DIVIDED_ADJOINT
338 CADJ loop = divided
339 #endif
340 c**************************************
341
342 #ifdef AUTODIFF_4_LEVEL_CHECKPOINT
343 do ilev_4 = 1,nchklev_4
344 if(ilev_4.le.max_lev4) then
345 c**************************************
346 #include "checkpoint_lev4_directives.h"
347 c**************************************
348 c-- Initialise storage for the middle loop.
349 CADJ INIT tapelev3 = USER
350 #endif /* AUTODIFF_4_LEVEL_CHECKPOINT */
351
352 #ifndef AUTODIFF_2_LEVEL_CHECKPOINT
353 do ilev_3 = 1,nchklev_3
354 if(ilev_3.le.max_lev3) then
355 c**************************************
356 #include "checkpoint_lev3_directives.h"
357 c**************************************
358 c-- Initialise storage for the middle loop.
359 CADJ INIT tapelev2 = USER
360 #endif /* AUTODIFF_2_LEVEL_CHECKPOINT */
361
362 do ilev_2 = 1,nchklev_2
363 if(ilev_2.le.max_lev2) then
364 c**************************************
365 #include "checkpoint_lev2_directives.h"
366 c**************************************
367
368 c**************************************
369 #ifdef ALLOW_AUTODIFF_TAMC
370 c-- Initialize storage for the innermost loop.
371 c-- Always check common block sizes for the checkpointing!
372 c--
373 CADJ INIT comlev1 = COMMON,nchklev_1
374 CADJ INIT comlev1_bibj = COMMON,nchklev_1*nsx*nsy*nthreads_chkpt
375 CADJ INIT comlev1_bibj_k = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt
376 c--
377 # ifdef ALLOW_KPP
378 CADJ INIT comlev1_kpp = COMMON,nchklev_1*nsx*nsy
379 CADJ INIT comlev1_kpp_k = COMMON,nchklev_1*nsx*nsy*nr
380 # endif /* ALLOW_KPP */
381 c--
382 # ifdef ALLOW_GMREDI
383 CADJ INIT comlev1_gmredi_k_gad
384 CADJ & = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
385 # endif /* ALLOW_GMREDI */
386 c--
387 # ifdef ALLOW_PTRACERS
388 CADJ INIT comlev1_bibj_ptracers = COMMON,
389 CADJ & nchklev_1*nsx*nsy*nthreads_chkpt*PTRACERS_num
390 # endif /* ALLOW_PTRACERS */
391 c--
392 cph Now also needed by seaice
393 cph# ifndef DISABLE_MULTIDIM_ADVECTION
394 CADJ INIT comlev1_bibj_k_gad
395 CADJ & = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
396 CADJ INIT comlev1_bibj_k_gad_pass
397 CADJ & = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass*maxcube
398 cph# endif /* DISABLE_MULTIDIM_ADVECTION */
399 c--
400 # if (defined (ALLOW_EXF) && defined (ALLOW_BULKFORMULAE))
401 CADJ INIT comlev1_exf_1
402 CADJ & = COMMON,nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt
403 CADJ INIT comlev1_exf_2
404 CADJ & = COMMON,niter_bulk*nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt
405 # endif
406 c--
407 # ifdef ALLOW_SEAICE
408 # ifdef SEAICE_ALLOW_DYNAMICS
409 CADJ INIT comlev1_lsr = COMMON,nchklev_1*2
410 # endif
411 # ifdef SEAICE_MULTILEVEL
412 CADJ INIT comlev1_multdim
413 CADJ & = COMMON,nchklev_1*nsx*nsy*nthreads_chkpt*multdim
414 # endif
415 # endif /* ALLOW_SEAICE */
416 c--
417 #endif /* ALLOW_AUTODIFF_TAMC */
418 c**************************************
419
420 do ilev_1 = 1,nchklev_1
421
422 c-- The if-statement below introduces a some flexibility in the
423 c-- choice of the 3-tupel ( nchklev_1, nchklev_2, nchklev_3 ).
424 c--
425 c-- Requirement: nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps .
426
427 iloop = (ilev_2 - 1)*nchklev_1 + ilev_1
428 #ifndef AUTODIFF_2_LEVEL_CHECKPOINT
429 & + (ilev_3 - 1)*nchklev_2*nchklev_1
430 #endif
431 #ifdef AUTODIFF_4_LEVEL_CHECKPOINT
432 & + (ilev_4 - 1)*nchklev_3*nchklev_2*nchklev_1
433 #endif
434
435 if ( iloop .le. nTimeSteps ) then
436
437 #else /* ALLOW_TAMC_CHECKPOINTING undefined */
438 c-- Initialise storage for the reference trajectory without TAMC check-
439 c-- pointing.
440 CADJ INIT history = USER
441 CADJ INIT comlev1_bibj = COMMON,nchklev_0*nsx*nsy*nthreads_chkpt
442 CADJ INIT comlev1_bibj_k = COMMON,nchklev_0*nsx*nsy*nr*nthreads_chkpt
443 CADJ INIT comlev1_kpp = COMMON,nchklev_0*nsx*nsy
444
445 c-- Check the choice of the checkpointing parameters in relation
446 c-- to nTimeSteps: (nchklev_0 .ge. nTimeSteps)
447 if (nchklev_0 .lt. nTimeSteps) then
448 print*
449 print*, ' the_main_loop: ',
450 & 'TAMC checkpointing parameter nchklev_0 = ',
451 & nchklev_0
452 print*, ' is not consistent with nTimeSteps = ',
453 & nTimeSteps
454 stop ' ... stopped in the_main_loop.'
455 endif
456
457 do iloop = 1, nTimeSteps
458
459 #endif /* ALLOW_TAMC_CHECKPOINTING */
460
461 #else /* ALLOW_AUTODIFF_TAMC undefined */
462 c-- Start the main loop of ecco_Objfunc. Automatic differentiation is
463 c-- NOT enabled.
464 do iloop = 1, nTimeSteps
465 #endif /* ALLOW_AUTODIFF_TAMC */
466
467 #ifdef ALLOW_TAMC_CHECKPOINTING
468 nIter0 = NINT( (startTime-baseTime)/deltaTClock )
469 ikey_dynamics = ilev_1
470 #endif
471
472 c-- Set the model iteration counter and the model time.
473 myiter = nIter0 + (iloop-1)
474 mytime = startTime + float(iloop-1)*deltaTclock
475
476 #ifdef ALLOW_AUTODIFF_TAMC
477 CALL AUTODIFF_INADMODE_UNSET( myThid )
478 #endif
479
480
481 C-- Switch on/off diagnostics for snap-shot output:
482 #ifdef ALLOW_DIAGNOSTICS
483 IF ( useDiagnostics ) THEN
484 CALL DIAGNOSTICS_SWITCH_ONOFF( myTime, myIter, myThid )
485 ENDIF
486 #endif
487
488 C-- State-variables diagnostics
489 IF ( usediagnostics ) THEN
490 CALL TIMER_START('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid)
491 CALL DO_STATEVARS_DIAGS( myTime, 0, myIter, myThid )
492 CALL TIMER_STOP ('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid)
493 ENDIF
494
495 #ifdef ALLOW_COST
496
497 c-- Accumulate time averages of temperature, salinity, and SSH.
498 call timer_start('COST_AVERAGESFIELDS [ECCO MAIN]', mythid)
499 call cost_averagesFields( mytime, mythid )
500 call timer_stop ('COST_AVERAGESFIELDS [ECCO MAIN]', mythid)
501 #ifdef ALLOW_COST_ATLANTIC
502 c-- Compute meridional heat transport
503 call timer_start('cost_atlantic [ECCO MAIN]', mythid)
504 call cost_atlantic( mytime, myiter,mythid )
505 call timer_stop ('cost_atlantic [ECCO MAIN]', mythid)
506 #endif
507 #endif /* ALLOW_COST */
508
509 #ifdef ALLOW_AUTODIFF_TAMC
510 c**************************************
511 #include "checkpoint_lev1_directives.h"
512 #include "checkpoint_lev1_template.h"
513 c**************************************
514 #endif
515
516 #ifndef DISABLE_DEBUGMODE
517 IF ( debugLevel .GE. debLevB )
518 & CALL DEBUG_CALL('EXF_GETFORCING',myThid)
519 #endif
520 CALL TIMER_START('EXF_GETFORCING [FORWARD_STEP]',mythid)
521 CALL EXF_GETFORCING( mytime, myiter, mythid )
522 CALL TIMER_STOP ('EXF_GETFORCING [FORWARD_STEP]',mythid)
523 # ifdef NONLIN_FRSURF
524 CADJ STORE hFacC = comlev1, key = ikey_dynamics
525 CADJ STORE SST = comlev1, key = ikey_dynamics
526 CADJ STORE SSS = comlev1, key = ikey_dynamics
527 # ifdef SHORTWAVE_HEATING
528 CADJ STORE Qsw = comlev1, key = ikey_dynamics
529 # endif
530 # endif
531
532 #ifdef ALLOW_AUTODIFF_TAMC
533 # if (defined (ALLOW_AUTODIFF_MONITOR))
534 CALL DUMMY_IN_STEPPING( myTime, myIter, myThid )
535 # endif
536 #endif
537
538 #ifdef ALLOW_SEAICE
539 cph this simple runtime flag causes a lot of recomp.
540 IF ( useSEAICE ) THEN
541 #ifndef DISABLE_DEBUGMODE
542 IF ( debugLevel .GE. debLevB )
543 & CALL DEBUG_CALL('SEAICE_MODEL',myThid)
544 #endif
545 CALL TIMER_START('SEAICE_MODEL [FORWARD_STEP]',myThid)
546 CALL SEAICE_MODEL( myTime, myIter, myThid )
547 CALL TIMER_STOP ('SEAICE_MODEL [FORWARD_STEP]',myThid)
548 #ifdef ALLOW_COST_ICE
549 CALL COST_ICE_TEST ( myTime, myIter, myThid )
550 #endif
551 ENDIF
552 #endif /* ALLOW_SEAICE */
553
554 #ifdef ALLOW_AUTODIFF_TAMC
555 # ifdef ALLOW_PTRACERS
556 cph this replaces _bibj storing of ptracer within thermodynamics
557 CADJ STORE ptracer = comlev1, key = ikey_dynamics
558 # endif
559 #endif
560
561 #ifdef ALLOW_EBM
562 IF ( useEBM ) THEN
563 # ifdef ALLOW_DEBUG
564 IF ( debugLevel .GE. debLevB )
565 & CALL DEBUG_CALL('EBM',myThid)
566 # endif
567 CALL TIMER_START('EBM [FORWARD_STEP]',mythid)
568 CALL EBM_DRIVER ( myTime, myIter, myThid )
569 CALL TIMER_STOP ('EBM [FORWARD_STEP]',mythid)
570 ENDIF
571 #endif
572
573 #ifdef ALLOW_PTRACERS
574 # ifdef ALLOW_GCHEM
575 IF ( useGCHEM ) THEN
576 #ifdef ALLOW_DEBUG
577 IF ( debugLevel .GE. debLevB )
578 & CALL DEBUG_CALL('GCHEM_FIELDS_LOAD',myThid)
579 #endif /* ALLOW_DEBUG */
580 CALL GCHEM_FIELDS_LOAD( mytime, myiter, mythid )
581 ENDIF
582 # endif
583 #endif
584
585 C-- Step forward fields and calculate time tendency terms.
586
587 #ifdef ALLOW_DEBUG
588 IF ( debugLevel .GE. debLevB )
589 & CALL DEBUG_CALL('DO_ATMOSPHERIC_PHYS',myThid)
590 #endif
591 CALL TIMER_START('DO_ATMOSPHERIC_PHYS [FORWARD_STEP]',mythid)
592 CALL DO_ATMOSPHERIC_PHYS( myTime, myIter, myThid )
593 CALL TIMER_STOP ('DO_ATMOSPHERIC_PHYS [FORWARD_STEP]',mythid)
594
595 #ifdef ALLOW_AUTODIFF_TAMC
596 # ifdef EXACT_CONSERV
597 cphCADJ STORE empmr = comlev1, key = ikey_dynamics
598 cphCADJ STORE pmepr = comlev1, key = ikey_dynamics
599 # endif
600 # ifdef NONLIN_FRSURF
601 cph-test
602 CADJ STORE hFacC = comlev1, key = ikey_dynamics
603 # endif
604 #endif /* ALLOW_AUTODIFF_TAMC */
605
606 #ifndef ALLOW_OFFLINE
607 #ifdef ALLOW_DEBUG
608 IF ( debugLevel .GE. debLevB )
609 & CALL DEBUG_CALL('DO_OCEANIC_PHYS',myThid)
610 #endif
611 CALL TIMER_START('DO_OCEANIC_PHYS [FORWARD_STEP]',mythid)
612 CALL DO_OCEANIC_PHYS( myTime, myIter, myThid )
613 CALL TIMER_STOP ('DO_OCEANIC_PHYS [FORWARD_STEP]',mythid)
614 #ifdef ALLOW_AUTODIFF_TAMC
615 CADJ STORE EmPmR = comlev1, key = ikey_dynamics
616 # ifdef EXACT_CONSERV
617 CADJ STORE pmepr = comlev1, key = ikey_dynamics
618 # endif
619 #endif
620 #endif
621
622 #ifdef ALLOW_AUTODIFF_TAMC
623 # ifdef NONLIN_FRSURF
624 cph-test
625 CADJ STORE hFac_surfC = comlev1, key = ikey_dynamics
626 CADJ STORE hfac_surfs = comlev1, key = ikey_dynamics
627 CADJ STORE hfac_surfw = comlev1, key = ikey_dynamics
628 CADJ STORE hFacC, hFacS, hFacW
629 CADJ & = comlev1, key = ikey_dynamics
630 CADJ STORE recip_hFacC, recip_hFacS, recip_hFacW
631 CADJ & = comlev1, key = ikey_dynamics
632 c
633 CADJ STORE surfaceforcingu = comlev1, key = ikey_dynamics
634 CADJ STORE surfaceforcingv = comlev1, key = ikey_dynamics
635 # endif
636 #endif /* ALLOW_AUTODIFF_TAMC */
637
638 #ifdef ALLOW_GCHEM
639 C GCHEM package is an interface for any bio-geochemical or
640 C ecosystem model you would like to include.
641 C If GCHEM_SEPARATE_FORCING is not defined, you are
642 C responsible for computing tendency terms for passive
643 C tracers and storing them on a 3DxNumPtracers-array called
644 C gchemTendency in GCHEM_CALC_TENDENCY. This tendency is then added
645 C to gPtr in ptracers_forcing later-on.
646 C If GCHEM_SEPARATE_FORCING is defined, you are reponsible for
647 C UPDATING ptracers directly in GCHEM_FORCING_SEP. This amounts
648 C to a completely separate time step that you have to implement
649 C yourself (Eulerian seems to be fine in most cases).
650 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
651 C CAVEAT: Up to now, when GCHEM is turned on the field ptracerForcingSurf,
652 C which is needed for KPP is not set properly. ptracerForcingSurf must
653 C be treated differently depending on whether GCHEM_SEPARATE_FORCING
654 C is define or not. TBD.
655 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
656 IF ( useGCHEM ) THEN
657 #ifdef ALLOW_DEBUG
658 IF ( debugLevel .GE. debLevB )
659 & CALL DEBUG_CALL('GCHEM_CALC_TENDENCY',myThid)
660 #endif
661 CALL TIMER_START('GCHEM_CALC_TENDENCY [FORWARD_STEP]',myThid)
662 CALL GCHEM_CALC_TENDENCY( myTime, myIter, myThid )
663 CALL TIMER_STOP ('GCHEM_CALC_TENDENCY [FORWARD_STEP]',myThid)
664 ENDIF
665 #endif /* ALLOW_GCHEM */
666
667 #ifdef ALLOW_AUTODIFF_TAMC
668 cph needed to be moved here from do_oceanic_physics
669 cph to be visible down the road
670 c
671 CADJ STORE surfaceForcingS = comlev1, key = ikey_dynamics
672 CADJ STORE surfaceForcingT = comlev1, key = ikey_dynamics
673 CADJ STORE surfaceForcingTice = comlev1, key = ikey_dynamics
674 ctest(
675 CADJ STORE IVDConvCount = comlev1, key = ikey_dynamics
676 ctest)
677 # ifdef ALLOW_PTRACERS
678 CADJ STORE surfaceForcingPtr = comlev1, key = ikey_dynamics
679 # endif
680 c
681 # ifdef ALLOW_GMREDI
682 CADJ STORE Kwx = comlev1, key = ikey_dynamics
683 CADJ STORE Kwy = comlev1, key = ikey_dynamics
684 CADJ STORE Kwz = comlev1, key = ikey_dynamics
685 # ifdef GM_BOLUS_ADVEC
686 CADJ STORE GM_PsiX = comlev1, key = ikey_dynamics
687 CADJ STORE GM_PsiY = comlev1, key = ikey_dynamics
688 # endif
689 # endif
690 c
691 # ifdef ALLOW_KPP
692 CADJ STORE KPPghat = comlev1, key = ikey_dynamics
693 CADJ STORE KPPfrac = comlev1, key = ikey_dynamics
694 CADJ STORE KPPdiffKzS = comlev1, key = ikey_dynamics
695 CADJ STORE KPPdiffKzT = comlev1, key = ikey_dynamics
696 # endif
697 #endif /* ALLOW_AUTODIFF_TAMC */
698
699 #ifdef ALLOW_AUTODIFF_TAMC
700 # ifdef NONLIN_FRSURF
701 CADJ STORE etaH = comlev1, key = ikey_dynamics
702 # ifdef ALLOW_CD_CODE
703 CADJ STORE etanm1 = comlev1, key = ikey_dynamics
704 # endif
705 # endif
706 #endif /* ALLOW_AUTODIFF_TAMC */
707
708 IF ( .NOT.staggerTimeStep ) THEN
709 #ifdef ALLOW_DEBUG
710 IF ( debugLevel .GE. debLevB )
711 & CALL DEBUG_CALL('THERMODYNAMICS',myThid)
712 #endif
713 CALL TIMER_START('THERMODYNAMICS [FORWARD_STEP]',mythid)
714 CALL THERMODYNAMICS( myTime, myIter, myThid )
715 CALL TIMER_STOP ('THERMODYNAMICS [FORWARD_STEP]',mythid)
716 C-- if not staggerTimeStep: end
717 ENDIF
718
719 #ifdef ALLOW_AUTODIFF_TAMC
720 # ifdef NONLIN_FRSURF
721 CADJ STORE hFacC = comlev1, key = ikey_dynamics
722 CADJ STORE hFacS = comlev1, key = ikey_dynamics
723 CADJ STORE hFacW = comlev1, key = ikey_dynamics
724 CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics
725 CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics
726 CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics
727 CADJ STORE etaN = comlev1, key = ikey_dynamics
728 # endif
729 #endif
730
731 C-- Step forward fields and calculate time tendency terms.
732 #ifndef ALLOW_OFFLINE
733 #ifndef ALLOW_AUTODIFF_TAMC
734 IF ( momStepping ) THEN
735 #endif
736 #ifdef ALLOW_DEBUG
737 IF ( debugLevel .GE. debLevB )
738 & CALL DEBUG_CALL('DYNAMICS',myThid)
739 #endif
740 CALL TIMER_START('DYNAMICS [FORWARD_STEP]',mythid)
741 CALL DYNAMICS( myTime, myIter, myThid )
742 CALL TIMER_STOP ('DYNAMICS [FORWARD_STEP]',mythid)
743 #ifndef ALLOW_AUTODIFF_TAMC
744 ENDIF
745 #endif
746 #endif
747
748 #ifdef ALLOW_AUTODIFF_TAMC
749 # ifdef NONLIN_FRSURF
750 cph-test
751 CADJ STORE gU, gV = comlev1, key = ikey_dynamics
752 # endif
753 #endif
754
755 C-- Update time-counter
756 myIter = nIter0 + iLoop
757 myTime = startTime + deltaTClock * float(iLoop)
758
759 #ifdef ALLOW_MNC
760 C Update the default next iter for MNC
761 IF ( useMNC ) THEN
762 CALL MNC_CW_CITER_SETG( 1, 1, -1, myIter , myThid )
763
764 C TODO: Logic should be added here so that users can specify, on
765 C a per-citer-group basis, when it is time to update the
766 C "current" (and not just the "next") iteration
767
768 C TODO: the following is just a temporary band-aid (mostly, for
769 C Baylor) until someone writes a routine that better handles time
770 C boundaries such as weeks, months, years, etc.
771 IF ( mnc_filefreq .GT. 0 ) THEN
772 IF (DIFFERENT_MULTIPLE(mnc_filefreq,myTime,deltaTClock))
773 & THEN
774 CALL MNC_CW_CITER_SETG( 1, 1, myIter, -1 , myThid )
775 ENDIF
776 ENDIF
777 ENDIF
778 #endif
779
780 C-- Update geometric factors:
781 #ifdef NONLIN_FRSURF
782 C- update hfacC,W,S and recip_hFac according to etaH(n+1) :
783 IF ( nonlinFreeSurf.GT.0) THEN
784 IF ( select_rStar.GT.0 ) THEN
785 # ifndef DISABLE_RSTAR_CODE
786 # ifdef ALLOW_AUTODIFF_TAMC
787 cph-test
788 CADJ STORE hFacC = comlev1, key = ikey_dynamics
789 CADJ STORE hFacS = comlev1, key = ikey_dynamics
790 CADJ STORE hFacW = comlev1, key = ikey_dynamics
791 CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics
792 CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics
793 CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics
794 # endif
795 CALL TIMER_START('UPDATE_R_STAR [FORWARD_STEP]',myThid)
796 CALL UPDATE_R_STAR( myTime, myIter, myThid )
797 CALL TIMER_STOP ('UPDATE_R_STAR [FORWARD_STEP]',myThid)
798 # ifdef ALLOW_AUTODIFF_TAMC
799 cph-test
800 CADJ STORE hFacC = comlev1, key = ikey_dynamics
801 CADJ STORE hFacS = comlev1, key = ikey_dynamics
802 CADJ STORE hFacW = comlev1, key = ikey_dynamics
803 CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics
804 CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics
805 CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics
806 # endif
807 # endif /* DISABLE_RSTAR_CODE */
808 ELSE
809 #ifdef ALLOW_AUTODIFF_TAMC
810 CADJ STORE hFac_surfC, hFac_surfS, hFac_surfW
811 CADJ & = comlev1, key = ikey_dynamics
812 #endif
813 CALL TIMER_START('UPDATE_SURF_DR [FORWARD_STEP]',myThid)
814 CALL UPDATE_SURF_DR( myTime, myIter, myThid )
815 CALL TIMER_STOP ('UPDATE_SURF_DR [FORWARD_STEP]',myThid)
816 ENDIF
817 ENDIF
818 # ifdef ALLOW_AUTODIFF_TAMC
819 cph-test
820 CADJ STORE hFacC = comlev1, key = ikey_dynamics
821 CADJ STORE hFacS = comlev1, key = ikey_dynamics
822 CADJ STORE hFacW = comlev1, key = ikey_dynamics
823 CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics
824 CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics
825 CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics
826 # endif
827 C- update also CG2D matrix (and preconditioner)
828 IF ( momStepping .AND. nonlinFreeSurf.GT.2 ) THEN
829 CALL TIMER_START('UPDATE_CG2D [FORWARD_STEP]',myThid)
830 CALL UPDATE_CG2D( myTime, myIter, myThid )
831 CALL TIMER_STOP ('UPDATE_CG2D [FORWARD_STEP]',myThid)
832 ENDIF
833 #endif /* NONLIN_FRSURF */
834
835 C-- Apply Filters to u*,v* before SOLVE_FOR_PRESSURE
836 #ifdef ALLOW_SHAP_FILT
837 IF (useSHAP_FILT .AND. shap_filt_uvStar) THEN
838 CALL TIMER_START('SHAP_FILT [FORWARD_STEP]',myThid)
839 IF (implicDiv2Dflow.LT.1.) THEN
840 C-- Explicit+Implicit part of the Barotropic Flow Divergence
841 C => Filtering of uVel,vVel is necessary
842 CALL SHAP_FILT_APPLY_UV( uVel,vVel,
843 & myTime, myIter, myThid )
844 ENDIF
845 CALL SHAP_FILT_APPLY_UV( gU,gV,myTime,myIter,myThid)
846 CALL TIMER_STOP ('SHAP_FILT [FORWARD_STEP]',myThid)
847 ENDIF
848 #endif
849 #ifdef ALLOW_ZONAL_FILT
850 IF (useZONAL_FILT .AND. zonal_filt_uvStar) THEN
851 CALL TIMER_START('ZONAL_FILT_APPLY [FORWARD_STEP]',myThid)
852 IF (implicDiv2Dflow.LT.1.) THEN
853 C-- Explicit+Implicit part of the Barotropic Flow Divergence
854 C => Filtering of uVel,vVel is necessary
855 CALL ZONAL_FILT_APPLY_UV( uVel, vVel, myThid )
856 ENDIF
857 CALL ZONAL_FILT_APPLY_UV( gU, gV, myThid )
858 CALL TIMER_STOP ('ZONAL_FILT_APPLY [FORWARD_STEP]',myThid)
859 ENDIF
860 #endif
861
862 C-- Solve elliptic equation(s).
863 C Two-dimensional only for conventional hydrostatic or
864 C three-dimensional for non-hydrostatic and/or IGW scheme.
865 #ifndef ALLOW_OFFLINE
866 IF ( momStepping ) THEN
867 #ifdef ALLOW_AUTODIFF_TAMC
868 # ifdef NONLIN_FRSURF
869 CADJ STORE uvel, vvel
870 CADJ & = comlev1, key = ikey_dynamics
871 CADJ STORE empmr,hfacs,hfacw
872 CADJ & = comlev1, key = ikey_dynamics
873 # endif
874 #endif
875 CALL TIMER_START('SOLVE_FOR_PRESSURE [FORWARD_STEP]',myThid)
876 CALL SOLVE_FOR_PRESSURE(myTime, myIter, myThid)
877 CALL TIMER_STOP ('SOLVE_FOR_PRESSURE [FORWARD_STEP]',myThid)
878 ENDIF
879 #endif
880
881 C-- Correct divergence in flow field and cycle time-stepping momentum
882 c IF ( momStepping ) THEN
883 #ifndef ALLOW_OFFLINE
884 CALL TIMER_START('UV_CORRECTION_STEP [FORWARD_STEP]',myThid)
885 CALL MOMENTUM_CORRECTION_STEP(myTime, myIter, myThid)
886 CALL TIMER_STOP ('UV_CORRECTION_STEP [FORWARD_STEP]',myThid)
887 #endif
888 c ENDIF
889
890 #ifdef EXACT_CONSERV
891 IF (exactConserv) THEN
892 #ifdef ALLOW_AUTODIFF_TAMC
893 cph-test
894 cphCADJ STORE etaH = comlev1, key = ikey_dynamics
895 #endif
896 C-- Update etaH(n+1) :
897 CALL TIMER_START('UPDATE_ETAH [FORWARD_STEP]',mythid)
898 CALL UPDATE_ETAH( myTime, myIter, myThid )
899 CALL TIMER_STOP ('UPDATE_ETAH [FORWARD_STEP]',mythid)
900 ENDIF
901 #endif /* EXACT_CONSERV */
902
903 #ifdef NONLIN_FRSURF
904 IF ( select_rStar.NE.0 ) THEN
905 # ifndef DISABLE_RSTAR_CODE
906 C-- r* : compute the future level thickness according to etaH(n+1)
907 CALL TIMER_START('CALC_R_STAR [FORWARD_STEP]',mythid)
908 CALL CALC_R_STAR(etaH, myTime, myIter, myThid )
909 CALL TIMER_STOP ('CALC_R_STAR [FORWARD_STEP]',mythid)
910 # endif /* DISABLE_RSTAR_CODE */
911 ELSEIF ( nonlinFreeSurf.GT.0) THEN
912 C-- compute the future surface level thickness according to etaH(n+1)
913 # ifdef ALLOW_AUTODIFF_TAMC
914 CADJ STORE etaH = comlev1, key = ikey_dynamics
915 # endif
916 CALL TIMER_START('CALC_SURF_DR [FORWARD_STEP]',mythid)
917 CALL CALC_SURF_DR(etaH, myTime, myIter, myThid )
918 CALL TIMER_STOP ('CALC_SURF_DR [FORWARD_STEP]',mythid)
919 ENDIF
920 # ifdef ALLOW_AUTODIFF_TAMC
921 cph-test
922 CADJ STORE hFac_surfC = comlev1, key = ikey_dynamics
923 # endif
924 #endif /* NONLIN_FRSURF */
925
926 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
927 IF ( staggerTimeStep ) THEN
928 C-- do exchanges of U,V (needed for multiDim) when using stagger time-step :
929 #ifdef ALLOW_DEBUG
930 IF ( debugLevel .GE. debLevB )
931 & CALL DEBUG_CALL('DO_STAGGER_FIELDS_EXCH.',myThid)
932 #endif
933 CALL TIMER_START('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid)
934 CALL DO_STAGGER_FIELDS_EXCHANGES( myTime, myIter, myThid )
935 CALL TIMER_STOP ('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid)
936
937 C-- State-variables diagnostics
938 IF ( usediagnostics ) THEN
939 CALL TIMER_START('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid)
940 CALL DO_STATEVARS_DIAGS( myTime, 1, myIter, myThid )
941 CALL TIMER_STOP ('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid)
942 ENDIF
943
944 #ifdef ALLOW_DEBUG
945 IF ( debugLevel .GE. debLevB )
946 & CALL DEBUG_CALL('THERMODYNAMICS',myThid)
947 #endif
948 CALL TIMER_START('THERMODYNAMICS [FORWARD_STEP]',mythid)
949 CALL THERMODYNAMICS( myTime, myIter, myThid )
950 CALL TIMER_STOP ('THERMODYNAMICS [FORWARD_STEP]',mythid)
951
952 C-- if staggerTimeStep: end
953 ENDIF
954 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
955
956 #ifdef ALLOW_AUTODIFF_TAMC
957 cph This is needed because convective_adjustment calls
958 cph find_rho which may use pressure()
959 CADJ STORE totphihyd = comlev1, key = ikey_dynamics
960 #endif
961 C-- Cycle time-stepping Tracers arrays (T,S,+pTracers)
962 CALL TIMER_START('TS_CORRECTION_STEP [FORWARD_STEP]',myThid)
963 CALL TRACERS_CORRECTION_STEP(myTime, myIter, myThid)
964 CALL TIMER_STOP ('TS_CORRECTION_STEP [FORWARD_STEP]',myThid)
965
966 #ifdef ALLOW_GCHEM
967 C Add separate timestepping of chemical/biological/forcing
968 C of ptracers here in GCHEM_FORCING_SEP
969 IF ( useGCHEM ) THEN
970 #ifdef ALLOW_DEBUG
971 IF ( debugLevel .GE. debLevB )
972 & CALL DEBUG_CALL('GCHEM_FORCING_SEP',myThid)
973 #endif /* ALLOW_DEBUG */
974 CALL TIMER_START('GCHEM_FORCING_SEP [FORWARD_STEP]',myThid)
975 CALL GCHEM_FORCING_SEP( myTime,myIter,myThid )
976 CALL TIMER_STOP ('GCHEM_FORCING_SEP [FORWARD_STEP]',myThid)
977 ENDIF
978 #endif /* ALLOW_GCHEM */
979
980 C-- Do "blocking" sends and receives for tendency "overlap" terms
981 c CALL TIMER_START('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid)
982 c CALL DO_GTERM_BLOCKING_EXCHANGES( myThid )
983 c CALL TIMER_STOP ('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid)
984
985 C-- Do "blocking" sends and receives for field "overlap" terms
986 CALL TIMER_START('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid)
987 CALL DO_FIELDS_BLOCKING_EXCHANGES( myThid )
988 CALL TIMER_STOP ('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid)
989
990 #ifdef ALLOW_FLT
991 C-- Calculate float trajectories
992 IF (useFLT) THEN
993 CALL TIMER_START('FLOATS [FORWARD_STEP]',myThid)
994 CALL FLT_MAIN(myIter,myTime, myThid)
995 CALL TIMER_STOP ('FLOATS [FORWARD_STEP]',myThid)
996 ENDIF
997 #endif
998
999 #ifdef ALLOW_AUTODIFF_TAMC
1000 CALL AUTODIFF_INADMODE_SET( myThid )
1001 #endif
1002
1003 C-- State-variables time-averaging
1004 CALL TIMER_START('DO_STATEVARS_TAVE [FORWARD_STEP]',myThid)
1005 CALL DO_STATEVARS_TAVE( myTime, myIter, myThid )
1006 CALL TIMER_STOP ('DO_STATEVARS_TAVE [FORWARD_STEP]',myThid)
1007
1008 #ifndef ALLOW_OFFLINE
1009 #ifdef ALLOW_MONITOR
1010 C-- Check status of solution (statistics, cfl, etc...)
1011 CALL TIMER_START('MONITOR [FORWARD_STEP]',myThid)
1012 CALL MONITOR( myIter, myTime, myThid )
1013 CALL TIMER_STOP ('MONITOR [FORWARD_STEP]',myThid)
1014 #endif /* ALLOW_MONITOR */
1015 #endif
1016
1017 C-- Do IO if needed.
1018 #ifdef ALLOW_OFFLINE
1019 CALL TIMER_START('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid)
1020 CALL OFFLINE_MODEL_IO( myTime, myIter, myThid )
1021 CALL TIMER_STOP ('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid)
1022 #else
1023 CALL TIMER_START('DO_THE_MODEL_IO [FORWARD_STEP]',myThid)
1024 CALL DO_THE_MODEL_IO( myTime, myIter, myThid )
1025 CALL TIMER_STOP ('DO_THE_MODEL_IO [FORWARD_STEP]',myThid)
1026 #endif
1027
1028 C-- Save state for restarts
1029 CALL TIMER_START('WRITE_CHECKPOINT [FORWARD_STEP]',myThid)
1030 CALL PACKAGES_WRITE_PICKUP(
1031 I .FALSE., myTime, myIter, myThid )
1032 #ifndef ALLOW_OFFLINE
1033 CALL WRITE_CHECKPOINT(
1034 I .FALSE., myTime, myIter, myThid )
1035 #endif
1036 CALL TIMER_STOP ('WRITE_CHECKPOINT [FORWARD_STEP]',myThid)
1037
1038 #ifdef TIME_PER_TIMESTEP
1039 CCE107 Time per timestep information
1040 _BEGIN_MASTER( myThid )
1041 CALL TIMER_GET_TIME( utnew, stnew, wtnew )
1042 WRITE(msgBuf,'(A34,3F10.6,I8)')
1043 $ 'User, system and wallclock time:', utnew - utold,
1044 $ stnew - stold, wtnew - wtold, iloop
1045 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1046 utold = utnew
1047 stold = stnew
1048 wtold = wtnew
1049 _END_MASTER( myThid )
1050 #endif
1051 #ifdef USE_PAPI_FLOPS
1052 CCE107 PAPI summary performance
1053 _BEGIN_MASTER( myThid )
1054 #ifdef USE_FLIPS
1055 call PAPIF_flips(real_time, proc_time, flpops, mflops, check)
1056 #else
1057 call PAPIF_flops(real_time, proc_time, flpops, mflops, check)
1058 #endif
1059 WRITE(msgBuf,'(F10.6,A34,I8)')
1060 $ mflops, 'Mflop/s during timestep ', iloop
1061 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1062 _END_MASTER( myThid )
1063 #ifdef PAPI_VERSION
1064 call PAPIF_ipc(real_time, proc_time, instr, ipc, check)
1065 WRITE(msgBuf,'(F10.6,A34,I8)')
1066 $ ipc, 'IPC during timestep ', iloop
1067 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1068 #endif
1069 _END_MASTER( myThid )
1070 #else
1071 #ifdef USE_PCL_FLOPS
1072 CCE107 PCL summary performance
1073 _BEGIN_MASTER( myThid )
1074 res = PCLstop(descr, i_result, fp_result, nevents)
1075 do ipcl = 1, nevents
1076 WRITE(msgBuf,'(F10.6,A2,A22,A17,I8)'), fp_result(ipcl),
1077 $ ' ', pcl_counter_name(pcl_counter_list(ipcl)),
1078 $ 'during timestep ', iloop
1079 CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1080 enddo
1081 res = PCLstart(descr, pcl_counter_list, nevents, flags)
1082 _END_MASTER( myThid )
1083 #endif
1084 #endif
1085
1086 #ifdef ALLOW_AUTODIFF_TAMC
1087 #ifdef ALLOW_TAMC_CHECKPOINTING
1088 endif
1089 enddo
1090 endif
1091 enddo
1092 #ifndef AUTODIFF_2_LEVEL_CHECKPOINT
1093 endif
1094 enddo
1095 #endif
1096 #ifdef AUTODIFF_4_LEVEL_CHECKPOINT
1097 endif
1098 enddo
1099 #endif
1100 c
1101 #else /* ndef ALLOW_TAMC_CHECKPOINTING */
1102 enddo
1103 #endif /* ALLOW_TAMC_CHECKPOINTING */
1104
1105 #else /* ndef ALLOW_AUTODIFF_TAMC */
1106 enddo
1107 #endif /* ALLOW_AUTODIFF_TAMC */
1108
1109 _BARRIER
1110 call timer_stop ('ECCO MAIN LOOP', mythid)
1111
1112 call timer_start('ECCO SPIN-DOWN', mythid)
1113
1114 #ifdef ALLOW_COST
1115
1116 #ifdef ALLOW_DIVIDED_ADJOINT
1117 CADJ STORE mytime = onetape
1118 #endif
1119 c-- Accumulate time averages of temperature, salinity, and SSH.
1120 #ifndef DISABLE_DEBUGMODE
1121 IF ( debugLevel .GE. debLevB )
1122 & CALL DEBUG_CALL('cost_averagesfields',myThid)
1123 #endif
1124 call timer_start('cost_averagesfields [ECCO SPIN-DOWN]', mythid)
1125 call cost_averagesfields( mytime, mythid )
1126 call timer_stop ('cost_averagesfields [ECCO SPIN-DOWN]', mythid)
1127 #ifdef ALLOW_DIVIDED_ADJOINT
1128 c**************************************
1129 #include "cost_averages_bar_directives.h"
1130 c**************************************
1131 #endif
1132
1133 #ifdef ALLOW_COST_ATLANTIC
1134 c-- Compute meridional heat transport
1135 #ifndef DISABLE_DEBUGMODE
1136 IF ( debugLevel .GE. debLevB )
1137 & CALL DEBUG_CALL('cost_atlantic',myThid)
1138 #endif
1139 call timer_start('cost_atlantic [ECCO SPIN-DOWN]', mythid)
1140 call cost_atlantic( mytime, myiter,mythid )
1141 call timer_stop ('cost_atlantic [ECCO SPIN-DOWN]', mythid)
1142 #endif
1143
1144 c-- Compute the cost function contribution of the boundary forcing,
1145 c-- i.e. heat flux, salt flux, zonal and meridional wind stress.
1146 #ifndef DISABLE_DEBUGMODE
1147 IF ( debugLevel .GE. debLevB )
1148 & CALL DEBUG_CALL('cost_forcing',myThid)
1149 #endif
1150 call timer_start('cost_forcing [ECCO SPIN-DOWN]', mythid)
1151 call cost_forcing( myiter, mytime, mythid )
1152 call timer_stop ('cost_forcing [ECCO SPIN-DOWN]', mythid)
1153
1154 c-- Compute cost function contribution of Temperature and Salinity.
1155 #ifndef DISABLE_DEBUGMODE
1156 IF ( debugLevel .GE. debLevB )
1157 & CALL DEBUG_CALL('cost_hyd',myThid)
1158 #endif
1159 call timer_start('cost_hyd [ECCO SPIN-DOWN]', mythid)
1160 call cost_hyd( myiter, mytime, mythid )
1161 call timer_stop ('cost_hyd [ECCO SPIN-DOWN]', mythid)
1162
1163 #ifdef ALLOW_SEAICE
1164 #ifndef DISABLE_DEBUGMODE
1165 IF ( debugLevel .GE. debLevB )
1166 & CALL DEBUG_CALL('seaice_cost_driver',myThid)
1167 #endif
1168 call timer_start('seaice_cost_driver [ECCO SPIN-DOWN]', mythid)
1169 call seaice_cost_driver( myiter, mytime, mythid )
1170 call timer_stop ('seaice_cost_driver [ECCO SPIN-DOWN]', mythid)
1171 #endif
1172
1173 #ifdef ALLOW_OBCS_COST_CONTRIBUTION
1174 #ifndef DISABLE_DEBUGMODE
1175 IF ( debugLevel .GE. debLevB )
1176 & CALL DEBUG_CALL('cost_obcs',myThid)
1177 #endif
1178 call timer_start('cost_obcs [ECCO SPIN-DOWN]', mythid)
1179 call cost_obcs( myiter, mytime, mythid )
1180 call timer_stop ('cost_obcs [ECCO SPIN-DOWN]', mythid)
1181 #endif
1182
1183 #ifdef ALLOW_CURMTR_COST_CONTRIBUTION
1184 #ifndef DISABLE_DEBUGMODE
1185 IF ( debugLevel .GE. debLevB )
1186 & CALL DEBUG_CALL('cost_curmtr',myThid)
1187 #endif
1188 call timer_start('cost_curmtr [ECCO SPIN-DOWN]', mythid)
1189 call cost_curmtr( myiter, mytime, mythid )
1190 call timer_stop ('cost_curmtr [ECCO SPIN-DOWN]', mythid)
1191 #endif
1192
1193 c-- Compute cost function contribution of SSH.
1194 #ifdef ALLOW_SSH_COST_CONTRIBUTION
1195 #ifndef DISABLE_DEBUGMODE
1196 IF ( debugLevel .GE. debLevB )
1197 & CALL DEBUG_CALL('cost_ssh',myThid)
1198 #endif
1199 call timer_start('cost_ssh [ECCO SPIN-DOWN]', mythid)
1200 call cost_ssh( myiter, mytime, mythid )
1201 call timer_stop ('cost_ssh [ECCO SPIN-DOWN]', mythid)
1202 #endif
1203
1204 c-- Compute cost function contribution of drifter's velocities.
1205 #ifdef ALLOW_DRIFTER_COST_CONTRIBUTION
1206 #ifndef DISABLE_DEBUGMODE
1207 IF ( debugLevel .GE. debLevB )
1208 & CALL DEBUG_CALL('cost_drifter',myThid)
1209 #endif
1210 call timer_start('cost_drifter [ECCO SPIN-DOWN]', mythid)
1211 call cost_drifter( myiter, mytime, mythid )
1212 call timer_stop ('cost_drifter [ECCO SPIN-DOWN]', mythid)
1213 #endif
1214
1215 c-- Compute cost function contribution of wind stress observations.
1216 #ifdef ALLOW_SCAT_COST_CONTRIBUTION
1217 #ifndef DISABLE_DEBUGMODE
1218 IF ( debugLevel .GE. debLevB )
1219 & CALL DEBUG_CALL('cost_scat',myThid)
1220 #endif
1221 call timer_start('cost_scat [ECCO SPIN-DOWN]', mythid)
1222 call cost_scat( myiter, mytime, mythid )
1223 call timer_stop ('cost_scat [ECCO SPIN-DOWN]', mythid)
1224 #endif
1225
1226 c-- Compute cost function contribution of drift between the first
1227 c and the last year.
1228 #ifdef ALLOW_DRIFT_COST_CONTRIBUTION
1229 #ifndef DISABLE_DEBUGMODE
1230 IF ( debugLevel .GE. debLevB )
1231 & CALL DEBUG_CALL('cost_drift',myThid)
1232 #endif
1233 call timer_start('cost_drift [ECCO SPIN-DOWN]', mythid)
1234 call cost_drift( myiter, mytime, mythid )
1235 call timer_stop ('cost_drift [ECCO SPIN-DOWN]', mythid)
1236 #endif
1237 #ifdef ALLOW_DRIFTW_COST_CONTRIBUTION
1238 #ifndef DISABLE_DEBUGMODE
1239 IF ( debugLevel .GE. debLevB )
1240 & CALL DEBUG_CALL('cost_driftw',myThid)
1241 #endif
1242 call timer_start('cost_driftw [ECCO SPIN-DOWN]', mythid)
1243 call cost_driftw( myiter, mytime, mythid )
1244 call timer_stop ('cost_driftw [ECCO SPIN-DOWN]', mythid)
1245 #endif
1246 _BARRIER
1247
1248 c-- Compute initial vs. final T/S deviation
1249 #ifdef ALLOW_COST_INI_FIN
1250 call timer_start('cost_ini_fin [ECCO SPIN-DOWN]', mythid)
1251 call cost_theta_ini_fin( myiter, mytime, mythid )
1252 call cost_salt_ini_fin( myiter, mytime, mythid )
1253 call timer_stop ('cost_ini_fin [ECCO SPIN-DOWN]', mythid)
1254 #endif
1255 _BARRIER
1256
1257 c-- Eddy stress penalty term
1258 #ifdef ALLOW_COST_TAU_EDDY
1259 call timer_start('cost_tau_eddy [ECCO SPIN-DOWN]', mythid)
1260 call cost_tau_eddy( mythid )
1261 call timer_stop ('cost_tau_eddy [ECCO SPIN-DOWN]', mythid)
1262 #endif
1263
1264 c-- Sum all cost function contributions.
1265 #ifndef DISABLE_DEBUGMODE
1266 IF ( debugLevel .GE. debLevB )
1267 & CALL DEBUG_CALL('cost_final',myThid)
1268 #endif
1269 call timer_start('COST_FINAL [ECCO SPIN-DOWN]', mythid)
1270 call cost_final( mythid )
1271 call timer_stop ('COST_FINAL [ECCO SPIN-DOWN]', mythid)
1272
1273 #endif /* ALLOW_COST */
1274
1275 call timer_stop ('ECCO SPIN-DOWN', mythid)
1276
1277 #ifndef DISABLE_DEBUGMODE
1278 IF ( debugLevel .GE. debLevB )
1279 & CALL DEBUG_LEAVE('THE_MAIN_LOOP',myThid)
1280 #endif
1281
1282 return
1283 end
1284

  ViewVC Help
Powered by ViewVC 1.1.22