1 |
C $Header: /u/gcmpack/MITgcm/pkg/ecco/the_main_loop.F,v 1.34 2006/04/28 22:53:14 heimbach Exp $ |
2 |
C $Name: $ |
3 |
|
4 |
#include "PACKAGES_CONFIG.h" |
5 |
#include "CPP_OPTIONS.h" |
6 |
|
7 |
#ifdef ALLOW_OBCS |
8 |
# include "OBCS_OPTIONS.h" |
9 |
#endif |
10 |
#ifdef ALLOW_SEAICE |
11 |
# include "SEAICE_OPTIONS.h" |
12 |
#endif |
13 |
#ifdef ALLOW_GMREDI |
14 |
# include "GMREDI_OPTIONS.h" |
15 |
#endif |
16 |
|
17 |
subroutine the_main_loop( myTime, myIter, mythid ) |
18 |
|
19 |
c ================================================================== |
20 |
c SUBROUTINE the_main_loop |
21 |
c ================================================================== |
22 |
c |
23 |
c o Run the ocean model and evaluate the specified cost function. |
24 |
c |
25 |
c *the_main_loop* is the top-level routine for the Tangent Linear and |
26 |
c Adjoint Model Compiler (TAMC). For this purpose, the initialization |
27 |
c of the model was split into two parts. Those parameters that do |
28 |
c not depend on a specific model run are set in *initialise_fixed*, |
29 |
c whereas those that do depend on the specific realization are |
30 |
c initialized in *initialise_varia*. In order to do a so called |
31 |
c checkpointing during the adjoint calculation and to account for the |
32 |
c typical data involved in oceanographic applications a call tree |
33 |
c that is divided into yearly, monthly, daily, and step parts can |
34 |
c be used. |
35 |
c |
36 |
c This routine is to be used in conjuction with the MITgcmuv release |
37 |
c checkpoint 24. |
38 |
c |
39 |
c started: Christian Eckert eckert@mit.edu 30-Jun-1999 |
40 |
c |
41 |
c changed: Christian Eckert eckert@mit.edu 14-Jul-1999 |
42 |
c |
43 |
c - The call to mapping was moved to initialise_varia, |
44 |
c since this routine has to be called before |
45 |
c ini_predictor. |
46 |
c |
47 |
c Christian Eckert eckert@mit.edu 11-Feb-2000 |
48 |
c |
49 |
c - Restructured the code in order to create a package |
50 |
c for the MITgcmUV. |
51 |
c |
52 |
c Patrick Heimbach heimbach@mit.edu 3-Jun-2000 |
53 |
c - corrected computation of ikey_dynamics and |
54 |
c added computation of ikey_dynamics for the case |
55 |
c undef ALLOW_TAMC_CHECKPOINTING |
56 |
c |
57 |
c Patrick Heimbach heimbach@mit.edu 6-Jun-2000 |
58 |
c - corrected initialisation of comlev1 common blocks |
59 |
c |
60 |
c Dimitris Menemenlis menemenlis@jpl.nasa.gov 26-Feb-2003 |
61 |
c - modifications for pkg/seaice |
62 |
c |
63 |
c ================================================================== |
64 |
c SUBROUTINE the_main_loop |
65 |
c ================================================================== |
66 |
|
67 |
implicit none |
68 |
|
69 |
c == global variables == |
70 |
|
71 |
#include "SIZE.h" |
72 |
#include "EEPARAMS.h" |
73 |
#include "PARAMS.h" |
74 |
|
75 |
c************************************** |
76 |
#ifdef ALLOW_AUTODIFF_TAMC |
77 |
|
78 |
c These includes are needed for |
79 |
c AD-checkpointing. |
80 |
c They provide the fields to be stored. |
81 |
|
82 |
# include "GRID.h" |
83 |
# include "DYNVARS.h" |
84 |
# include "FFIELDS.h" |
85 |
# include "EOS.h" |
86 |
|
87 |
# ifdef ALLOW_GENERIC_ADVDIFF |
88 |
# include "GAD.h" |
89 |
# endif |
90 |
# ifdef ALLOW_CD_CODE |
91 |
# include "CD_CODE_VARS.h" |
92 |
# endif |
93 |
# ifdef ALLOW_PTRACERS |
94 |
# include "PTRACERS_SIZE.h" |
95 |
# include "PTRACERS.h" |
96 |
# endif |
97 |
# ifdef ALLOW_NONHYDROSTATIC |
98 |
# include "CG3D.h" |
99 |
# endif |
100 |
# if (defined (EXACT_CONSERV) || defined (NONLIN_FRSURF)) |
101 |
# include "SURFACE.h" |
102 |
# endif |
103 |
# ifdef ALLOW_OBCS |
104 |
# include "OBCS.h" |
105 |
# endif |
106 |
# ifdef ALLOW_EXF |
107 |
# include "exf_fields.h" |
108 |
# include "exf_clim_fields.h" |
109 |
# ifdef ALLOW_BULKFORMULAE |
110 |
# include "exf_constants.h" |
111 |
# endif |
112 |
# endif /* ALLOW_EXF */ |
113 |
# ifdef ALLOW_SEAICE |
114 |
# include "SEAICE.h" |
115 |
# endif |
116 |
# ifdef ALLOW_KPP |
117 |
# include "KPP.h" |
118 |
# endif |
119 |
# ifdef ALLOW_GMREDI |
120 |
# include "GMREDI.h" |
121 |
# endif |
122 |
# ifdef ALLOW_DIVIDED_ADJOINT_MPI |
123 |
# include "mpif.h" |
124 |
# endif |
125 |
|
126 |
# include "tamc.h" |
127 |
# include "ctrl.h" |
128 |
# include "ctrl_dummy.h" |
129 |
# include "cost.h" |
130 |
# include "ecco_cost.h" |
131 |
|
132 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
133 |
c************************************** |
134 |
|
135 |
c == routine arguments == |
136 |
c note: under the multi-threaded model myiter and |
137 |
c mytime are local variables passed around as routine |
138 |
c arguments. Although this is fiddly it saves the need to |
139 |
c impose additional synchronisation points when they are |
140 |
c updated. |
141 |
c myiter - iteration counter for this thread |
142 |
c mytime - time counter for this thread |
143 |
c mythid - thread number for this instance of the routine. |
144 |
integer mythid |
145 |
integer myiter |
146 |
_RL mytime |
147 |
|
148 |
c == local variables == |
149 |
|
150 |
integer bi,bj |
151 |
integer iloop |
152 |
integer mydate(4) |
153 |
#ifdef ALLOW_SNAPSHOTS |
154 |
character yprefix*3 |
155 |
#endif |
156 |
|
157 |
#ifdef ALLOW_TAMC_CHECKPOINTING |
158 |
integer ilev_1 |
159 |
integer ilev_2 |
160 |
integer ilev_3 |
161 |
integer ilev_4 |
162 |
integer max_lev2 |
163 |
integer max_lev3 |
164 |
integer max_lev4 |
165 |
#endif |
166 |
|
167 |
#if defined(TIME_PER_TIMESTEP) || defined(USE_PAPI_FLOPS) || defined(USE_PCL_FLOPS) |
168 |
CHARACTER*(MAX_LEN_MBUF) msgBuf |
169 |
#ifdef TIME_PER_TIMESTEP |
170 |
CCE107 common block for per timestep timing |
171 |
C !TIMING VARIABLES |
172 |
C == Timing variables == |
173 |
REAL*8 utnew, utold, stnew, stold, wtnew, wtold |
174 |
DATA utnew, utold, stnew, stold, wtnew, wtold /6*0.0D0/ |
175 |
#endif |
176 |
#ifdef USE_PAPI_FLOPS |
177 |
CCE107 common block for PAPI summary performance |
178 |
#include <fpapi.h> |
179 |
INTEGER*8 flpops, instr |
180 |
DATA flpops, instr /2*0/ |
181 |
INTEGER check |
182 |
REAL*4 real_time, proc_time, mflops, ipc |
183 |
DATA real_time, proc_time, mflops, ipc /4*0.0E0/ |
184 |
#else |
185 |
#ifdef USE_PCL_FLOPS |
186 |
CCE107 common block for PCL summary performance |
187 |
#include <pclh.f> |
188 |
INTEGER pcl_counter_list(5), flags, nevents, res, ipcl |
189 |
INTEGER*8 i_result(5), descr |
190 |
REAL*8 fp_result(5) |
191 |
COMMON /pclvars/ i_result, descr, fp_result, pcl_counter_list, |
192 |
$ flags, nevents |
193 |
INTEGER nmaxevents |
194 |
PARAMETER (nmaxevents = 61) |
195 |
CHARACTER*22 pcl_counter_name(0:nmaxevents-1) |
196 |
COMMON /pclnames/ pcl_counter_name |
197 |
#endif |
198 |
#endif |
199 |
#endif |
200 |
c-- == end of interface == |
201 |
|
202 |
#ifndef DISABLE_DEBUGMODE |
203 |
IF ( debugLevel .GE. debLevB ) |
204 |
& CALL DEBUG_ENTER('THE_MAIN_LOOP',myThid) |
205 |
#endif |
206 |
|
207 |
#ifdef ALLOW_AUTODIFF_TAMC |
208 |
c-- Initialize storage for the initialisations. |
209 |
CADJ INIT tapelev_ini_bibj_k = USER |
210 |
CADJ INIT tapelev_init = USER |
211 |
c |
212 |
#if (defined (AUTODIFF_2_LEVEL_CHECKPOINT)) |
213 |
CADJ INIT tapelev2 = USER |
214 |
#elif (defined (AUTODIFF_4_LEVEL_CHECKPOINT)) |
215 |
CADJ INIT tapelev4 = USER |
216 |
#else |
217 |
CADJ INIT tapelev3 = USER |
218 |
#endif |
219 |
c |
220 |
# ifdef ALLOW_DIVIDED_ADJOINT |
221 |
CADJ INIT onetape = user |
222 |
cphCADJ INIT onetape = common, 1 |
223 |
cph We want to avoid common blocks except in the inner loop. |
224 |
cph Reason: the active write and consecutive read may occur |
225 |
cph in separate model executions for which the info |
226 |
cph in common blocks are lost. |
227 |
cph Thus, we can only store real values (no integers) |
228 |
cph because we only have active file handling to real available. |
229 |
# endif |
230 |
# ifdef ALLOW_TAMC_CHECKPOINTING |
231 |
ikey_dynamics = 1 |
232 |
# endif |
233 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
234 |
|
235 |
CALL TIMER_START('ECCO SPIN-UP', mythid) |
236 |
|
237 |
c-- Get the current date. |
238 |
call CAL_TIMESTAMP( myiter, mytime, mydate, mythid ) |
239 |
|
240 |
#ifdef ALLOW_AUTODIFF_TAMC |
241 |
# ifdef NONLIN_FRSURF |
242 |
CADJ STORE hFacC = tapelev_init, key = 1 |
243 |
# endif |
244 |
#endif |
245 |
|
246 |
C-- Set initial conditions (variable arrays) |
247 |
#ifndef DISABLE_DEBUGMODE |
248 |
IF ( debugLevel .GE. debLevB ) |
249 |
& CALL DEBUG_CALL('INITIALISE_VARIA',myThid) |
250 |
#endif |
251 |
CALL TIMER_START('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid) |
252 |
CALL INITIALISE_VARIA( mythid ) |
253 |
CALL TIMER_STOP ('INITIALISE_VARIA [THE_MAIN_LOOP]', mythid) |
254 |
|
255 |
#ifdef ALLOW_MONITOR |
256 |
#ifdef ALLOW_DEBUG |
257 |
IF (debugMode) CALL DEBUG_CALL('MONITOR',myThid) |
258 |
#endif |
259 |
C-- Check status of solution (statistics, cfl, etc...) |
260 |
CALL TIMER_START('MONITOR [THE_MAIN_LOOP]', mythid) |
261 |
CALL MONITOR( myIter, myTime, myThid ) |
262 |
CALL TIMER_STOP ('MONITOR [THE_MAIN_LOOP]', mythid) |
263 |
#endif /* ALLOW_MONITOR */ |
264 |
|
265 |
C-- Do IO if needed (Dump for start state). |
266 |
#ifdef ALLOW_DEBUG |
267 |
IF (debugMode) CALL DEBUG_CALL('DO_THE_MODEL_IO',myThid) |
268 |
#endif |
269 |
|
270 |
#ifdef ALLOW_OFFLINE |
271 |
CALL TIMER_START('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid) |
272 |
CALL OFFLINE_MODEL_IO( myTime, myIter, myThid ) |
273 |
CALL TIMER_STOP ('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid) |
274 |
#else |
275 |
CALL TIMER_START('DO_THE_MODEL_IO [THE_MAIN_LOOP]', mythid) |
276 |
CALL DO_THE_MODEL_IO( myTime, myIter, mythid ) |
277 |
CALL TIMER_STOP ('DO_THE_MODEL_IO [THE_MAIN_LOOP]', mythid) |
278 |
#endif |
279 |
|
280 |
call timer_stop ('ECCO SPIN-UP', mythid) |
281 |
_BARRIER |
282 |
|
283 |
#ifdef TIME_PER_TIMESTEP |
284 |
CCE107 Initial call for timers |
285 |
_BEGIN_MASTER( myThid ) |
286 |
CALL TIMER_GET_TIME( utold, stold, wtold ) |
287 |
_END_MASTER( myThid ) |
288 |
#endif |
289 |
#ifdef USE_PAPI_FLOPS |
290 |
CCE107 Initial call for PAPI |
291 |
_BEGIN_MASTER( myThid ) |
292 |
#ifdef USE_FLIPS |
293 |
call PAPIF_flips(real_time, proc_time, flpops, mflops, check) |
294 |
#else |
295 |
call PAPIF_flops(real_time, proc_time, flpops, mflops, check) |
296 |
#endif |
297 |
WRITE(msgBuf,'(A34,F10.6)') |
298 |
$ 'Mflop/s before timestepping:', mflops |
299 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
300 |
#ifdef PAPI_VERSION |
301 |
call PAPIF_ipc(real_time, proc_time, instr, ipc, check) |
302 |
WRITE(msgBuf,'(A34,F10.6)') |
303 |
$ 'IPC before timestepping:', ipc |
304 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
305 |
#endif |
306 |
_END_MASTER( myThid ) |
307 |
#else |
308 |
#ifdef USE_PCL_FLOPS |
309 |
CCE107 Initial call for PCL |
310 |
_BEGIN_MASTER( myThid ) |
311 |
res = PCLstop(descr, i_result, fp_result, nevents) |
312 |
do ipcl = 1, nevents |
313 |
WRITE(msgBuf,'(A22,A26,F10.6)'), |
314 |
$ pcl_counter_name(pcl_counter_list(ipcl)), |
315 |
$ 'before timestepping:', fp_result(ipcl) |
316 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
317 |
enddo |
318 |
res = PCLstart(descr, pcl_counter_list, nevents, flags) |
319 |
_END_MASTER( myThid ) |
320 |
#endif |
321 |
#endif |
322 |
|
323 |
c-- Do the model integration. |
324 |
call timer_start('ECCO MAIN LOOP',mythid) |
325 |
|
326 |
c >>>>>>>>>>>>>>>>>>>>>>>>>>> LOOP <<<<<<<<<<<<<<<<<<<<<<<<<<<< |
327 |
c >>>>>>>>>>>>>>>>>>>>>>>>>>> STARTS <<<<<<<<<<<<<<<<<<<<<<<<<<<< |
328 |
|
329 |
#ifdef ALLOW_AUTODIFF_TAMC |
330 |
#ifdef ALLOW_TAMC_CHECKPOINTING |
331 |
|
332 |
max_lev4=nTimeSteps/(nchklev_1*nchklev_2*nchklev_3)+1 |
333 |
max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1 |
334 |
max_lev2=nTimeSteps/nchklev_1+1 |
335 |
|
336 |
c************************************** |
337 |
#ifdef ALLOW_DIVIDED_ADJOINT |
338 |
CADJ loop = divided |
339 |
#endif |
340 |
c************************************** |
341 |
|
342 |
#ifdef AUTODIFF_4_LEVEL_CHECKPOINT |
343 |
do ilev_4 = 1,nchklev_4 |
344 |
if(ilev_4.le.max_lev4) then |
345 |
c************************************** |
346 |
#include "checkpoint_lev4_directives.h" |
347 |
c************************************** |
348 |
c-- Initialise storage for the middle loop. |
349 |
CADJ INIT tapelev3 = USER |
350 |
#endif /* AUTODIFF_4_LEVEL_CHECKPOINT */ |
351 |
|
352 |
#ifndef AUTODIFF_2_LEVEL_CHECKPOINT |
353 |
do ilev_3 = 1,nchklev_3 |
354 |
if(ilev_3.le.max_lev3) then |
355 |
c************************************** |
356 |
#include "checkpoint_lev3_directives.h" |
357 |
c************************************** |
358 |
c-- Initialise storage for the middle loop. |
359 |
CADJ INIT tapelev2 = USER |
360 |
#endif /* AUTODIFF_2_LEVEL_CHECKPOINT */ |
361 |
|
362 |
do ilev_2 = 1,nchklev_2 |
363 |
if(ilev_2.le.max_lev2) then |
364 |
c************************************** |
365 |
#include "checkpoint_lev2_directives.h" |
366 |
c************************************** |
367 |
|
368 |
c************************************** |
369 |
#ifdef ALLOW_AUTODIFF_TAMC |
370 |
c-- Initialize storage for the innermost loop. |
371 |
c-- Always check common block sizes for the checkpointing! |
372 |
c-- |
373 |
CADJ INIT comlev1 = COMMON,nchklev_1 |
374 |
CADJ INIT comlev1_bibj = COMMON,nchklev_1*nsx*nsy*nthreads_chkpt |
375 |
CADJ INIT comlev1_bibj_k = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt |
376 |
c-- |
377 |
# ifdef ALLOW_KPP |
378 |
CADJ INIT comlev1_kpp = COMMON,nchklev_1*nsx*nsy |
379 |
CADJ INIT comlev1_kpp_k = COMMON,nchklev_1*nsx*nsy*nr |
380 |
# endif /* ALLOW_KPP */ |
381 |
c-- |
382 |
# ifdef ALLOW_GMREDI |
383 |
CADJ INIT comlev1_gmredi_k_gad |
384 |
CADJ & = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass |
385 |
# endif /* ALLOW_GMREDI */ |
386 |
c-- |
387 |
# ifdef ALLOW_PTRACERS |
388 |
CADJ INIT comlev1_bibj_ptracers = COMMON, |
389 |
CADJ & nchklev_1*nsx*nsy*nthreads_chkpt*PTRACERS_num |
390 |
# endif /* ALLOW_PTRACERS */ |
391 |
c-- |
392 |
cph Now also needed by seaice |
393 |
cph# ifndef DISABLE_MULTIDIM_ADVECTION |
394 |
CADJ INIT comlev1_bibj_k_gad |
395 |
CADJ & = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass |
396 |
CADJ INIT comlev1_bibj_k_gad_pass |
397 |
CADJ & = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass*maxcube |
398 |
cph# endif /* DISABLE_MULTIDIM_ADVECTION */ |
399 |
c-- |
400 |
# if (defined (ALLOW_EXF) && defined (ALLOW_BULKFORMULAE)) |
401 |
CADJ INIT comlev1_exf_1 |
402 |
CADJ & = COMMON,nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt |
403 |
CADJ INIT comlev1_exf_2 |
404 |
CADJ & = COMMON,niter_bulk*nchklev_1*snx*nsx*sny*nsy*nthreads_chkpt |
405 |
# endif |
406 |
c-- |
407 |
# ifdef ALLOW_SEAICE |
408 |
# ifdef SEAICE_ALLOW_DYNAMICS |
409 |
CADJ INIT comlev1_lsr = COMMON,nchklev_1*2 |
410 |
# endif |
411 |
# ifdef SEAICE_MULTILEVEL |
412 |
CADJ INIT comlev1_multdim |
413 |
CADJ & = COMMON,nchklev_1*nsx*nsy*nthreads_chkpt*multdim |
414 |
# endif |
415 |
# endif /* ALLOW_SEAICE */ |
416 |
c-- |
417 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
418 |
c************************************** |
419 |
|
420 |
do ilev_1 = 1,nchklev_1 |
421 |
|
422 |
c-- The if-statement below introduces a some flexibility in the |
423 |
c-- choice of the 3-tupel ( nchklev_1, nchklev_2, nchklev_3 ). |
424 |
c-- |
425 |
c-- Requirement: nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps . |
426 |
|
427 |
iloop = (ilev_2 - 1)*nchklev_1 + ilev_1 |
428 |
#ifndef AUTODIFF_2_LEVEL_CHECKPOINT |
429 |
& + (ilev_3 - 1)*nchklev_2*nchklev_1 |
430 |
#endif |
431 |
#ifdef AUTODIFF_4_LEVEL_CHECKPOINT |
432 |
& + (ilev_4 - 1)*nchklev_3*nchklev_2*nchklev_1 |
433 |
#endif |
434 |
|
435 |
if ( iloop .le. nTimeSteps ) then |
436 |
|
437 |
#else /* ALLOW_TAMC_CHECKPOINTING undefined */ |
438 |
c-- Initialise storage for the reference trajectory without TAMC check- |
439 |
c-- pointing. |
440 |
CADJ INIT history = USER |
441 |
CADJ INIT comlev1_bibj = COMMON,nchklev_0*nsx*nsy*nthreads_chkpt |
442 |
CADJ INIT comlev1_bibj_k = COMMON,nchklev_0*nsx*nsy*nr*nthreads_chkpt |
443 |
CADJ INIT comlev1_kpp = COMMON,nchklev_0*nsx*nsy |
444 |
|
445 |
c-- Check the choice of the checkpointing parameters in relation |
446 |
c-- to nTimeSteps: (nchklev_0 .ge. nTimeSteps) |
447 |
if (nchklev_0 .lt. nTimeSteps) then |
448 |
print* |
449 |
print*, ' the_main_loop: ', |
450 |
& 'TAMC checkpointing parameter nchklev_0 = ', |
451 |
& nchklev_0 |
452 |
print*, ' is not consistent with nTimeSteps = ', |
453 |
& nTimeSteps |
454 |
stop ' ... stopped in the_main_loop.' |
455 |
endif |
456 |
|
457 |
do iloop = 1, nTimeSteps |
458 |
|
459 |
#endif /* ALLOW_TAMC_CHECKPOINTING */ |
460 |
|
461 |
#else /* ALLOW_AUTODIFF_TAMC undefined */ |
462 |
c-- Start the main loop of ecco_Objfunc. Automatic differentiation is |
463 |
c-- NOT enabled. |
464 |
do iloop = 1, nTimeSteps |
465 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
466 |
|
467 |
#ifdef ALLOW_TAMC_CHECKPOINTING |
468 |
nIter0 = NINT( (startTime-baseTime)/deltaTClock ) |
469 |
ikey_dynamics = ilev_1 |
470 |
#endif |
471 |
|
472 |
c-- Set the model iteration counter and the model time. |
473 |
myiter = nIter0 + (iloop-1) |
474 |
mytime = startTime + float(iloop-1)*deltaTclock |
475 |
|
476 |
#ifdef ALLOW_AUTODIFF_TAMC |
477 |
CALL AUTODIFF_INADMODE_UNSET( myThid ) |
478 |
#endif |
479 |
|
480 |
|
481 |
C-- Switch on/off diagnostics for snap-shot output: |
482 |
#ifdef ALLOW_DIAGNOSTICS |
483 |
IF ( useDiagnostics ) THEN |
484 |
CALL DIAGNOSTICS_SWITCH_ONOFF( myTime, myIter, myThid ) |
485 |
ENDIF |
486 |
#endif |
487 |
|
488 |
C-- State-variables diagnostics |
489 |
IF ( usediagnostics ) THEN |
490 |
CALL TIMER_START('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid) |
491 |
CALL DO_STATEVARS_DIAGS( myTime, 0, myIter, myThid ) |
492 |
CALL TIMER_STOP ('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid) |
493 |
ENDIF |
494 |
|
495 |
#ifdef ALLOW_COST |
496 |
|
497 |
c-- Accumulate time averages of temperature, salinity, and SSH. |
498 |
call timer_start('COST_AVERAGESFIELDS [ECCO MAIN]', mythid) |
499 |
call cost_averagesFields( mytime, mythid ) |
500 |
call timer_stop ('COST_AVERAGESFIELDS [ECCO MAIN]', mythid) |
501 |
#ifdef ALLOW_COST_ATLANTIC |
502 |
c-- Compute meridional heat transport |
503 |
call timer_start('cost_atlantic [ECCO MAIN]', mythid) |
504 |
call cost_atlantic( mytime, myiter,mythid ) |
505 |
call timer_stop ('cost_atlantic [ECCO MAIN]', mythid) |
506 |
#endif |
507 |
#endif /* ALLOW_COST */ |
508 |
|
509 |
#ifdef ALLOW_AUTODIFF_TAMC |
510 |
c************************************** |
511 |
#include "checkpoint_lev1_directives.h" |
512 |
#include "checkpoint_lev1_template.h" |
513 |
c************************************** |
514 |
#endif |
515 |
|
516 |
#ifndef DISABLE_DEBUGMODE |
517 |
IF ( debugLevel .GE. debLevB ) |
518 |
& CALL DEBUG_CALL('EXF_GETFORCING',myThid) |
519 |
#endif |
520 |
CALL TIMER_START('EXF_GETFORCING [FORWARD_STEP]',mythid) |
521 |
CALL EXF_GETFORCING( mytime, myiter, mythid ) |
522 |
CALL TIMER_STOP ('EXF_GETFORCING [FORWARD_STEP]',mythid) |
523 |
# ifdef NONLIN_FRSURF |
524 |
CADJ STORE hFacC = comlev1, key = ikey_dynamics |
525 |
CADJ STORE SST = comlev1, key = ikey_dynamics |
526 |
CADJ STORE SSS = comlev1, key = ikey_dynamics |
527 |
# ifdef SHORTWAVE_HEATING |
528 |
CADJ STORE Qsw = comlev1, key = ikey_dynamics |
529 |
# endif |
530 |
# endif |
531 |
|
532 |
#ifdef ALLOW_AUTODIFF_TAMC |
533 |
# if (defined (ALLOW_AUTODIFF_MONITOR)) |
534 |
CALL DUMMY_IN_STEPPING( myTime, myIter, myThid ) |
535 |
# endif |
536 |
#endif |
537 |
|
538 |
#ifdef ALLOW_SEAICE |
539 |
cph this simple runtime flag causes a lot of recomp. |
540 |
IF ( useSEAICE ) THEN |
541 |
#ifndef DISABLE_DEBUGMODE |
542 |
IF ( debugLevel .GE. debLevB ) |
543 |
& CALL DEBUG_CALL('SEAICE_MODEL',myThid) |
544 |
#endif |
545 |
CALL TIMER_START('SEAICE_MODEL [FORWARD_STEP]',myThid) |
546 |
CALL SEAICE_MODEL( myTime, myIter, myThid ) |
547 |
CALL TIMER_STOP ('SEAICE_MODEL [FORWARD_STEP]',myThid) |
548 |
#ifdef ALLOW_COST_ICE |
549 |
CALL COST_ICE_TEST ( myTime, myIter, myThid ) |
550 |
#endif |
551 |
ENDIF |
552 |
#endif /* ALLOW_SEAICE */ |
553 |
|
554 |
#ifdef ALLOW_AUTODIFF_TAMC |
555 |
# ifdef ALLOW_PTRACERS |
556 |
cph this replaces _bibj storing of ptracer within thermodynamics |
557 |
CADJ STORE ptracer = comlev1, key = ikey_dynamics |
558 |
# endif |
559 |
#endif |
560 |
|
561 |
#ifdef ALLOW_EBM |
562 |
IF ( useEBM ) THEN |
563 |
# ifdef ALLOW_DEBUG |
564 |
IF ( debugLevel .GE. debLevB ) |
565 |
& CALL DEBUG_CALL('EBM',myThid) |
566 |
# endif |
567 |
CALL TIMER_START('EBM [FORWARD_STEP]',mythid) |
568 |
CALL EBM_DRIVER ( myTime, myIter, myThid ) |
569 |
CALL TIMER_STOP ('EBM [FORWARD_STEP]',mythid) |
570 |
ENDIF |
571 |
#endif |
572 |
|
573 |
#ifdef ALLOW_PTRACERS |
574 |
# ifdef ALLOW_GCHEM |
575 |
IF ( useGCHEM ) THEN |
576 |
#ifdef ALLOW_DEBUG |
577 |
IF ( debugLevel .GE. debLevB ) |
578 |
& CALL DEBUG_CALL('GCHEM_FIELDS_LOAD',myThid) |
579 |
#endif /* ALLOW_DEBUG */ |
580 |
CALL GCHEM_FIELDS_LOAD( mytime, myiter, mythid ) |
581 |
ENDIF |
582 |
# endif |
583 |
#endif |
584 |
|
585 |
C-- Step forward fields and calculate time tendency terms. |
586 |
|
587 |
#ifdef ALLOW_DEBUG |
588 |
IF ( debugLevel .GE. debLevB ) |
589 |
& CALL DEBUG_CALL('DO_ATMOSPHERIC_PHYS',myThid) |
590 |
#endif |
591 |
CALL TIMER_START('DO_ATMOSPHERIC_PHYS [FORWARD_STEP]',mythid) |
592 |
CALL DO_ATMOSPHERIC_PHYS( myTime, myIter, myThid ) |
593 |
CALL TIMER_STOP ('DO_ATMOSPHERIC_PHYS [FORWARD_STEP]',mythid) |
594 |
|
595 |
#ifdef ALLOW_AUTODIFF_TAMC |
596 |
# ifdef EXACT_CONSERV |
597 |
cphCADJ STORE empmr = comlev1, key = ikey_dynamics |
598 |
cphCADJ STORE pmepr = comlev1, key = ikey_dynamics |
599 |
# endif |
600 |
# ifdef NONLIN_FRSURF |
601 |
cph-test |
602 |
CADJ STORE hFacC = comlev1, key = ikey_dynamics |
603 |
# endif |
604 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
605 |
|
606 |
#ifndef ALLOW_OFFLINE |
607 |
#ifdef ALLOW_DEBUG |
608 |
IF ( debugLevel .GE. debLevB ) |
609 |
& CALL DEBUG_CALL('DO_OCEANIC_PHYS',myThid) |
610 |
#endif |
611 |
CALL TIMER_START('DO_OCEANIC_PHYS [FORWARD_STEP]',mythid) |
612 |
CALL DO_OCEANIC_PHYS( myTime, myIter, myThid ) |
613 |
CALL TIMER_STOP ('DO_OCEANIC_PHYS [FORWARD_STEP]',mythid) |
614 |
#ifdef ALLOW_AUTODIFF_TAMC |
615 |
CADJ STORE EmPmR = comlev1, key = ikey_dynamics |
616 |
# ifdef EXACT_CONSERV |
617 |
CADJ STORE pmepr = comlev1, key = ikey_dynamics |
618 |
# endif |
619 |
#endif |
620 |
#endif |
621 |
|
622 |
#ifdef ALLOW_AUTODIFF_TAMC |
623 |
# ifdef NONLIN_FRSURF |
624 |
cph-test |
625 |
CADJ STORE hFac_surfC = comlev1, key = ikey_dynamics |
626 |
CADJ STORE hfac_surfs = comlev1, key = ikey_dynamics |
627 |
CADJ STORE hfac_surfw = comlev1, key = ikey_dynamics |
628 |
CADJ STORE hFacC, hFacS, hFacW |
629 |
CADJ & = comlev1, key = ikey_dynamics |
630 |
CADJ STORE recip_hFacC, recip_hFacS, recip_hFacW |
631 |
CADJ & = comlev1, key = ikey_dynamics |
632 |
c |
633 |
CADJ STORE surfaceforcingu = comlev1, key = ikey_dynamics |
634 |
CADJ STORE surfaceforcingv = comlev1, key = ikey_dynamics |
635 |
# endif |
636 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
637 |
|
638 |
#ifdef ALLOW_GCHEM |
639 |
C GCHEM package is an interface for any bio-geochemical or |
640 |
C ecosystem model you would like to include. |
641 |
C If GCHEM_SEPARATE_FORCING is not defined, you are |
642 |
C responsible for computing tendency terms for passive |
643 |
C tracers and storing them on a 3DxNumPtracers-array called |
644 |
C gchemTendency in GCHEM_CALC_TENDENCY. This tendency is then added |
645 |
C to gPtr in ptracers_forcing later-on. |
646 |
C If GCHEM_SEPARATE_FORCING is defined, you are reponsible for |
647 |
C UPDATING ptracers directly in GCHEM_FORCING_SEP. This amounts |
648 |
C to a completely separate time step that you have to implement |
649 |
C yourself (Eulerian seems to be fine in most cases). |
650 |
CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC |
651 |
C CAVEAT: Up to now, when GCHEM is turned on the field ptracerForcingSurf, |
652 |
C which is needed for KPP is not set properly. ptracerForcingSurf must |
653 |
C be treated differently depending on whether GCHEM_SEPARATE_FORCING |
654 |
C is define or not. TBD. |
655 |
CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC |
656 |
IF ( useGCHEM ) THEN |
657 |
#ifdef ALLOW_DEBUG |
658 |
IF ( debugLevel .GE. debLevB ) |
659 |
& CALL DEBUG_CALL('GCHEM_CALC_TENDENCY',myThid) |
660 |
#endif |
661 |
CALL TIMER_START('GCHEM_CALC_TENDENCY [FORWARD_STEP]',myThid) |
662 |
CALL GCHEM_CALC_TENDENCY( myTime, myIter, myThid ) |
663 |
CALL TIMER_STOP ('GCHEM_CALC_TENDENCY [FORWARD_STEP]',myThid) |
664 |
ENDIF |
665 |
#endif /* ALLOW_GCHEM */ |
666 |
|
667 |
#ifdef ALLOW_AUTODIFF_TAMC |
668 |
cph needed to be moved here from do_oceanic_physics |
669 |
cph to be visible down the road |
670 |
c |
671 |
CADJ STORE surfaceForcingS = comlev1, key = ikey_dynamics |
672 |
CADJ STORE surfaceForcingT = comlev1, key = ikey_dynamics |
673 |
CADJ STORE surfaceForcingTice = comlev1, key = ikey_dynamics |
674 |
ctest( |
675 |
CADJ STORE IVDConvCount = comlev1, key = ikey_dynamics |
676 |
ctest) |
677 |
# ifdef ALLOW_PTRACERS |
678 |
CADJ STORE surfaceForcingPtr = comlev1, key = ikey_dynamics |
679 |
# endif |
680 |
c |
681 |
# ifdef ALLOW_GMREDI |
682 |
CADJ STORE Kwx = comlev1, key = ikey_dynamics |
683 |
CADJ STORE Kwy = comlev1, key = ikey_dynamics |
684 |
CADJ STORE Kwz = comlev1, key = ikey_dynamics |
685 |
# ifdef GM_BOLUS_ADVEC |
686 |
CADJ STORE GM_PsiX = comlev1, key = ikey_dynamics |
687 |
CADJ STORE GM_PsiY = comlev1, key = ikey_dynamics |
688 |
# endif |
689 |
# endif |
690 |
c |
691 |
# ifdef ALLOW_KPP |
692 |
CADJ STORE KPPghat = comlev1, key = ikey_dynamics |
693 |
CADJ STORE KPPfrac = comlev1, key = ikey_dynamics |
694 |
CADJ STORE KPPdiffKzS = comlev1, key = ikey_dynamics |
695 |
CADJ STORE KPPdiffKzT = comlev1, key = ikey_dynamics |
696 |
# endif |
697 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
698 |
|
699 |
#ifdef ALLOW_AUTODIFF_TAMC |
700 |
# ifdef NONLIN_FRSURF |
701 |
CADJ STORE etaH = comlev1, key = ikey_dynamics |
702 |
# ifdef ALLOW_CD_CODE |
703 |
CADJ STORE etanm1 = comlev1, key = ikey_dynamics |
704 |
# endif |
705 |
# endif |
706 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
707 |
|
708 |
IF ( .NOT.staggerTimeStep ) THEN |
709 |
#ifdef ALLOW_DEBUG |
710 |
IF ( debugLevel .GE. debLevB ) |
711 |
& CALL DEBUG_CALL('THERMODYNAMICS',myThid) |
712 |
#endif |
713 |
CALL TIMER_START('THERMODYNAMICS [FORWARD_STEP]',mythid) |
714 |
CALL THERMODYNAMICS( myTime, myIter, myThid ) |
715 |
CALL TIMER_STOP ('THERMODYNAMICS [FORWARD_STEP]',mythid) |
716 |
C-- if not staggerTimeStep: end |
717 |
ENDIF |
718 |
|
719 |
#ifdef ALLOW_AUTODIFF_TAMC |
720 |
# ifdef NONLIN_FRSURF |
721 |
CADJ STORE hFacC = comlev1, key = ikey_dynamics |
722 |
CADJ STORE hFacS = comlev1, key = ikey_dynamics |
723 |
CADJ STORE hFacW = comlev1, key = ikey_dynamics |
724 |
CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics |
725 |
CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics |
726 |
CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics |
727 |
CADJ STORE etaN = comlev1, key = ikey_dynamics |
728 |
# endif |
729 |
#endif |
730 |
|
731 |
C-- Step forward fields and calculate time tendency terms. |
732 |
#ifndef ALLOW_OFFLINE |
733 |
#ifndef ALLOW_AUTODIFF_TAMC |
734 |
IF ( momStepping ) THEN |
735 |
#endif |
736 |
#ifdef ALLOW_DEBUG |
737 |
IF ( debugLevel .GE. debLevB ) |
738 |
& CALL DEBUG_CALL('DYNAMICS',myThid) |
739 |
#endif |
740 |
CALL TIMER_START('DYNAMICS [FORWARD_STEP]',mythid) |
741 |
CALL DYNAMICS( myTime, myIter, myThid ) |
742 |
CALL TIMER_STOP ('DYNAMICS [FORWARD_STEP]',mythid) |
743 |
#ifndef ALLOW_AUTODIFF_TAMC |
744 |
ENDIF |
745 |
#endif |
746 |
#endif |
747 |
|
748 |
#ifdef ALLOW_AUTODIFF_TAMC |
749 |
# ifdef NONLIN_FRSURF |
750 |
cph-test |
751 |
CADJ STORE gU, gV = comlev1, key = ikey_dynamics |
752 |
# endif |
753 |
#endif |
754 |
|
755 |
C-- Update time-counter |
756 |
myIter = nIter0 + iLoop |
757 |
myTime = startTime + deltaTClock * float(iLoop) |
758 |
|
759 |
#ifdef ALLOW_MNC |
760 |
C Update the default next iter for MNC |
761 |
IF ( useMNC ) THEN |
762 |
CALL MNC_CW_CITER_SETG( 1, 1, -1, myIter , myThid ) |
763 |
|
764 |
C TODO: Logic should be added here so that users can specify, on |
765 |
C a per-citer-group basis, when it is time to update the |
766 |
C "current" (and not just the "next") iteration |
767 |
|
768 |
C TODO: the following is just a temporary band-aid (mostly, for |
769 |
C Baylor) until someone writes a routine that better handles time |
770 |
C boundaries such as weeks, months, years, etc. |
771 |
IF ( mnc_filefreq .GT. 0 ) THEN |
772 |
IF (DIFFERENT_MULTIPLE(mnc_filefreq,myTime,deltaTClock)) |
773 |
& THEN |
774 |
CALL MNC_CW_CITER_SETG( 1, 1, myIter, -1 , myThid ) |
775 |
ENDIF |
776 |
ENDIF |
777 |
ENDIF |
778 |
#endif |
779 |
|
780 |
C-- Update geometric factors: |
781 |
#ifdef NONLIN_FRSURF |
782 |
C- update hfacC,W,S and recip_hFac according to etaH(n+1) : |
783 |
IF ( nonlinFreeSurf.GT.0) THEN |
784 |
IF ( select_rStar.GT.0 ) THEN |
785 |
# ifndef DISABLE_RSTAR_CODE |
786 |
# ifdef ALLOW_AUTODIFF_TAMC |
787 |
cph-test |
788 |
CADJ STORE hFacC = comlev1, key = ikey_dynamics |
789 |
CADJ STORE hFacS = comlev1, key = ikey_dynamics |
790 |
CADJ STORE hFacW = comlev1, key = ikey_dynamics |
791 |
CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics |
792 |
CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics |
793 |
CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics |
794 |
# endif |
795 |
CALL TIMER_START('UPDATE_R_STAR [FORWARD_STEP]',myThid) |
796 |
CALL UPDATE_R_STAR( myTime, myIter, myThid ) |
797 |
CALL TIMER_STOP ('UPDATE_R_STAR [FORWARD_STEP]',myThid) |
798 |
# ifdef ALLOW_AUTODIFF_TAMC |
799 |
cph-test |
800 |
CADJ STORE hFacC = comlev1, key = ikey_dynamics |
801 |
CADJ STORE hFacS = comlev1, key = ikey_dynamics |
802 |
CADJ STORE hFacW = comlev1, key = ikey_dynamics |
803 |
CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics |
804 |
CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics |
805 |
CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics |
806 |
# endif |
807 |
# endif /* DISABLE_RSTAR_CODE */ |
808 |
ELSE |
809 |
#ifdef ALLOW_AUTODIFF_TAMC |
810 |
CADJ STORE hFac_surfC, hFac_surfS, hFac_surfW |
811 |
CADJ & = comlev1, key = ikey_dynamics |
812 |
#endif |
813 |
CALL TIMER_START('UPDATE_SURF_DR [FORWARD_STEP]',myThid) |
814 |
CALL UPDATE_SURF_DR( myTime, myIter, myThid ) |
815 |
CALL TIMER_STOP ('UPDATE_SURF_DR [FORWARD_STEP]',myThid) |
816 |
ENDIF |
817 |
ENDIF |
818 |
# ifdef ALLOW_AUTODIFF_TAMC |
819 |
cph-test |
820 |
CADJ STORE hFacC = comlev1, key = ikey_dynamics |
821 |
CADJ STORE hFacS = comlev1, key = ikey_dynamics |
822 |
CADJ STORE hFacW = comlev1, key = ikey_dynamics |
823 |
CADJ STORE recip_hFacC = comlev1, key = ikey_dynamics |
824 |
CADJ STORE recip_hFacS = comlev1, key = ikey_dynamics |
825 |
CADJ STORE recip_hFacW = comlev1, key = ikey_dynamics |
826 |
# endif |
827 |
C- update also CG2D matrix (and preconditioner) |
828 |
IF ( momStepping .AND. nonlinFreeSurf.GT.2 ) THEN |
829 |
CALL TIMER_START('UPDATE_CG2D [FORWARD_STEP]',myThid) |
830 |
CALL UPDATE_CG2D( myTime, myIter, myThid ) |
831 |
CALL TIMER_STOP ('UPDATE_CG2D [FORWARD_STEP]',myThid) |
832 |
ENDIF |
833 |
#endif /* NONLIN_FRSURF */ |
834 |
|
835 |
C-- Apply Filters to u*,v* before SOLVE_FOR_PRESSURE |
836 |
#ifdef ALLOW_SHAP_FILT |
837 |
IF (useSHAP_FILT .AND. shap_filt_uvStar) THEN |
838 |
CALL TIMER_START('SHAP_FILT [FORWARD_STEP]',myThid) |
839 |
IF (implicDiv2Dflow.LT.1.) THEN |
840 |
C-- Explicit+Implicit part of the Barotropic Flow Divergence |
841 |
C => Filtering of uVel,vVel is necessary |
842 |
CALL SHAP_FILT_APPLY_UV( uVel,vVel, |
843 |
& myTime, myIter, myThid ) |
844 |
ENDIF |
845 |
CALL SHAP_FILT_APPLY_UV( gU,gV,myTime,myIter,myThid) |
846 |
CALL TIMER_STOP ('SHAP_FILT [FORWARD_STEP]',myThid) |
847 |
ENDIF |
848 |
#endif |
849 |
#ifdef ALLOW_ZONAL_FILT |
850 |
IF (useZONAL_FILT .AND. zonal_filt_uvStar) THEN |
851 |
CALL TIMER_START('ZONAL_FILT_APPLY [FORWARD_STEP]',myThid) |
852 |
IF (implicDiv2Dflow.LT.1.) THEN |
853 |
C-- Explicit+Implicit part of the Barotropic Flow Divergence |
854 |
C => Filtering of uVel,vVel is necessary |
855 |
CALL ZONAL_FILT_APPLY_UV( uVel, vVel, myThid ) |
856 |
ENDIF |
857 |
CALL ZONAL_FILT_APPLY_UV( gU, gV, myThid ) |
858 |
CALL TIMER_STOP ('ZONAL_FILT_APPLY [FORWARD_STEP]',myThid) |
859 |
ENDIF |
860 |
#endif |
861 |
|
862 |
C-- Solve elliptic equation(s). |
863 |
C Two-dimensional only for conventional hydrostatic or |
864 |
C three-dimensional for non-hydrostatic and/or IGW scheme. |
865 |
#ifndef ALLOW_OFFLINE |
866 |
IF ( momStepping ) THEN |
867 |
#ifdef ALLOW_AUTODIFF_TAMC |
868 |
# ifdef NONLIN_FRSURF |
869 |
CADJ STORE uvel, vvel |
870 |
CADJ & = comlev1, key = ikey_dynamics |
871 |
CADJ STORE empmr,hfacs,hfacw |
872 |
CADJ & = comlev1, key = ikey_dynamics |
873 |
# endif |
874 |
#endif |
875 |
CALL TIMER_START('SOLVE_FOR_PRESSURE [FORWARD_STEP]',myThid) |
876 |
CALL SOLVE_FOR_PRESSURE(myTime, myIter, myThid) |
877 |
CALL TIMER_STOP ('SOLVE_FOR_PRESSURE [FORWARD_STEP]',myThid) |
878 |
ENDIF |
879 |
#endif |
880 |
|
881 |
C-- Correct divergence in flow field and cycle time-stepping momentum |
882 |
c IF ( momStepping ) THEN |
883 |
#ifndef ALLOW_OFFLINE |
884 |
CALL TIMER_START('UV_CORRECTION_STEP [FORWARD_STEP]',myThid) |
885 |
CALL MOMENTUM_CORRECTION_STEP(myTime, myIter, myThid) |
886 |
CALL TIMER_STOP ('UV_CORRECTION_STEP [FORWARD_STEP]',myThid) |
887 |
#endif |
888 |
c ENDIF |
889 |
|
890 |
#ifdef EXACT_CONSERV |
891 |
IF (exactConserv) THEN |
892 |
#ifdef ALLOW_AUTODIFF_TAMC |
893 |
cph-test |
894 |
cphCADJ STORE etaH = comlev1, key = ikey_dynamics |
895 |
#endif |
896 |
C-- Update etaH(n+1) : |
897 |
CALL TIMER_START('UPDATE_ETAH [FORWARD_STEP]',mythid) |
898 |
CALL UPDATE_ETAH( myTime, myIter, myThid ) |
899 |
CALL TIMER_STOP ('UPDATE_ETAH [FORWARD_STEP]',mythid) |
900 |
ENDIF |
901 |
#endif /* EXACT_CONSERV */ |
902 |
|
903 |
#ifdef NONLIN_FRSURF |
904 |
IF ( select_rStar.NE.0 ) THEN |
905 |
# ifndef DISABLE_RSTAR_CODE |
906 |
C-- r* : compute the future level thickness according to etaH(n+1) |
907 |
CALL TIMER_START('CALC_R_STAR [FORWARD_STEP]',mythid) |
908 |
CALL CALC_R_STAR(etaH, myTime, myIter, myThid ) |
909 |
CALL TIMER_STOP ('CALC_R_STAR [FORWARD_STEP]',mythid) |
910 |
# endif /* DISABLE_RSTAR_CODE */ |
911 |
ELSEIF ( nonlinFreeSurf.GT.0) THEN |
912 |
C-- compute the future surface level thickness according to etaH(n+1) |
913 |
# ifdef ALLOW_AUTODIFF_TAMC |
914 |
CADJ STORE etaH = comlev1, key = ikey_dynamics |
915 |
# endif |
916 |
CALL TIMER_START('CALC_SURF_DR [FORWARD_STEP]',mythid) |
917 |
CALL CALC_SURF_DR(etaH, myTime, myIter, myThid ) |
918 |
CALL TIMER_STOP ('CALC_SURF_DR [FORWARD_STEP]',mythid) |
919 |
ENDIF |
920 |
# ifdef ALLOW_AUTODIFF_TAMC |
921 |
cph-test |
922 |
CADJ STORE hFac_surfC = comlev1, key = ikey_dynamics |
923 |
# endif |
924 |
#endif /* NONLIN_FRSURF */ |
925 |
|
926 |
C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----| |
927 |
IF ( staggerTimeStep ) THEN |
928 |
C-- do exchanges of U,V (needed for multiDim) when using stagger time-step : |
929 |
#ifdef ALLOW_DEBUG |
930 |
IF ( debugLevel .GE. debLevB ) |
931 |
& CALL DEBUG_CALL('DO_STAGGER_FIELDS_EXCH.',myThid) |
932 |
#endif |
933 |
CALL TIMER_START('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid) |
934 |
CALL DO_STAGGER_FIELDS_EXCHANGES( myTime, myIter, myThid ) |
935 |
CALL TIMER_STOP ('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid) |
936 |
|
937 |
C-- State-variables diagnostics |
938 |
IF ( usediagnostics ) THEN |
939 |
CALL TIMER_START('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid) |
940 |
CALL DO_STATEVARS_DIAGS( myTime, 1, myIter, myThid ) |
941 |
CALL TIMER_STOP ('DO_STATEVARS_DIAGS [FORWARD_STEP]',myThid) |
942 |
ENDIF |
943 |
|
944 |
#ifdef ALLOW_DEBUG |
945 |
IF ( debugLevel .GE. debLevB ) |
946 |
& CALL DEBUG_CALL('THERMODYNAMICS',myThid) |
947 |
#endif |
948 |
CALL TIMER_START('THERMODYNAMICS [FORWARD_STEP]',mythid) |
949 |
CALL THERMODYNAMICS( myTime, myIter, myThid ) |
950 |
CALL TIMER_STOP ('THERMODYNAMICS [FORWARD_STEP]',mythid) |
951 |
|
952 |
C-- if staggerTimeStep: end |
953 |
ENDIF |
954 |
C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----| |
955 |
|
956 |
#ifdef ALLOW_AUTODIFF_TAMC |
957 |
cph This is needed because convective_adjustment calls |
958 |
cph find_rho which may use pressure() |
959 |
CADJ STORE totphihyd = comlev1, key = ikey_dynamics |
960 |
#endif |
961 |
C-- Cycle time-stepping Tracers arrays (T,S,+pTracers) |
962 |
CALL TIMER_START('TS_CORRECTION_STEP [FORWARD_STEP]',myThid) |
963 |
CALL TRACERS_CORRECTION_STEP(myTime, myIter, myThid) |
964 |
CALL TIMER_STOP ('TS_CORRECTION_STEP [FORWARD_STEP]',myThid) |
965 |
|
966 |
#ifdef ALLOW_GCHEM |
967 |
C Add separate timestepping of chemical/biological/forcing |
968 |
C of ptracers here in GCHEM_FORCING_SEP |
969 |
IF ( useGCHEM ) THEN |
970 |
#ifdef ALLOW_DEBUG |
971 |
IF ( debugLevel .GE. debLevB ) |
972 |
& CALL DEBUG_CALL('GCHEM_FORCING_SEP',myThid) |
973 |
#endif /* ALLOW_DEBUG */ |
974 |
CALL TIMER_START('GCHEM_FORCING_SEP [FORWARD_STEP]',myThid) |
975 |
CALL GCHEM_FORCING_SEP( myTime,myIter,myThid ) |
976 |
CALL TIMER_STOP ('GCHEM_FORCING_SEP [FORWARD_STEP]',myThid) |
977 |
ENDIF |
978 |
#endif /* ALLOW_GCHEM */ |
979 |
|
980 |
C-- Do "blocking" sends and receives for tendency "overlap" terms |
981 |
c CALL TIMER_START('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid) |
982 |
c CALL DO_GTERM_BLOCKING_EXCHANGES( myThid ) |
983 |
c CALL TIMER_STOP ('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid) |
984 |
|
985 |
C-- Do "blocking" sends and receives for field "overlap" terms |
986 |
CALL TIMER_START('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid) |
987 |
CALL DO_FIELDS_BLOCKING_EXCHANGES( myThid ) |
988 |
CALL TIMER_STOP ('BLOCKING_EXCHANGES [FORWARD_STEP]',myThid) |
989 |
|
990 |
#ifdef ALLOW_FLT |
991 |
C-- Calculate float trajectories |
992 |
IF (useFLT) THEN |
993 |
CALL TIMER_START('FLOATS [FORWARD_STEP]',myThid) |
994 |
CALL FLT_MAIN(myIter,myTime, myThid) |
995 |
CALL TIMER_STOP ('FLOATS [FORWARD_STEP]',myThid) |
996 |
ENDIF |
997 |
#endif |
998 |
|
999 |
#ifdef ALLOW_AUTODIFF_TAMC |
1000 |
CALL AUTODIFF_INADMODE_SET( myThid ) |
1001 |
#endif |
1002 |
|
1003 |
C-- State-variables time-averaging |
1004 |
CALL TIMER_START('DO_STATEVARS_TAVE [FORWARD_STEP]',myThid) |
1005 |
CALL DO_STATEVARS_TAVE( myTime, myIter, myThid ) |
1006 |
CALL TIMER_STOP ('DO_STATEVARS_TAVE [FORWARD_STEP]',myThid) |
1007 |
|
1008 |
#ifndef ALLOW_OFFLINE |
1009 |
#ifdef ALLOW_MONITOR |
1010 |
C-- Check status of solution (statistics, cfl, etc...) |
1011 |
CALL TIMER_START('MONITOR [FORWARD_STEP]',myThid) |
1012 |
CALL MONITOR( myIter, myTime, myThid ) |
1013 |
CALL TIMER_STOP ('MONITOR [FORWARD_STEP]',myThid) |
1014 |
#endif /* ALLOW_MONITOR */ |
1015 |
#endif |
1016 |
|
1017 |
C-- Do IO if needed. |
1018 |
#ifdef ALLOW_OFFLINE |
1019 |
CALL TIMER_START('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid) |
1020 |
CALL OFFLINE_MODEL_IO( myTime, myIter, myThid ) |
1021 |
CALL TIMER_STOP ('OFFLINE_MODEL_IO [FORWARD_STEP]',myThid) |
1022 |
#else |
1023 |
CALL TIMER_START('DO_THE_MODEL_IO [FORWARD_STEP]',myThid) |
1024 |
CALL DO_THE_MODEL_IO( myTime, myIter, myThid ) |
1025 |
CALL TIMER_STOP ('DO_THE_MODEL_IO [FORWARD_STEP]',myThid) |
1026 |
#endif |
1027 |
|
1028 |
C-- Save state for restarts |
1029 |
CALL TIMER_START('WRITE_CHECKPOINT [FORWARD_STEP]',myThid) |
1030 |
CALL PACKAGES_WRITE_PICKUP( |
1031 |
I .FALSE., myTime, myIter, myThid ) |
1032 |
#ifndef ALLOW_OFFLINE |
1033 |
CALL WRITE_CHECKPOINT( |
1034 |
I .FALSE., myTime, myIter, myThid ) |
1035 |
#endif |
1036 |
CALL TIMER_STOP ('WRITE_CHECKPOINT [FORWARD_STEP]',myThid) |
1037 |
|
1038 |
#ifdef TIME_PER_TIMESTEP |
1039 |
CCE107 Time per timestep information |
1040 |
_BEGIN_MASTER( myThid ) |
1041 |
CALL TIMER_GET_TIME( utnew, stnew, wtnew ) |
1042 |
WRITE(msgBuf,'(A34,3F10.6,I8)') |
1043 |
$ 'User, system and wallclock time:', utnew - utold, |
1044 |
$ stnew - stold, wtnew - wtold, iloop |
1045 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
1046 |
utold = utnew |
1047 |
stold = stnew |
1048 |
wtold = wtnew |
1049 |
_END_MASTER( myThid ) |
1050 |
#endif |
1051 |
#ifdef USE_PAPI_FLOPS |
1052 |
CCE107 PAPI summary performance |
1053 |
_BEGIN_MASTER( myThid ) |
1054 |
#ifdef USE_FLIPS |
1055 |
call PAPIF_flips(real_time, proc_time, flpops, mflops, check) |
1056 |
#else |
1057 |
call PAPIF_flops(real_time, proc_time, flpops, mflops, check) |
1058 |
#endif |
1059 |
WRITE(msgBuf,'(F10.6,A34,I8)') |
1060 |
$ mflops, 'Mflop/s during timestep ', iloop |
1061 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
1062 |
_END_MASTER( myThid ) |
1063 |
#ifdef PAPI_VERSION |
1064 |
call PAPIF_ipc(real_time, proc_time, instr, ipc, check) |
1065 |
WRITE(msgBuf,'(F10.6,A34,I8)') |
1066 |
$ ipc, 'IPC during timestep ', iloop |
1067 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
1068 |
#endif |
1069 |
_END_MASTER( myThid ) |
1070 |
#else |
1071 |
#ifdef USE_PCL_FLOPS |
1072 |
CCE107 PCL summary performance |
1073 |
_BEGIN_MASTER( myThid ) |
1074 |
res = PCLstop(descr, i_result, fp_result, nevents) |
1075 |
do ipcl = 1, nevents |
1076 |
WRITE(msgBuf,'(F10.6,A2,A22,A17,I8)'), fp_result(ipcl), |
1077 |
$ ' ', pcl_counter_name(pcl_counter_list(ipcl)), |
1078 |
$ 'during timestep ', iloop |
1079 |
CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1) |
1080 |
enddo |
1081 |
res = PCLstart(descr, pcl_counter_list, nevents, flags) |
1082 |
_END_MASTER( myThid ) |
1083 |
#endif |
1084 |
#endif |
1085 |
|
1086 |
#ifdef ALLOW_AUTODIFF_TAMC |
1087 |
#ifdef ALLOW_TAMC_CHECKPOINTING |
1088 |
endif |
1089 |
enddo |
1090 |
endif |
1091 |
enddo |
1092 |
#ifndef AUTODIFF_2_LEVEL_CHECKPOINT |
1093 |
endif |
1094 |
enddo |
1095 |
#endif |
1096 |
#ifdef AUTODIFF_4_LEVEL_CHECKPOINT |
1097 |
endif |
1098 |
enddo |
1099 |
#endif |
1100 |
c |
1101 |
#else /* ndef ALLOW_TAMC_CHECKPOINTING */ |
1102 |
enddo |
1103 |
#endif /* ALLOW_TAMC_CHECKPOINTING */ |
1104 |
|
1105 |
#else /* ndef ALLOW_AUTODIFF_TAMC */ |
1106 |
enddo |
1107 |
#endif /* ALLOW_AUTODIFF_TAMC */ |
1108 |
|
1109 |
_BARRIER |
1110 |
call timer_stop ('ECCO MAIN LOOP', mythid) |
1111 |
|
1112 |
call timer_start('ECCO SPIN-DOWN', mythid) |
1113 |
|
1114 |
#ifdef ALLOW_COST |
1115 |
|
1116 |
#ifdef ALLOW_DIVIDED_ADJOINT |
1117 |
CADJ STORE mytime = onetape |
1118 |
#endif |
1119 |
c-- Accumulate time averages of temperature, salinity, and SSH. |
1120 |
#ifndef DISABLE_DEBUGMODE |
1121 |
IF ( debugLevel .GE. debLevB ) |
1122 |
& CALL DEBUG_CALL('cost_averagesfields',myThid) |
1123 |
#endif |
1124 |
call timer_start('cost_averagesfields [ECCO SPIN-DOWN]', mythid) |
1125 |
call cost_averagesfields( mytime, mythid ) |
1126 |
call timer_stop ('cost_averagesfields [ECCO SPIN-DOWN]', mythid) |
1127 |
#ifdef ALLOW_DIVIDED_ADJOINT |
1128 |
c************************************** |
1129 |
#include "cost_averages_bar_directives.h" |
1130 |
c************************************** |
1131 |
#endif |
1132 |
|
1133 |
#ifdef ALLOW_COST_ATLANTIC |
1134 |
c-- Compute meridional heat transport |
1135 |
#ifndef DISABLE_DEBUGMODE |
1136 |
IF ( debugLevel .GE. debLevB ) |
1137 |
& CALL DEBUG_CALL('cost_atlantic',myThid) |
1138 |
#endif |
1139 |
call timer_start('cost_atlantic [ECCO SPIN-DOWN]', mythid) |
1140 |
call cost_atlantic( mytime, myiter,mythid ) |
1141 |
call timer_stop ('cost_atlantic [ECCO SPIN-DOWN]', mythid) |
1142 |
#endif |
1143 |
|
1144 |
c-- Compute the cost function contribution of the boundary forcing, |
1145 |
c-- i.e. heat flux, salt flux, zonal and meridional wind stress. |
1146 |
#ifndef DISABLE_DEBUGMODE |
1147 |
IF ( debugLevel .GE. debLevB ) |
1148 |
& CALL DEBUG_CALL('cost_forcing',myThid) |
1149 |
#endif |
1150 |
call timer_start('cost_forcing [ECCO SPIN-DOWN]', mythid) |
1151 |
call cost_forcing( myiter, mytime, mythid ) |
1152 |
call timer_stop ('cost_forcing [ECCO SPIN-DOWN]', mythid) |
1153 |
|
1154 |
c-- Compute cost function contribution of Temperature and Salinity. |
1155 |
#ifndef DISABLE_DEBUGMODE |
1156 |
IF ( debugLevel .GE. debLevB ) |
1157 |
& CALL DEBUG_CALL('cost_hyd',myThid) |
1158 |
#endif |
1159 |
call timer_start('cost_hyd [ECCO SPIN-DOWN]', mythid) |
1160 |
call cost_hyd( myiter, mytime, mythid ) |
1161 |
call timer_stop ('cost_hyd [ECCO SPIN-DOWN]', mythid) |
1162 |
|
1163 |
#ifdef ALLOW_SEAICE |
1164 |
#ifndef DISABLE_DEBUGMODE |
1165 |
IF ( debugLevel .GE. debLevB ) |
1166 |
& CALL DEBUG_CALL('seaice_cost_driver',myThid) |
1167 |
#endif |
1168 |
call timer_start('seaice_cost_driver [ECCO SPIN-DOWN]', mythid) |
1169 |
call seaice_cost_driver( myiter, mytime, mythid ) |
1170 |
call timer_stop ('seaice_cost_driver [ECCO SPIN-DOWN]', mythid) |
1171 |
#endif |
1172 |
|
1173 |
#ifdef ALLOW_OBCS_COST_CONTRIBUTION |
1174 |
#ifndef DISABLE_DEBUGMODE |
1175 |
IF ( debugLevel .GE. debLevB ) |
1176 |
& CALL DEBUG_CALL('cost_obcs',myThid) |
1177 |
#endif |
1178 |
call timer_start('cost_obcs [ECCO SPIN-DOWN]', mythid) |
1179 |
call cost_obcs( myiter, mytime, mythid ) |
1180 |
call timer_stop ('cost_obcs [ECCO SPIN-DOWN]', mythid) |
1181 |
#endif |
1182 |
|
1183 |
#ifdef ALLOW_CURMTR_COST_CONTRIBUTION |
1184 |
#ifndef DISABLE_DEBUGMODE |
1185 |
IF ( debugLevel .GE. debLevB ) |
1186 |
& CALL DEBUG_CALL('cost_curmtr',myThid) |
1187 |
#endif |
1188 |
call timer_start('cost_curmtr [ECCO SPIN-DOWN]', mythid) |
1189 |
call cost_curmtr( myiter, mytime, mythid ) |
1190 |
call timer_stop ('cost_curmtr [ECCO SPIN-DOWN]', mythid) |
1191 |
#endif |
1192 |
|
1193 |
c-- Compute cost function contribution of SSH. |
1194 |
#ifdef ALLOW_SSH_COST_CONTRIBUTION |
1195 |
#ifndef DISABLE_DEBUGMODE |
1196 |
IF ( debugLevel .GE. debLevB ) |
1197 |
& CALL DEBUG_CALL('cost_ssh',myThid) |
1198 |
#endif |
1199 |
call timer_start('cost_ssh [ECCO SPIN-DOWN]', mythid) |
1200 |
call cost_ssh( myiter, mytime, mythid ) |
1201 |
call timer_stop ('cost_ssh [ECCO SPIN-DOWN]', mythid) |
1202 |
#endif |
1203 |
|
1204 |
c-- Compute cost function contribution of drifter's velocities. |
1205 |
#ifdef ALLOW_DRIFTER_COST_CONTRIBUTION |
1206 |
#ifndef DISABLE_DEBUGMODE |
1207 |
IF ( debugLevel .GE. debLevB ) |
1208 |
& CALL DEBUG_CALL('cost_drifter',myThid) |
1209 |
#endif |
1210 |
call timer_start('cost_drifter [ECCO SPIN-DOWN]', mythid) |
1211 |
call cost_drifter( myiter, mytime, mythid ) |
1212 |
call timer_stop ('cost_drifter [ECCO SPIN-DOWN]', mythid) |
1213 |
#endif |
1214 |
|
1215 |
c-- Compute cost function contribution of wind stress observations. |
1216 |
#ifdef ALLOW_SCAT_COST_CONTRIBUTION |
1217 |
#ifndef DISABLE_DEBUGMODE |
1218 |
IF ( debugLevel .GE. debLevB ) |
1219 |
& CALL DEBUG_CALL('cost_scat',myThid) |
1220 |
#endif |
1221 |
call timer_start('cost_scat [ECCO SPIN-DOWN]', mythid) |
1222 |
call cost_scat( myiter, mytime, mythid ) |
1223 |
call timer_stop ('cost_scat [ECCO SPIN-DOWN]', mythid) |
1224 |
#endif |
1225 |
|
1226 |
c-- Compute cost function contribution of drift between the first |
1227 |
c and the last year. |
1228 |
#ifdef ALLOW_DRIFT_COST_CONTRIBUTION |
1229 |
#ifndef DISABLE_DEBUGMODE |
1230 |
IF ( debugLevel .GE. debLevB ) |
1231 |
& CALL DEBUG_CALL('cost_drift',myThid) |
1232 |
#endif |
1233 |
call timer_start('cost_drift [ECCO SPIN-DOWN]', mythid) |
1234 |
call cost_drift( myiter, mytime, mythid ) |
1235 |
call timer_stop ('cost_drift [ECCO SPIN-DOWN]', mythid) |
1236 |
#endif |
1237 |
#ifdef ALLOW_DRIFTW_COST_CONTRIBUTION |
1238 |
#ifndef DISABLE_DEBUGMODE |
1239 |
IF ( debugLevel .GE. debLevB ) |
1240 |
& CALL DEBUG_CALL('cost_driftw',myThid) |
1241 |
#endif |
1242 |
call timer_start('cost_driftw [ECCO SPIN-DOWN]', mythid) |
1243 |
call cost_driftw( myiter, mytime, mythid ) |
1244 |
call timer_stop ('cost_driftw [ECCO SPIN-DOWN]', mythid) |
1245 |
#endif |
1246 |
_BARRIER |
1247 |
|
1248 |
c-- Compute initial vs. final T/S deviation |
1249 |
#ifdef ALLOW_COST_INI_FIN |
1250 |
call timer_start('cost_ini_fin [ECCO SPIN-DOWN]', mythid) |
1251 |
call cost_theta_ini_fin( myiter, mytime, mythid ) |
1252 |
call cost_salt_ini_fin( myiter, mytime, mythid ) |
1253 |
call timer_stop ('cost_ini_fin [ECCO SPIN-DOWN]', mythid) |
1254 |
#endif |
1255 |
_BARRIER |
1256 |
|
1257 |
c-- Eddy stress penalty term |
1258 |
#ifdef ALLOW_COST_TAU_EDDY |
1259 |
call timer_start('cost_tau_eddy [ECCO SPIN-DOWN]', mythid) |
1260 |
call cost_tau_eddy( mythid ) |
1261 |
call timer_stop ('cost_tau_eddy [ECCO SPIN-DOWN]', mythid) |
1262 |
#endif |
1263 |
|
1264 |
c-- Sum all cost function contributions. |
1265 |
#ifndef DISABLE_DEBUGMODE |
1266 |
IF ( debugLevel .GE. debLevB ) |
1267 |
& CALL DEBUG_CALL('cost_final',myThid) |
1268 |
#endif |
1269 |
call timer_start('COST_FINAL [ECCO SPIN-DOWN]', mythid) |
1270 |
call cost_final( mythid ) |
1271 |
call timer_stop ('COST_FINAL [ECCO SPIN-DOWN]', mythid) |
1272 |
|
1273 |
#endif /* ALLOW_COST */ |
1274 |
|
1275 |
call timer_stop ('ECCO SPIN-DOWN', mythid) |
1276 |
|
1277 |
#ifndef DISABLE_DEBUGMODE |
1278 |
IF ( debugLevel .GE. debLevB ) |
1279 |
& CALL DEBUG_LEAVE('THE_MAIN_LOOP',myThid) |
1280 |
#endif |
1281 |
|
1282 |
return |
1283 |
end |
1284 |
|