/[MITgcm]/MITgcm/pkg/ecco/the_main_loop.F
ViewVC logotype

Diff of /MITgcm/pkg/ecco/the_main_loop.F

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.14 by heimbach, Sat Nov 13 06:19:10 2004 UTC revision 1.48 by heimbach, Wed Dec 13 22:31:19 2006 UTC
# Line 72  c     == global variables == Line 72  c     == global variables ==
72  #include "EEPARAMS.h"  #include "EEPARAMS.h"
73  #include "PARAMS.h"  #include "PARAMS.h"
74    
75    #ifdef ALLOW_MNC
76    #include "MNC_PARAMS.h"
77          EXTERNAL DIFFERENT_MULTIPLE
78          LOGICAL  DIFFERENT_MULTIPLE
79    #endif
80    
81    #ifdef HAVE_SIGREG
82    #include "SIGREG.h"
83    #endif
84    
85    #ifdef ALLOW_SHAP_FILT
86    # include "SHAP_FILT.h"
87    #endif
88    #ifdef ALLOW_ZONAL_FILT
89    # include "ZONAL_FILT.h"
90    #endif
91    #ifdef COMPONENT_MODULE
92    # include "CPL_PARAMS.h"
93    #endif
94    
95  c**************************************  c**************************************
96  #ifdef ALLOW_AUTODIFF_TAMC  #ifdef ALLOW_AUTODIFF_TAMC
97    
# Line 83  c They provide the fields to be stored. Line 103  c They provide the fields to be stored.
103  # include "DYNVARS.h"  # include "DYNVARS.h"
104  # include "FFIELDS.h"  # include "FFIELDS.h"
105  # include "EOS.h"  # include "EOS.h"
 # include "GAD.h"  
106    
107    # ifdef ALLOW_GENERIC_ADVDIFF
108    #  include "GAD.h"
109    # endif
110  # ifdef ALLOW_CD_CODE  # ifdef ALLOW_CD_CODE
111  #  include "CD_CODE_VARS.h"  #  include "CD_CODE_VARS.h"
112  # endif  # endif
# Line 95  c They provide the fields to be stored. Line 117  c They provide the fields to be stored.
117  # ifdef ALLOW_NONHYDROSTATIC  # ifdef ALLOW_NONHYDROSTATIC
118  #  include "CG3D.h"  #  include "CG3D.h"
119  # endif  # endif
120  # ifdef EXACT_CONSERV  # if (defined (EXACT_CONSERV) || defined (NONLIN_FRSURF))
121  #  include "SURFACE.h"  #  include "SURFACE.h"
122  # endif  # endif
123  # ifdef ALLOW_OBCS  # ifdef ALLOW_OBCS
# Line 110  c They provide the fields to be stored. Line 132  c They provide the fields to be stored.
132  # endif /* ALLOW_EXF */  # endif /* ALLOW_EXF */
133  # ifdef ALLOW_SEAICE  # ifdef ALLOW_SEAICE
134  #  include "SEAICE.h"  #  include "SEAICE.h"
135    #  include "SEAICE_COST.h"
136  # endif  # endif
137  # ifdef ALLOW_KPP  # ifdef ALLOW_KPP
138  #  include "KPP.h"  #  include "KPP.h"
# Line 117  c They provide the fields to be stored. Line 140  c They provide the fields to be stored.
140  # ifdef ALLOW_GMREDI  # ifdef ALLOW_GMREDI
141  #  include "GMREDI.h"  #  include "GMREDI.h"
142  # endif  # endif
143    # ifdef ALLOW_RBCS
144    #  include "RBCS.h"
145    # endif
146    # ifdef ALLOW_PROFILES
147    #  include "profiles.h"
148    # endif
149  # ifdef ALLOW_DIVIDED_ADJOINT_MPI  # ifdef ALLOW_DIVIDED_ADJOINT_MPI
150  #  include "mpif.h"  #  include "mpif.h"
151  # endif  # endif
# Line 156  c     == local variables == Line 185  c     == local variables ==
185        integer ilev_1        integer ilev_1
186        integer ilev_2        integer ilev_2
187        integer ilev_3        integer ilev_3
188          integer ilev_4
189        integer max_lev2        integer max_lev2
190        integer max_lev3        integer max_lev3
191          integer max_lev4
192  #endif  #endif
193    
194    #if defined(TIME_PER_TIMESTEP) || defined(USE_PAPI_FLOPS) || defined(USE_PCL_FLOPS)
195          CHARACTER*(MAX_LEN_MBUF) msgBuf
196    #ifdef TIME_PER_TIMESTEP
197    CCE107 common block for per timestep timing
198    C     !TIMING VARIABLES
199    C     == Timing variables ==
200          REAL*8 utnew, utold, stnew, stold, wtnew, wtold
201          DATA utnew, utold, stnew, stold, wtnew, wtold /6*0.0D0/
202    #endif
203    #ifdef USE_PAPI_FLOPS
204    CCE107 common block for PAPI summary performance
205    #include <fpapi.h>
206          INTEGER*8 flpops, instr
207          DATA flpops, instr /2*0/
208          INTEGER check
209          REAL*4 real_time, proc_time, mflops, ipc
210          DATA real_time, proc_time, mflops, ipc /4*0.0E0/
211    #else
212    #ifdef USE_PCL_FLOPS
213    CCE107 common block for PCL summary performance
214    #include <pclh.f>
215          INTEGER pcl_counter_list(5), flags, nevents, res, ipcl
216          INTEGER*8 i_result(5), descr
217          REAL*8 fp_result(5)
218          COMMON /pclvars/ i_result, descr, fp_result, pcl_counter_list,
219         $     flags, nevents
220          INTEGER nmaxevents
221          PARAMETER (nmaxevents = 61)
222          CHARACTER*22 pcl_counter_name(0:nmaxevents-1)
223          COMMON /pclnames/ pcl_counter_name
224    #endif
225    #endif
226    #endif
227  c--   == end of interface ==  c--   == end of interface ==
228    
229  #ifndef DISABLE_DEBUGMODE  #ifndef DISABLE_DEBUGMODE
# Line 171  c--   == end of interface == Line 235  c--   == end of interface ==
235  c--   Initialize storage for the initialisations.  c--   Initialize storage for the initialisations.
236  CADJ INIT tapelev_ini_bibj_k   = USER  CADJ INIT tapelev_ini_bibj_k   = USER
237  CADJ INIT tapelev_init = USER  CADJ INIT tapelev_init = USER
238  #ifdef AUTODIFF_2_LEVEL_CHECKPOINT  c
239    #if (defined (AUTODIFF_2_LEVEL_CHECKPOINT))
240  CADJ  INIT tapelev2 = USER  CADJ  INIT tapelev2 = USER
241    #elif (defined (AUTODIFF_4_LEVEL_CHECKPOINT))
242    CADJ  INIT tapelev4 = USER
243  #else  #else
244  CADJ  INIT tapelev3 = USER  CADJ  INIT tapelev3 = USER
245  #endif  #endif
246    c
247  # ifdef ALLOW_DIVIDED_ADJOINT  # ifdef ALLOW_DIVIDED_ADJOINT
248  CADJ INIT onetape = user  CADJ INIT onetape = user
249  cphCADJ INIT onetape = common, 1  cphCADJ INIT onetape = common, 1
# Line 193  cph because we only have active file han Line 261  cph because we only have active file han
261    
262        CALL TIMER_START('ECCO SPIN-UP', mythid)        CALL TIMER_START('ECCO SPIN-UP', mythid)
263    
264    #ifdef ALLOW_CAL
265  c--   Get the current date.  c--   Get the current date.
266        call CAL_TIMESTAMP( myiter, mytime, mydate, mythid )        call CAL_TIMESTAMP( myiter, mytime, mydate, mythid )
267    #endif
268    
269    #ifdef ALLOW_AUTODIFF_TAMC
270    # ifdef NONLIN_FRSURF
271    CADJ STORE hFacC = tapelev_init, key = 1
272    # endif
273    #endif
274    
275  C--   Set initial conditions (variable arrays)  C--   Set initial conditions (variable arrays)
276  #ifndef DISABLE_DEBUGMODE  #ifndef DISABLE_DEBUGMODE
# Line 233  C--   Do IO if needed (Dump for start st Line 309  C--   Do IO if needed (Dump for start st
309        call timer_stop ('ECCO SPIN-UP', mythid)        call timer_stop ('ECCO SPIN-UP', mythid)
310        _BARRIER        _BARRIER
311    
312    #ifdef TIME_PER_TIMESTEP
313    CCE107 Initial call for timers
314          _BEGIN_MASTER( myThid )
315          CALL TIMER_GET_TIME( utold, stold, wtold )
316          _END_MASTER( myThid )
317    #endif
318    #ifdef USE_PAPI_FLOPS
319    CCE107 Initial call for PAPI
320          _BEGIN_MASTER( myThid )
321    #ifdef USE_FLIPS
322          call PAPIF_flips(real_time, proc_time, flpops, mflops, check)
323    #else
324          call PAPIF_flops(real_time, proc_time, flpops, mflops, check)
325    #endif
326          WRITE(msgBuf,'(A34,F10.6,A,F10.6)')
327         $     'Mflop/s before timestepping:', mflops, ' ', mflops*proc_time
328         $     /(real_time + 1E-36)
329          CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
330    #ifdef PAPI_VERSION
331          call PAPIF_ipc(real_time, proc_time, instr, ipc, check)
332          WRITE(msgBuf,'(A34,F10.6,A,F10.6)')
333         $     'IPC before timestepping:', ipc, ' ', ipc*proc_time
334         $     /(real_time + 1E-36)
335          CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
336    #endif
337          _END_MASTER( myThid )
338    #else
339    #ifdef USE_PCL_FLOPS
340    CCE107 Initial call for PCL
341          _BEGIN_MASTER( myThid )
342          res = PCLstop(descr, i_result, fp_result, nevents)
343          do ipcl = 1, nevents
344             WRITE(msgBuf,'(A22,A26,F10.6)'),
345         $        pcl_counter_name(pcl_counter_list(ipcl)),
346         $        'before timestepping:', fp_result(ipcl)
347             CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
348          enddo
349          res = PCLstart(descr, pcl_counter_list, nevents, flags)
350          _END_MASTER( myThid )
351    #endif
352    #endif
353    
354  c--   Do the model integration.  c--   Do the model integration.
355        call timer_start('ECCO MAIN LOOP',mythid)        call timer_start('ECCO MAIN LOOP',mythid)
356    
# Line 242  c     >>>>>>>>>>>>>>>>>>>>>>>>>>>  START Line 360  c     >>>>>>>>>>>>>>>>>>>>>>>>>>>  START
360  #ifdef ALLOW_AUTODIFF_TAMC  #ifdef ALLOW_AUTODIFF_TAMC
361  #ifdef ALLOW_TAMC_CHECKPOINTING  #ifdef ALLOW_TAMC_CHECKPOINTING
362    
363          max_lev4=nTimeSteps/(nchklev_1*nchklev_2*nchklev_3)+1
364        max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1        max_lev3=nTimeSteps/(nchklev_1*nchklev_2)+1
365        max_lev2=nTimeSteps/nchklev_1+1        max_lev2=nTimeSteps/nchklev_1+1
366    
# Line 251  CADJ loop = divided Line 370  CADJ loop = divided
370  #endif  #endif
371  c**************************************  c**************************************
372    
373  #ifndef AUTODIFF_2_LEVEL_CHECKPOINT  #ifdef AUTODIFF_4_LEVEL_CHECKPOINT
374          do ilev_4 = 1,nchklev_4
375             if(ilev_4.le.max_lev4) then
376    c**************************************
377    #include "checkpoint_lev4_directives.h"
378    c**************************************
379    c--     Initialise storage for the middle loop.
380    CADJ    INIT tapelev3 = USER
381    #endif /* AUTODIFF_4_LEVEL_CHECKPOINT */
382    
383    #ifndef AUTODIFF_2_LEVEL_CHECKPOINT
384        do ilev_3 = 1,nchklev_3        do ilev_3 = 1,nchklev_3
385           if(ilev_3.le.max_lev3) then           if(ilev_3.le.max_lev3) then
386  c**************************************  c**************************************
387  #include "checkpoint_lev3_directives.h"  #include "checkpoint_lev3_directives.h"
388  c**************************************  c**************************************
   
389  c--     Initialise storage for the middle loop.  c--     Initialise storage for the middle loop.
390  CADJ    INIT tapelev2 = USER  CADJ    INIT tapelev2 = USER
   
391  #endif /* AUTODIFF_2_LEVEL_CHECKPOINT */  #endif /* AUTODIFF_2_LEVEL_CHECKPOINT */
392    
393          do ilev_2 = 1,nchklev_2          do ilev_2 = 1,nchklev_2
# Line 292  c-- Line 418  c--
418  # ifdef ALLOW_PTRACERS  # ifdef ALLOW_PTRACERS
419  CADJ INIT comlev1_bibj_ptracers = COMMON,  CADJ INIT comlev1_bibj_ptracers = COMMON,
420  CADJ &    nchklev_1*nsx*nsy*nthreads_chkpt*PTRACERS_num  CADJ &    nchklev_1*nsx*nsy*nthreads_chkpt*PTRACERS_num
421    CADJ INIT comlev1_bibj_k_ptracers = COMMON,
422    CADJ &    nchklev_1*nsx*nsy*nthreads_chkpt*PTRACERS_num*nr
423  # endif /* ALLOW_PTRACERS */  # endif /* ALLOW_PTRACERS */
424  c--  c--
425  # ifndef DISABLE_MULTIDIM_ADVECTION  cph Now also needed by seaice
426    cph# ifndef DISABLE_MULTIDIM_ADVECTION
427  CADJ INIT comlev1_bibj_k_gad  CADJ INIT comlev1_bibj_k_gad
428  CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass  CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass
429  CADJ INIT comlev1_bibj_k_gad_pass  CADJ INIT comlev1_bibj_k_gad_pass
430  CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass*maxcube  CADJ &    = COMMON,nchklev_1*nsx*nsy*nr*nthreads_chkpt*maxpass*maxcube
431  # endif /* DISABLE_MULTIDIM_ADVECTION */  cph# endif /* DISABLE_MULTIDIM_ADVECTION */
432  c--  c--
433  # if (defined (ALLOW_EXF) && defined (ALLOW_BULKFORMULAE))  # if (defined (ALLOW_EXF) && defined (ALLOW_BULKFORMULAE))
434  CADJ INIT comlev1_exf_1  CADJ INIT comlev1_exf_1
# Line 328  c--         choice of the 3-tupel ( nchk Line 457  c--         choice of the 3-tupel ( nchk
457  c--  c--
458  c--         Requirement: nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps .  c--         Requirement: nchklev_1*nchklev_2*nchklev_3 .ge. nTimeSteps .
459    
460              iloop = (ilev_2 - 1)*nchklev_1           + ilev_1              iloop = (ilev_2 - 1)*nchklev_1                     + ilev_1
461  #ifndef AUTODIFF_2_LEVEL_CHECKPOINT  #ifndef AUTODIFF_2_LEVEL_CHECKPOINT
462       &            + (ilev_3 - 1)*nchklev_2*nchklev_1       &            + (ilev_3 - 1)*nchklev_2*nchklev_1
463  #endif  #endif
464    #ifdef AUTODIFF_4_LEVEL_CHECKPOINT
465         &            + (ilev_4 - 1)*nchklev_3*nchklev_2*nchklev_1
466    #endif
467    
468              if ( iloop .le. nTimeSteps ) then              if ( iloop .le. nTimeSteps ) then
469    
# Line 366  c--   NOT enabled. Line 498  c--   NOT enabled.
498  #endif /* ALLOW_AUTODIFF_TAMC */  #endif /* ALLOW_AUTODIFF_TAMC */
499    
500  #ifdef ALLOW_TAMC_CHECKPOINTING  #ifdef ALLOW_TAMC_CHECKPOINTING
501          nIter0 = INT( startTime/deltaTClock )          nIter0 = NINT( (startTime-baseTime)/deltaTClock )
502          ikey_dynamics = ilev_1          ikey_dynamics = ilev_1
503  #endif  #endif
504    
# Line 374  c--     Set the model iteration counter Line 506  c--     Set the model iteration counter
506          myiter = nIter0 + (iloop-1)          myiter = nIter0 + (iloop-1)
507          mytime = startTime + float(iloop-1)*deltaTclock          mytime = startTime + float(iloop-1)*deltaTclock
508    
509    #ifdef ALLOW_AUTODIFF_TAMC
510            CALL AUTODIFF_INADMODE_UNSET( myThid )
511    #endif
512    
513    #ifdef ALLOW_DIAGNOSTICS
514    C--   State-variables diagnostics
515          IF ( useDiagnostics ) THEN
516    C--   Switch on/off diagnostics for snap-shot output:
517            CALL DIAGNOSTICS_SWITCH_ONOFF( myTime, myIter, myThid )
518            CALL TIMER_START('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)
519            CALL DO_STATEVARS_DIAGS( myTime, 0, myIter, myThid )
520            CALL TIMER_STOP ('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)
521          ENDIF
522    #endif
523    
524    #ifdef ALLOW_PROFILES
525    c--     Accumulate in-situ time averages of temperature, salinity, and SSH.
526            call timer_start('PROFILES_INLOOP       [ECCO MAIN]', mythid)
527            call profiles_inloop( mytime, mythid )
528            call timer_stop ('PROFILES_INLOOP       [ECCO MAIN]', mythid)
529    #endif
530    
531  #ifdef ALLOW_COST  #ifdef ALLOW_COST
532    
533  c--     Accumulate time averages of temperature, salinity, and SSH.  c--     Accumulate time averages of temperature, salinity
534          call timer_start('COST_AVERAGESFIELDS   [ECCO MAIN]', mythid)          call timer_start('COST_AVERAGESFIELDS   [ECCO MAIN]', mythid)
535          call cost_averagesFields( mytime, mythid )          call cost_averagesFields( mytime, mythid )
536          call timer_stop ('COST_AVERAGESFIELDS   [ECCO MAIN]', mythid)          call timer_stop ('COST_AVERAGESFIELDS   [ECCO MAIN]', mythid)
537    
538    
539  #ifdef ALLOW_COST_ATLANTIC  #ifdef ALLOW_COST_ATLANTIC
540  c--     Compute meridional heat transport  c--     Compute meridional heat transport
541          call timer_start('cost_atlantic         [ECCO MAIN]', mythid)          call timer_start('cost_atlantic         [ECCO MAIN]', mythid)
# Line 391  c--     Compute meridional heat transpor Line 547  c--     Compute meridional heat transpor
547  #ifdef ALLOW_AUTODIFF_TAMC  #ifdef ALLOW_AUTODIFF_TAMC
548  c**************************************  c**************************************
549  #include "checkpoint_lev1_directives.h"  #include "checkpoint_lev1_directives.h"
550    #include "checkpoint_lev1_template.h"
551  c**************************************  c**************************************
552  #endif  #endif
553    
554  #ifndef DISABLE_DEBUGMODE  C--   Call driver to load external forcing fields from file
555        IF ( debugLevel .GE. debLevB )  #ifdef ALLOW_DEBUG
556       &    CALL DEBUG_CALL('EXF_GETFORCING',myThid)        IF ( debugLevel .GE. debLevB )
557         &    CALL DEBUG_CALL('LOAD_FIELDS_DRIVER',myThid)
558  #endif  #endif
559        CALL TIMER_START('EXF_GETFORCING     [FORWARD_STEP]',mythid)        CALL TIMER_START('LOAD_FIELDS_DRIVER  [FORWARD_STEP]',myThid)
560        CALL EXF_GETFORCING( mytime, myiter, mythid )        CALL LOAD_FIELDS_DRIVER( myTime, myIter, myThid )
561        CALL TIMER_STOP ('EXF_GETFORCING     [FORWARD_STEP]',mythid)        CALL TIMER_STOP ('LOAD_FIELDS_DRIVER  [FORWARD_STEP]',myThid)
562    
563  #ifdef ALLOW_SEAICE  
564  cph this simple runtime flag causes a lot of recomp.  #ifdef ALLOW_AUTODIFF_TAMC
565  cph      IF ( useSEAICE ) THEN  # if (defined (ALLOW_AUTODIFF_MONITOR))
566  #ifndef DISABLE_DEBUGMODE          CALL DUMMY_IN_STEPPING( myTime, myIter, myThid )
567           IF ( debugLevel .GE. debLevB )  # endif
      &    CALL DEBUG_CALL('SEAICE_MODEL',myThid)  
 #endif  
          CALL TIMER_START('SEAICE_MODEL       [FORWARD_STEP]',myThid)  
          CALL SEAICE_MODEL( myTime, myIter, myThid )  
          CALL TIMER_STOP ('SEAICE_MODEL       [FORWARD_STEP]',myThid)  
 #ifdef ALLOW_COST_ICE  
          CALL COST_ICE    ( myTime, myIter, myThid )  
568  #endif  #endif
 cph      ENDIF  
 #endif /* ALLOW_SEAICE */  
569    
570  #ifdef ALLOW_AUTODIFF_TAMC  #ifdef ALLOW_AUTODIFF_TAMC
571  # ifdef ALLOW_PTRACERS  # ifdef ALLOW_PTRACERS
# Line 425  CADJ STORE ptracer  = comlev1, key = ike Line 574  CADJ STORE ptracer  = comlev1, key = ike
574  # endif  # endif
575  #endif  #endif
576    
 #if (defined (ALLOW_AUTODIFF_TAMC) && \  
      defined (ALLOW_AUTODIFF_MONITOR))  
 C       Include call to a dummy routine. Its adjoint will be  
 C       called at the proper place in the adjoint code.  
 C       The adjoint routine will print out adjoint values  
 C       if requested. The location of the call is important,  
 C       it has to be after the adjoint of the exchanges  
 C       (DO_GTERM_BLOCKING_EXCHANGES).  
         CALL DUMMY_IN_STEPPING( myTime, myIter, myThid )  
 #endif  
   
577  #ifdef ALLOW_EBM  #ifdef ALLOW_EBM
578          IF ( useEBM ) THEN          IF ( useEBM ) THEN
579  # ifdef ALLOW_DEBUG  # ifdef ALLOW_DEBUG
# Line 458  C--     Step forward fields and calculat Line 596  C--     Step forward fields and calculat
596         CALL DO_ATMOSPHERIC_PHYS( myTime, myIter, myThid )         CALL DO_ATMOSPHERIC_PHYS( myTime, myIter, myThid )
597         CALL TIMER_STOP ('DO_ATMOSPHERIC_PHYS [FORWARD_STEP]',mythid)         CALL TIMER_STOP ('DO_ATMOSPHERIC_PHYS [FORWARD_STEP]',mythid)
598    
599    #ifdef ALLOW_AUTODIFF_TAMC
600    # ifdef EXACT_CONSERV
601    cphCADJ STORE empmr              = comlev1, key = ikey_dynamics
602    cphCADJ STORE pmepr              = comlev1, key = ikey_dynamics
603    # endif
604    # ifdef ALLOW_PTRACERS
605    CADJ STORE ptracer               = comlev1, key = ikey_dynamics
606    # endif
607    # ifdef NONLIN_FRSURF
608    cph-test
609    CADJ STORE hFacC              = comlev1, key = ikey_dynamics
610    # endif
611    #endif /* ALLOW_AUTODIFF_TAMC */
612    
613  #ifndef ALLOW_OFFLINE  #ifndef ALLOW_OFFLINE
614  #ifdef ALLOW_DEBUG  #ifdef ALLOW_DEBUG
615         IF ( debugLevel .GE. debLevB )         IF ( debugLevel .GE. debLevB )
# Line 466  C--     Step forward fields and calculat Line 618  C--     Step forward fields and calculat
618         CALL TIMER_START('DO_OCEANIC_PHYS     [FORWARD_STEP]',mythid)         CALL TIMER_START('DO_OCEANIC_PHYS     [FORWARD_STEP]',mythid)
619         CALL DO_OCEANIC_PHYS( myTime, myIter, myThid )         CALL DO_OCEANIC_PHYS( myTime, myIter, myThid )
620         CALL TIMER_STOP ('DO_OCEANIC_PHYS     [FORWARD_STEP]',mythid)         CALL TIMER_STOP ('DO_OCEANIC_PHYS     [FORWARD_STEP]',mythid)
621    #ifdef ALLOW_AUTODIFF_TAMC
622    CADJ STORE EmPmR    = comlev1, key = ikey_dynamics
623    # ifdef EXACT_CONSERV
624    CADJ STORE pmepr    = comlev1, key = ikey_dynamics
625    # endif
626    #endif
627  #endif  #endif
628    
629  #ifdef ALLOW_AUTODIFF_TAMC  #ifdef ALLOW_AUTODIFF_TAMC
630    # ifdef NONLIN_FRSURF
631    cph-test
632    CADJ STORE hFac_surfC         = comlev1, key = ikey_dynamics
633    CADJ STORE hfac_surfs         = comlev1, key = ikey_dynamics
634    CADJ STORE hfac_surfw         = comlev1, key = ikey_dynamics
635    CADJ STORE hFacC, hFacS, hFacW
636    CADJ &     = comlev1, key = ikey_dynamics
637    CADJ STORE recip_hFacC, recip_hFacS, recip_hFacW
638    CADJ &     = comlev1, key = ikey_dynamics
639    c
640    CADJ STORE surfaceforcingu = comlev1, key = ikey_dynamics
641    CADJ STORE surfaceforcingv = comlev1, key = ikey_dynamics
642    # endif
643    #endif /* ALLOW_AUTODIFF_TAMC */
644    
645    #ifdef ALLOW_GCHEM
646    C     GCHEM package is an interface for any bio-geochemical or
647    C     ecosystem model you would like to include.
648    C     If GCHEM_SEPARATE_FORCING is not defined, you are
649    C     responsible for computing tendency terms for passive
650    C     tracers and storing them on a 3DxNumPtracers-array called
651    C     gchemTendency in GCHEM_CALC_TENDENCY. This tendency is then added
652    C     to gPtr in ptracers_forcing later-on.
653    C     If GCHEM_SEPARATE_FORCING is defined, you are reponsible for
654    C     UPDATING ptracers directly in GCHEM_FORCING_SEP. This amounts
655    C     to a completely separate time step that you have to implement
656    C     yourself (Eulerian seems to be fine in most cases).
657    CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
658    C     CAVEAT: Up to now, when GCHEM is turned on the field ptracerForcingSurf,
659    C     which is needed for KPP is not set properly. ptracerForcingSurf must
660    C     be treated differently depending on whether GCHEM_SEPARATE_FORCING
661    C     is define or not. TBD.
662    CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
663           IF ( useGCHEM ) THEN
664    #ifdef ALLOW_DEBUG
665            IF ( debugLevel .GE. debLevB )
666         &       CALL DEBUG_CALL('GCHEM_CALC_TENDENCY',myThid)
667    #endif
668            CALL TIMER_START('GCHEM_CALC_TENDENCY [FORWARD_STEP]',myThid)
669            CALL GCHEM_CALC_TENDENCY( myTime, myIter, myThid )
670            CALL TIMER_STOP ('GCHEM_CALC_TENDENCY [FORWARD_STEP]',myThid)
671           ENDIF
672    #endif /* ALLOW_GCHEM */
673    
674    #ifdef ALLOW_AUTODIFF_TAMC
675  cph needed to be moved here from do_oceanic_physics  cph needed to be moved here from do_oceanic_physics
676  cph to be visible down the road  cph to be visible down the road
677  c  c
# Line 500  CADJ STORE KPPdiffKzT         = comlev1, Line 703  CADJ STORE KPPdiffKzT         = comlev1,
703  # endif  # endif
704  #endif /* ALLOW_AUTODIFF_TAMC */  #endif /* ALLOW_AUTODIFF_TAMC */
705    
706    #ifdef ALLOW_AUTODIFF_TAMC
707    # ifdef NONLIN_FRSURF
708    CADJ STORE etaH               = comlev1, key = ikey_dynamics
709    #  ifdef ALLOW_CD_CODE
710    CADJ STORE etanm1             = comlev1, key = ikey_dynamics
711    #  endif
712    # endif
713    #endif /* ALLOW_AUTODIFF_TAMC */
714    
715        IF ( .NOT.staggerTimeStep ) THEN        IF ( .NOT.staggerTimeStep ) THEN
716  #ifdef ALLOW_DEBUG  #ifdef ALLOW_DEBUG
# Line 512  CADJ STORE KPPdiffKzT         = comlev1, Line 723  CADJ STORE KPPdiffKzT         = comlev1,
723  C--    if not staggerTimeStep: end  C--    if not staggerTimeStep: end
724        ENDIF        ENDIF
725    
726    #ifdef ALLOW_AUTODIFF_TAMC
727    # ifdef NONLIN_FRSURF
728    CADJ STORE hFacC          = comlev1, key = ikey_dynamics
729    CADJ STORE hFacS          = comlev1, key = ikey_dynamics
730    CADJ STORE hFacW          = comlev1, key = ikey_dynamics
731    CADJ STORE recip_hFacC    = comlev1, key = ikey_dynamics
732    CADJ STORE recip_hFacS    = comlev1, key = ikey_dynamics
733    CADJ STORE recip_hFacW    = comlev1, key = ikey_dynamics
734    CADJ STORE etaN           = comlev1, key = ikey_dynamics
735    # endif
736    #endif
737    
738  C--   Step forward fields and calculate time tendency terms.  C--   Step forward fields and calculate time tendency terms.
739  #ifndef ALLOW_OFFLINE  #ifndef ALLOW_OFFLINE
740  #ifndef ALLOW_AUTODIFF_TAMC  #ifndef ALLOW_AUTODIFF_TAMC
# Line 529  C--   Step forward fields and calculate Line 752  C--   Step forward fields and calculate
752  #endif  #endif
753  #endif  #endif
754    
755  #ifdef ALLOW_NONHYDROSTATIC  #ifdef ALLOW_AUTODIFF_TAMC
756  C--   Step forward W field in N-H algorithm  # ifdef NONLIN_FRSURF
757        IF ( momStepping .AND. nonHydrostatic ) THEN  cph-test
758  #ifdef ALLOW_DEBUG  CADJ STORE gU, gV  = comlev1, key = ikey_dynamics
759            IF ( debugLevel .GE. debLevB )  # endif
      &     CALL DEBUG_CALL('CALC_GW',myThid)  
 #endif  
          CALL TIMER_START('CALC_GW          [FORWARD_STEP]',myThid)  
          CALL CALC_GW(myThid)  
          CALL TIMER_STOP ('CALC_GW          [FORWARD_STEP]',myThid)  
       ENDIF  
760  #endif  #endif
761    
762  C--   Update time-counter  C--   Update time-counter
763        myIter = nIter0 + iLoop        myIter = nIter0 + iLoop
764        myTime = startTime + deltaTClock * float(iLoop)        myTime = startTime + deltaTClock * float(iLoop)
765    
766    #ifdef ALLOW_MNC
767    C     Update the default next iter for MNC
768          IF ( useMNC ) THEN
769             CALL MNC_CW_CITER_SETG( 1, 1, -1, myIter , myThid )
770    
771    C        TODO: Logic should be added here so that users can specify, on
772    C        a per-citer-group basis, when it is time to update the
773    C        "current" (and not just the "next") iteration
774    
775    C        TODO: the following is just a temporary band-aid (mostly, for
776    C        Baylor) until someone writes a routine that better handles time
777    C        boundaries such as weeks, months, years, etc.
778             IF ( mnc_filefreq .GT. 0 ) THEN
779               IF (DIFFERENT_MULTIPLE(mnc_filefreq,myTime,deltaTClock))
780         &          THEN
781                 CALL MNC_CW_CITER_SETG( 1, 1, myIter, -1 , myThid )
782               ENDIF
783             ENDIF
784           ENDIF
785    #endif
786    
787  C--   Update geometric factors:  C--   Update geometric factors:
788  #ifdef NONLIN_FRSURF  #ifdef NONLIN_FRSURF
789  C-    update hfacC,W,S and recip_hFac according to etaH(n+1) :  C-    update hfacC,W,S and recip_hFac according to etaH(n+1) :
790        IF ( nonlinFreeSurf.GT.0) THEN        IF ( nonlinFreeSurf.GT.0) THEN
791         IF ( select_rStar.GT.0 ) THEN         IF ( select_rStar.GT.0 ) THEN
792    # ifndef DISABLE_RSTAR_CODE
793    # ifdef ALLOW_AUTODIFF_TAMC
794    cph-test
795    CADJ STORE hFacC    = comlev1, key = ikey_dynamics
796    CADJ STORE hFacS    = comlev1, key = ikey_dynamics
797    CADJ STORE hFacW    = comlev1, key = ikey_dynamics
798    CADJ STORE recip_hFacC    = comlev1, key = ikey_dynamics
799    CADJ STORE recip_hFacS    = comlev1, key = ikey_dynamics
800    CADJ STORE recip_hFacW    = comlev1, key = ikey_dynamics
801    # endif
802          CALL TIMER_START('UPDATE_R_STAR      [FORWARD_STEP]',myThid)          CALL TIMER_START('UPDATE_R_STAR      [FORWARD_STEP]',myThid)
803          CALL UPDATE_R_STAR( myTime, myIter, myThid )          CALL UPDATE_R_STAR( myTime, myIter, myThid )
804          CALL TIMER_STOP ('UPDATE_R_STAR      [FORWARD_STEP]',myThid)          CALL TIMER_STOP ('UPDATE_R_STAR      [FORWARD_STEP]',myThid)
805    # ifdef ALLOW_AUTODIFF_TAMC
806    cph-test
807    CADJ STORE hFacC    = comlev1, key = ikey_dynamics
808    CADJ STORE hFacS    = comlev1, key = ikey_dynamics
809    CADJ STORE hFacW    = comlev1, key = ikey_dynamics
810    CADJ STORE recip_hFacC    = comlev1, key = ikey_dynamics
811    CADJ STORE recip_hFacS    = comlev1, key = ikey_dynamics
812    CADJ STORE recip_hFacW    = comlev1, key = ikey_dynamics
813    # endif
814    # endif /* DISABLE_RSTAR_CODE */
815         ELSE         ELSE
816    #ifdef ALLOW_AUTODIFF_TAMC
817    CADJ STORE hFac_surfC, hFac_surfS, hFac_surfW
818    CADJ &     = comlev1, key = ikey_dynamics
819    #endif
820          CALL TIMER_START('UPDATE_SURF_DR     [FORWARD_STEP]',myThid)          CALL TIMER_START('UPDATE_SURF_DR     [FORWARD_STEP]',myThid)
821          CALL UPDATE_SURF_DR( myTime, myIter, myThid )          CALL UPDATE_SURF_DR( myTime, myIter, myThid )
822          CALL TIMER_STOP ('UPDATE_SURF_DR     [FORWARD_STEP]',myThid)          CALL TIMER_STOP ('UPDATE_SURF_DR     [FORWARD_STEP]',myThid)
823         ENDIF         ENDIF
824        ENDIF        ENDIF
825    # ifdef ALLOW_AUTODIFF_TAMC
826    cph-test
827    CADJ STORE hFacC    = comlev1, key = ikey_dynamics
828    CADJ STORE hFacS    = comlev1, key = ikey_dynamics
829    CADJ STORE hFacW    = comlev1, key = ikey_dynamics
830    CADJ STORE recip_hFacC    = comlev1, key = ikey_dynamics
831    CADJ STORE recip_hFacS    = comlev1, key = ikey_dynamics
832    CADJ STORE recip_hFacW    = comlev1, key = ikey_dynamics
833    # endif
834  C-    update also CG2D matrix (and preconditioner)  C-    update also CG2D matrix (and preconditioner)
835        IF ( momStepping .AND. nonlinFreeSurf.GT.2 ) THEN        IF ( momStepping .AND. nonlinFreeSurf.GT.2 ) THEN
836          CALL TIMER_START('UPDATE_CG2D        [FORWARD_STEP]',myThid)          CALL TIMER_START('UPDATE_CG2D        [FORWARD_STEP]',myThid)
837          CALL UPDATE_CG2D( myTime, myIter, myThid )          CALL UPDATE_CG2D( myTime, myIter, myThid )
838          CALL TIMER_STOP ('UPDATE_CG2D        [FORWARD_STEP]',myThid)          CALL TIMER_STOP ('UPDATE_CG2D        [FORWARD_STEP]',myThid)
839        ENDIF        ENDIF
840  #endif  #endif /* NONLIN_FRSURF */
841    
842  C--   Apply Filters to u*,v* before SOLVE_FOR_PRESSURE  C--   Apply Filters to u*,v* before SOLVE_FOR_PRESSURE
843  #ifdef ALLOW_SHAP_FILT  #ifdef ALLOW_SHAP_FILT
# Line 600  C     Two-dimensional only for conventio Line 871  C     Two-dimensional only for conventio
871  C     three-dimensional for non-hydrostatic and/or IGW scheme.  C     three-dimensional for non-hydrostatic and/or IGW scheme.
872  #ifndef ALLOW_OFFLINE  #ifndef ALLOW_OFFLINE
873        IF ( momStepping ) THEN        IF ( momStepping ) THEN
874    #ifdef ALLOW_AUTODIFF_TAMC
875    # ifdef NONLIN_FRSURF
876    CADJ STORE uvel, vvel
877    CADJ &     = comlev1, key = ikey_dynamics
878    CADJ STORE empmr,hfacs,hfacw
879    CADJ &     = comlev1, key = ikey_dynamics
880    # endif
881    #endif
882          CALL TIMER_START('SOLVE_FOR_PRESSURE  [FORWARD_STEP]',myThid)          CALL TIMER_START('SOLVE_FOR_PRESSURE  [FORWARD_STEP]',myThid)
883          CALL SOLVE_FOR_PRESSURE(myTime, myIter, myThid)          CALL SOLVE_FOR_PRESSURE(myTime, myIter, myThid)
884          CALL TIMER_STOP ('SOLVE_FOR_PRESSURE  [FORWARD_STEP]',myThid)          CALL TIMER_STOP ('SOLVE_FOR_PRESSURE  [FORWARD_STEP]',myThid)
# Line 617  c     ENDIF Line 896  c     ENDIF
896    
897  #ifdef EXACT_CONSERV  #ifdef EXACT_CONSERV
898        IF (exactConserv) THEN        IF (exactConserv) THEN
899    #ifdef ALLOW_AUTODIFF_TAMC
900    cph-test
901    cphCADJ STORE etaH          = comlev1, key = ikey_dynamics
902    #endif
903  C--   Update etaH(n+1) :  C--   Update etaH(n+1) :
904          CALL TIMER_START('UPDATE_ETAH        [FORWARD_STEP]',mythid)          CALL TIMER_START('UPDATE_ETAH        [FORWARD_STEP]',mythid)
905          CALL UPDATE_ETAH( myTime, myIter, myThid )          CALL UPDATE_ETAH( myTime, myIter, myThid )
# Line 626  C--   Update etaH(n+1) : Line 909  C--   Update etaH(n+1) :
909    
910  #ifdef NONLIN_FRSURF  #ifdef NONLIN_FRSURF
911        IF ( select_rStar.NE.0 ) THEN        IF ( select_rStar.NE.0 ) THEN
912    # ifndef DISABLE_RSTAR_CODE
913  C--   r* : compute the future level thickness according to etaH(n+1)  C--   r* : compute the future level thickness according to etaH(n+1)
914          CALL TIMER_START('CALC_R_STAR       [FORWARD_STEP]',mythid)          CALL TIMER_START('CALC_R_STAR       [FORWARD_STEP]',mythid)
915          CALL CALC_R_STAR(etaH, myTime, myIter, myThid )          CALL CALC_R_STAR(etaH, myTime, myIter, myThid )
916          CALL TIMER_STOP ('CALC_R_STAR       [FORWARD_STEP]',mythid)          CALL TIMER_STOP ('CALC_R_STAR       [FORWARD_STEP]',mythid)
917    # endif /* DISABLE_RSTAR_CODE */
918        ELSEIF ( nonlinFreeSurf.GT.0) THEN        ELSEIF ( nonlinFreeSurf.GT.0) THEN
919  C--   compute the future surface level thickness according to etaH(n+1)  C--   compute the future surface level thickness according to etaH(n+1)
920    # ifdef ALLOW_AUTODIFF_TAMC
921    CADJ STORE etaH          = comlev1, key = ikey_dynamics
922    # endif
923          CALL TIMER_START('CALC_SURF_DR      [FORWARD_STEP]',mythid)          CALL TIMER_START('CALC_SURF_DR      [FORWARD_STEP]',mythid)
924          CALL CALC_SURF_DR(etaH, myTime, myIter, myThid )          CALL CALC_SURF_DR(etaH, myTime, myIter, myThid )
925          CALL TIMER_STOP ('CALC_SURF_DR      [FORWARD_STEP]',mythid)          CALL TIMER_STOP ('CALC_SURF_DR      [FORWARD_STEP]',mythid)
926        ENDIF        ENDIF
927    # ifdef ALLOW_AUTODIFF_TAMC
928    cph-test
929    CADJ STORE hFac_surfC    = comlev1, key = ikey_dynamics
930    # endif
931  #endif /* NONLIN_FRSURF */  #endif /* NONLIN_FRSURF */
932    
933  C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|  C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
# Line 649  C--   do exchanges of U,V (needed for mu Line 941  C--   do exchanges of U,V (needed for mu
941          CALL DO_STAGGER_FIELDS_EXCHANGES( myTime, myIter, myThid )          CALL DO_STAGGER_FIELDS_EXCHANGES( myTime, myIter, myThid )
942          CALL TIMER_STOP ('BLOCKING_EXCHANGES  [FORWARD_STEP]',myThid)          CALL TIMER_STOP ('BLOCKING_EXCHANGES  [FORWARD_STEP]',myThid)
943    
944    #ifdef ALLOW_DIAGNOSTICS
945    C--   State-variables diagnostics
946           IF ( usediagnostics ) THEN
947            CALL TIMER_START('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)
948            CALL DO_STATEVARS_DIAGS( myTime, 1, myIter, myThid )
949            CALL TIMER_STOP ('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)
950           ENDIF
951    #endif
952    
953  #ifdef ALLOW_DEBUG  #ifdef ALLOW_DEBUG
954          IF ( debugLevel .GE. debLevB )          IF ( debugLevel .GE. debLevB )
955       &    CALL DEBUG_CALL('THERMODYNAMICS',myThid)       &    CALL DEBUG_CALL('THERMODYNAMICS',myThid)
# Line 671  C--   Cycle time-stepping Tracers arrays Line 972  C--   Cycle time-stepping Tracers arrays
972          CALL TRACERS_CORRECTION_STEP(myTime, myIter, myThid)          CALL TRACERS_CORRECTION_STEP(myTime, myIter, myThid)
973          CALL TIMER_STOP ('TS_CORRECTION_STEP  [FORWARD_STEP]',myThid)          CALL TIMER_STOP ('TS_CORRECTION_STEP  [FORWARD_STEP]',myThid)
974    
975    #ifdef ALLOW_GCHEM
976    C     Add separate timestepping of chemical/biological/forcing
977    C     of ptracers here in GCHEM_FORCING_SEP
978            IF ( useGCHEM ) THEN
979    #ifdef ALLOW_DEBUG
980             IF ( debugLevel .GE. debLevB )
981         &        CALL DEBUG_CALL('GCHEM_FORCING_SEP',myThid)
982    #endif /* ALLOW_DEBUG */
983             CALL TIMER_START('GCHEM_FORCING_SEP  [FORWARD_STEP]',myThid)
984             CALL GCHEM_FORCING_SEP( myTime,myIter,myThid )
985             CALL TIMER_STOP ('GCHEM_FORCING_SEP  [FORWARD_STEP]',myThid)
986            ENDIF  
987    #endif /* ALLOW_GCHEM */
988    
989  C--   Do "blocking" sends and receives for tendency "overlap" terms  C--   Do "blocking" sends and receives for tendency "overlap" terms
990  c     CALL TIMER_START('BLOCKING_EXCHANGES  [FORWARD_STEP]',myThid)  c     CALL TIMER_START('BLOCKING_EXCHANGES  [FORWARD_STEP]',myThid)
991  c     CALL DO_GTERM_BLOCKING_EXCHANGES( myThid )  c     CALL DO_GTERM_BLOCKING_EXCHANGES( myThid )
# Line 681  C--   Do "blocking" sends and receives f Line 996  C--   Do "blocking" sends and receives f
996        CALL DO_FIELDS_BLOCKING_EXCHANGES( myThid )        CALL DO_FIELDS_BLOCKING_EXCHANGES( myThid )
997        CALL TIMER_STOP ('BLOCKING_EXCHANGES  [FORWARD_STEP]',myThid)        CALL TIMER_STOP ('BLOCKING_EXCHANGES  [FORWARD_STEP]',myThid)
998    
999    #ifdef ALLOW_DIAGNOSTICS
1000          IF ( useDiagnostics ) THEN
1001           CALL TIMER_START('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)
1002           CALL DO_STATEVARS_DIAGS( myTime, 2, myIter, myThid )
1003           CALL TIMER_STOP ('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)
1004          ENDIF
1005    #endif
1006    
1007  #ifdef ALLOW_FLT  #ifdef ALLOW_FLT
1008  C--   Calculate float trajectories  C--   Calculate float trajectories
1009        IF (useFLT) THEN        IF (useFLT) THEN
# Line 690  C--   Calculate float trajectories Line 1013  C--   Calculate float trajectories
1013        ENDIF        ENDIF
1014  #endif  #endif
1015    
1016  C--   State-variables statistics (time-aver, diagnostics ...)  #ifdef ALLOW_AUTODIFF_TAMC
1017        CALL TIMER_START('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)          CALL AUTODIFF_INADMODE_SET( myThid )
1018        CALL DO_STATEVARS_DIAGS( myTime, myIter, myThid )  #endif
1019        CALL TIMER_STOP ('DO_STATEVARS_DIAGS  [FORWARD_STEP]',myThid)  
1020    #ifdef ALLOW_TIMEAVE
1021    C--   State-variables time-averaging
1022          CALL TIMER_START('DO_STATEVARS_TAVE   [FORWARD_STEP]',myThid)
1023          CALL DO_STATEVARS_TAVE( myTime, myIter, myThid )
1024          CALL TIMER_STOP ('DO_STATEVARS_TAVE   [FORWARD_STEP]',myThid)
1025    #endif
1026    
1027  #ifndef ALLOW_OFFLINE  #ifndef ALLOW_OFFLINE
1028  #ifdef ALLOW_MONITOR  #ifdef ALLOW_MONITOR
# Line 715  C--   Do IO if needed. Line 1044  C--   Do IO if needed.
1044        CALL TIMER_STOP ('DO_THE_MODEL_IO     [FORWARD_STEP]',myThid)        CALL TIMER_STOP ('DO_THE_MODEL_IO     [FORWARD_STEP]',myThid)
1045  #endif  #endif
1046    
1047    #ifndef ALLOW_DIVIDED_ADJOINT
1048    # ifdef HAVE_SIGREG
1049          IF ( useSIGREG ) THEN
1050            IF ( i_got_signal .GT. 0 ) THEN
1051              CALL PACKAGES_WRITE_PICKUP(
1052         I         .TRUE., myTime, myIter, myThid )
1053              CALL WRITE_PICKUP(
1054         I         .TRUE., myTime, myIter, myThid )
1055              STOP 'Checkpoint completed -- killed by signal handler'
1056            ENDIF
1057          ENDIF
1058    # endif /* HAVE_SIGREG */
1059  C--   Save state for restarts  C--   Save state for restarts
1060        CALL TIMER_START('WRITE_CHECKPOINT    [FORWARD_STEP]',myThid)        CALL TIMER_START('DO_WRITE_PICKUP     [FORWARD_STEP]',myThid)
1061        CALL PACKAGES_WRITE_PICKUP(        CALL DO_WRITE_PICKUP(
1062       I               .FALSE., myTime, myIter, myThid )       I               .FALSE., myTime, myIter, myThid )
1063  #ifndef ALLOW_OFFLINE        CALL TIMER_STOP ('DO_WRITE_PICKUP     [FORWARD_STEP]',myThid)
1064        CALL WRITE_CHECKPOINT(  #endif /* ALLOW_DIVIDED_ADJOINT */
1065       I               .FALSE., myTime, myIter, myThid )    
1066    #ifdef TIME_PER_TIMESTEP
1067    CCE107 Time per timestep information
1068          _BEGIN_MASTER( myThid )
1069          CALL TIMER_GET_TIME( utnew, stnew, wtnew )
1070          WRITE(msgBuf,'(A34,3F10.6,I8)')
1071         $        'User, system and wallclock time:', utnew - utold,
1072         $        stnew - stold, wtnew - wtold, iloop
1073          CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1074          utold = utnew
1075          stold = stnew
1076          wtold = wtnew
1077          _END_MASTER( myThid )
1078    #endif
1079    #ifdef USE_PAPI_FLOPS
1080    CCE107 PAPI summary performance
1081          _BEGIN_MASTER( myThid )
1082    #ifdef USE_FLIPS
1083          call PAPIF_flips(real_time, proc_time, flpops, mflops, check)
1084    #else
1085          call PAPIF_flops(real_time, proc_time, flpops, mflops, check)
1086    #endif
1087          WRITE(msgBuf,'(F10.6,A,F10.6,A34,I8)')
1088         $     mflops, ' ', mflops*proc_time/(real_time + 1E-36),
1089         $     'Mflop/s during timestep ', iloop
1090          CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1091    #ifdef PAPI_VERSION
1092          call PAPIF_ipc(real_time, proc_time, instr, ipc, check)
1093          WRITE(msgBuf,'(F10.6,A,F10.6,A34,I8)')
1094         $     ipc, ' ', ipc*proc_time/(real_time + 1E-36),
1095         $     'IPC during timestep ', iloop
1096          CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1097    #endif
1098          _END_MASTER( myThid )
1099    #else
1100    #ifdef USE_PCL_FLOPS
1101    CCE107 PCL summary performance
1102          _BEGIN_MASTER( myThid )
1103          res = PCLstop(descr, i_result, fp_result, nevents)
1104          do ipcl = 1, nevents
1105             WRITE(msgBuf,'(F10.6,A2,A22,A17,I8)'), fp_result(ipcl),
1106         $        ' ', pcl_counter_name(pcl_counter_list(ipcl)),
1107         $        'during timestep ', iloop
1108             CALL PRINT_MESSAGE(msgBuf,standardMessageUnit,SQUEEZE_RIGHT,1)
1109          enddo
1110           res = PCLstart(descr, pcl_counter_list, nevents, flags)
1111          _END_MASTER( myThid )
1112    #endif
1113  #endif  #endif
       CALL TIMER_STOP ('WRITE_CHECKPOINT    [FORWARD_STEP]',myThid)  
1114    
1115  #ifdef ALLOW_AUTODIFF_TAMC  #ifdef ALLOW_AUTODIFF_TAMC
1116  #ifdef ALLOW_TAMC_CHECKPOINTING  #ifdef ALLOW_TAMC_CHECKPOINTING
1117              endif               endif
1118                enddo
1119               endif
1120            enddo            enddo
           endif  
         enddo  
1121  #ifndef AUTODIFF_2_LEVEL_CHECKPOINT  #ifndef AUTODIFF_2_LEVEL_CHECKPOINT
1122          endif           endif
1123        enddo          enddo
1124  #endif  #endif
1125  #else  #ifdef AUTODIFF_4_LEVEL_CHECKPOINT
1126           endif
1127        enddo        enddo
1128  #endif  #endif
1129    c
1130    #else /* ndef ALLOW_TAMC_CHECKPOINTING */
1131          enddo
1132    #endif /* ALLOW_TAMC_CHECKPOINTING */
1133    
1134  #else  #else /* ndef ALLOW_AUTODIFF_TAMC */
1135        enddo        enddo
1136  #endif /* ALLOW_AUTODIFF_TAMC */  #endif /* ALLOW_AUTODIFF_TAMC */
1137    
# Line 748  C--   Save state for restarts Line 1140  C--   Save state for restarts
1140    
1141        call timer_start('ECCO SPIN-DOWN', mythid)        call timer_start('ECCO SPIN-DOWN', mythid)
1142    
1143    #ifdef ALLOW_PROFILES
1144    #ifndef ALLOW_DIVIDED_ADJOINT
1145    c--     Accumulate in-situ time averages of temperature, salinity, and SSH.
1146          call timer_start('PROFILES_INLOOP     [ECCO SPIN-DOWN]', mythid)
1147          call profiles_inloop( mytime, mythid )
1148          call timer_stop ('PROFILES_INLOOP     [ECCO SPIN-DOWN]', mythid)
1149    #endif
1150    #endif
1151    
1152  #ifdef ALLOW_COST  #ifdef ALLOW_COST
1153    
1154  #ifdef ALLOW_DIVIDED_ADJOINT  #ifdef ALLOW_DIVIDED_ADJOINT
# Line 787  c--   i.e. heat flux, salt flux, zonal a Line 1188  c--   i.e. heat flux, salt flux, zonal a
1188        call timer_start('cost_forcing       [ECCO SPIN-DOWN]', mythid)        call timer_start('cost_forcing       [ECCO SPIN-DOWN]', mythid)
1189        call cost_forcing( myiter, mytime, mythid )        call cost_forcing( myiter, mytime, mythid )
1190        call timer_stop ('cost_forcing       [ECCO SPIN-DOWN]', mythid)        call timer_stop ('cost_forcing       [ECCO SPIN-DOWN]', mythid)
1191    cph(
1192    c--   Compute cost function contribution of wind stress observations.  
1193    #ifdef ALLOW_MEAN_HFLUX_COST_CONTRIBUTION
1194          call cost_mean_heatflux( myiter, mytime, mythid )
1195    # ifdef ALLOW_AUTODIFF_TAMC
1196    CADJ STORE objf_hfluxmm = tapelev_init, key = 1
1197    # endif
1198    #endif
1199    
1200    c--   Compute cost function contribution of wind stress observations.
1201    #ifdef ALLOW_MEAN_SFLUX_COST_CONTRIBUTION
1202          call cost_mean_saltflux( myiter, mytime, mythid )
1203    # ifdef ALLOW_AUTODIFF_TAMC
1204    CADJ STORE objf_sfluxmm = tapelev_init, key = 1
1205    # endif
1206    #endif
1207    cph)
1208    
1209    c--   Compute cost function contribution of SSH.
1210    #ifdef ALLOW_SSH_COST_CONTRIBUTION
1211    # ifndef DISABLE_DEBUGMODE
1212          IF ( debugLevel .GE. debLevB )
1213         &    CALL DEBUG_CALL('cost_ssh',myThid)
1214    # endif
1215          call timer_start('cost_ssh           [ECCO SPIN-DOWN]', mythid)
1216          call cost_ssh( myiter, mytime, mythid )
1217          call timer_stop ('cost_ssh           [ECCO SPIN-DOWN]', mythid)
1218    # ifdef ALLOW_AUTODIFF_TAMC
1219    #  ifdef ALLOW_PROFILES
1220    CADJ STORE prof_etan_mean = tapelev_init, key = 1
1221    #  endif
1222    # endif
1223    #endif
1224    
1225  c--   Compute cost function contribution of Temperature and Salinity.  c--   Compute cost function contribution of Temperature and Salinity.
1226  #ifndef DISABLE_DEBUGMODE  #ifndef DISABLE_DEBUGMODE
# Line 797  c--   Compute cost function contribution Line 1231  c--   Compute cost function contribution
1231        call cost_hyd( myiter, mytime, mythid )        call cost_hyd( myiter, mytime, mythid )
1232        call timer_stop ('cost_hyd           [ECCO SPIN-DOWN]', mythid)        call timer_stop ('cost_hyd           [ECCO SPIN-DOWN]', mythid)
1233    
1234    #ifdef ALLOW_SEAICE
1235    #ifndef DISABLE_DEBUGMODE
1236          IF ( debugLevel .GE. debLevB )
1237         &    CALL DEBUG_CALL('seaice_cost_driver',myThid)
1238    #endif
1239          call timer_start('seaice_cost_driver [ECCO SPIN-DOWN]', mythid)
1240          call seaice_cost_driver( myiter, mytime, mythid )
1241          call timer_stop ('seaice_cost_driver [ECCO SPIN-DOWN]', mythid)
1242    #endif
1243    
1244  #ifdef ALLOW_OBCS_COST_CONTRIBUTION  #ifdef ALLOW_OBCS_COST_CONTRIBUTION
1245  #ifndef DISABLE_DEBUGMODE  #ifndef DISABLE_DEBUGMODE
1246        IF ( debugLevel .GE. debLevB )        IF ( debugLevel .GE. debLevB )
# Line 817  c--   Compute cost function contribution Line 1261  c--   Compute cost function contribution
1261        call timer_stop ('cost_curmtr        [ECCO SPIN-DOWN]', mythid)        call timer_stop ('cost_curmtr        [ECCO SPIN-DOWN]', mythid)
1262  #endif  #endif
1263    
 c--   Compute cost function contribution of SSH.  
 #ifdef ALLOW_SSH_COST_CONTRIBUTION  
 #ifndef DISABLE_DEBUGMODE  
       IF ( debugLevel .GE. debLevB )  
      &    CALL DEBUG_CALL('cost_ssh',myThid)  
 #endif  
       call timer_start('cost_ssh           [ECCO SPIN-DOWN]', mythid)  
       call cost_ssh( myiter, mytime, mythid )  
       call timer_stop ('cost_ssh           [ECCO SPIN-DOWN]', mythid)  
 #endif  
   
1264  c--   Compute cost function contribution of drifter's velocities.  c--   Compute cost function contribution of drifter's velocities.
1265  #ifdef ALLOW_DRIFTER_COST_CONTRIBUTION  #ifdef ALLOW_DRIFTER_COST_CONTRIBUTION
1266  #ifndef DISABLE_DEBUGMODE  #ifndef DISABLE_DEBUGMODE
# Line 850  c--   Compute cost function contribution Line 1283  c--   Compute cost function contribution
1283        call timer_stop ('cost_scat            [ECCO SPIN-DOWN]', mythid)        call timer_stop ('cost_scat            [ECCO SPIN-DOWN]', mythid)
1284  #endif  #endif
1285    
 c--   Compute cost function contribution of wind stress observations.    
 #ifdef ALLOW_MEAN_HFLUX_COST_CONTRIBUTION  
       call timer_start('cost_mean_heatflux   [ECCO SPIN-DOWN]', mythid)  
       call cost_mean_heatflux( myiter, mytime, mythid )  
       call timer_stop ('cost_mean_heatflux   [ECCO SPIN-DOWN]', mythid)  
 #endif  
   
 c--   Compute cost function contribution of wind stress observations.  
 #ifdef ALLOW_MEAN_SFLUX_COST_CONTRIBUTION  
       call timer_start('cost_mean_saltflux   [ECCO SPIN-DOWN]', mythid)  
       call cost_mean_saltflux( myiter, mytime, mythid )  
       call timer_stop ('cost_mean_saltflux   [ECCO SPIN-DOWN]', mythid)          
 #endif  
   
1286  c--   Compute cost function contribution of drift between the first  c--   Compute cost function contribution of drift between the first
1287  c     and the last year.  c     and the last year.
1288  #ifdef ALLOW_DRIFT_COST_CONTRIBUTION  #ifdef ALLOW_DRIFT_COST_CONTRIBUTION
# Line 895  c--   Compute initial vs. final T/S devi Line 1314  c--   Compute initial vs. final T/S devi
1314  #endif  #endif
1315        _BARRIER        _BARRIER
1316    
1317    c--   Eddy stress penalty term
1318    #ifdef ALLOW_COST_TAU_EDDY
1319          call timer_start('cost_tau_eddy      [ECCO SPIN-DOWN]', mythid)
1320          call cost_tau_eddy( mythid )
1321          call timer_stop ('cost_tau_eddy      [ECCO SPIN-DOWN]', mythid)
1322    #endif
1323    
1324  c--   Sum all cost function contributions.  c--   Sum all cost function contributions.
1325  #ifndef DISABLE_DEBUGMODE  #ifndef DISABLE_DEBUGMODE
1326        IF ( debugLevel .GE. debLevB )        IF ( debugLevel .GE. debLevB )
1327       &    CALL DEBUG_CALL('cost_final',myThid)       &    CALL DEBUG_CALL('cost_final',myThid)
1328  #endif  #endif
1329        call timer_start('COST_FINAL         [ECCO SPIN-DOWN]', mythid)        call timer_start('COST_FINAL         [ECCO SPIN-DOWN]', mythid)
1330        call ecco_cost_final( mythid )        call cost_final( mythid )
1331        call timer_stop ('COST_FINAL         [ECCO SPIN-DOWN]', mythid)        call timer_stop ('COST_FINAL         [ECCO SPIN-DOWN]', mythid)
1332    
1333  #endif /* ALLOW_COST */  #endif /* ALLOW_COST */
Legend:
Removed from v.1.14  
changed lines
  Added in v.1.48
  ViewVC Help
Powered by ViewVC 1.1.22