Merge branch 'master' into darwin

.readthedocs.yml

@@ -5,7 +5,13 @@
 # Required
 version: 2
 
-# Build documentation in the docs/ directory with Sphinx
+# Set the OS and Python version (required)
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.7"
+
+# Build documentation in the doc/ directory with Sphinx
 sphinx:
   configuration: doc/conf.py
   fail_on_warning: true

doc/phys_pkgs/seaice.rst

@@ -46,7 +46,10 @@ Parts of the :filelink:`seaice <pkg/seaice>` code can be enabled or disabled at
 compile time via CPP preprocessor flags. These options are set in
 :filelink:`SEAICE_OPTIONS.h <pkg/seaice/SEAICE_OPTIONS.h>`.
 :numref:`tab_phys_pkg_seaice_cpp` summarizes the most important ones. For more
-options see :filelink:`SEAICE_OPTIONS.h <pkg/seaice/SEAICE_OPTIONS.h>`.
+options see :filelink:`SEAICE_OPTIONS.h <pkg/seaice/SEAICE_OPTIONS.h>`. Note
+that defining :varlink:`SEAICE_BGRID_DYNAMICS` turns on legacy code and thus
+automatically undefines more recent features, see :filelink:`SEAICE_OPTIONS.h
+<pkg/seaice/SEAICE_OPTIONS.h>` for details.
 
 .. tabularcolumns:: |\Y{.375}|\Y{.1}|\Y{.55}|
 
@@ -56,8 +59,7 @@ options see :filelink:`SEAICE_OPTIONS.h <pkg/seaice/SEAICE_OPTIONS.h>`.
    :name: tab_phys_pkg_seaice_cpp
 
    :varlink:`SEAICE_DEBUG`, #undef, enhance STDOUT for debugging
-   :varlink:`SEAICE_ALLOW_DYNAMICS`, #define, sea ice dynamics code
-   :varlink:`SEAICE_CGRID`, #define, LSR solver on C-grid (rather than original B-grid)
+   :varlink:`SEAICE_CGRID`, #define, use sea ice dynamics on C-grid
    :varlink:`SEAICE_ALLOW_EVP`, #define, enable use of EVP rheology solver
    :varlink:`SEAICE_ALLOW_JFNK`, #define, enable use of JFNK rheology solver
    :varlink:`SEAICE_ALLOW_KRYLOV`, #define, enable use of Krylov rheology solver
@@ -71,12 +73,14 @@ options see :filelink:`SEAICE_OPTIONS.h <pkg/seaice/SEAICE_OPTIONS.h>`.
    :varlink:`SEAICE_ZETA_SMOOTHREG`, #define, use differentiable regularization for viscosities
    :varlink:`SEAICE_DELTA_SMOOTHREG`, #undef, use differentiable regularization for :math:`1/\Delta`
    :varlink:`SEAICE_ALLOW_BOTTOMDRAG`, #undef, enable grounding parameterization for improved fastice in shallow seas
+   :varlink:`SEAICE_BGRID_DYNAMICS`, #undef, use sea ice dynamics code on legacy B-grid; most of the previous flags are not available with B-grid
+   :varlink:`SEAICE_BICE_STRESS`, #undef, B-grid only for backward compatiblity: turn on ice-stress on ocean; defined by default if :varlink:`SEAICE_BGRID_DYNAMICS` is defined
+   :varlink:`EXPLICIT_SSH_SLOPE`, #undef, B-grid only for backward compatiblity: use ETAN for tilt computations rather than geostrophic velocities; defined by default if :varlink:`SEAICE_BGRID_DYNAMICS` is defined
+   :varlink:`SEAICE_LSRBNEW`, #undef, FV discretization for B-grid
    :varlink:`SEAICE_ITD`, #undef, run with dynamical sea Ice Thickness Distribution (ITD)
    :varlink:`SEAICE_VARIABLE_SALINITY`, #undef, enable sea ice with variable salinity
    :varlink:`SEAICE_CAP_ICELOAD`, #undef, enable to limit seaice load (:varlink:`siceLoad`) on the sea surface
    :varlink:`ALLOW_SITRACER`, #undef, enable sea ice tracer package
-   :varlink:`SEAICE_BICE_STRESS`, #undef, B-grid only for backward compatiblity: turn on ice-stress on ocean
-   :varlink:`EXPLICIT_SSH_SLOPE`, #undef, B-grid only for backward compatiblity: use ETAN for tilt computations rather than geostrophic velocities
    :varlink:`SEAICE_USE_GROWTH_ADX`, #undef, use of adjointable but more simplified sea ice thermodynamics model in :filelink:`seaice_growth_adx.F <pkg/seaice/seaice_growth_adx.F>` instead of :filelink:`seaice_growth.F <pkg/seaice/seaice_growth.F>`
 
 .. _ssub_phys_pkg_seaice_runtime:

doc/tag-index

@@ -1,6 +1,37 @@
     Notes on tags used in MITgcmUV
     ==============================
 
+checkpoint68s (2023/09/11)
+o verification:
+  - change default ALLOW_IB_CORR (in pkg/ecco/ECCO_OPTIONS.h) to "#undef" and
+    test that this code compiles in experiment global_oce_biogeo_bling ;
+  - clean/update AD related CPP header files across all experiments and reduce
+    differences with standard version in pkg autodiff, cost, ctrl and ecco.
+o verification:
+  - change global_oce_latlon TAF set-up to multi-tiles (4 tiles, as in code_oad
+    & code_tap) and add SIZE.h_mpi to enable MPI tests (in all 3 code_* dir).
+o model/src & pkgs, TAF storage dir:
+  - rearrange and add storing of variables to reduce number of (unwarned)
+    recomp. in fluxform & vecinv pkgs and in model/src (e.g., calc_phi_hyd.F) ;
+  - add store directives in solve_tri/pentadiagonal.F to avoid recomputation
+    warnings with SOLVE_DIAGONAL_KINNER defined ;
+  - add CPP condition inside GAD SOM advection S/R (-> empty ".f" file if
+    neither GAD_ALLOW_TS_SOM_ADV nor PTRACERS_ALLOW_DYN_STATE is defined ) ;
+  - clean-up/refine many unnecessary store directives -> less TAF warnings ;
+  - remove unnecessary checkpointing code from pkg/smooth_diff2/3d.F
+o pkg/ggl90:
+  - move computing the mixingLength to a dedicated routine ggl90_mixinglength.F
+  - adjust/add TAF store directives to avoid hidden recomputations.
+o pkg/seaice:
+  - new option SEAICE_BGRID_DYNAMICS allows to remove/replace
+    SEAICE_ALLOW_DYNAMICS by ( SEAICE_CGRID or SEAICE_BGRID_DYNAMICS ) ;
+  - exclude more code if SEAICE_CGRID is undefined and clean up/rearrange
+    SEAICE.h to form more consistent groups and common-blocks ;
+  - update SEAICE_OPTIONS.h in verification experiments.
+o pkg/mdsio:
+  - Add 2 new arguments (kLo & kHi) to pkg/mdsio Slice I/O routines that allow
+    to select which levels in Slice-array to read-in or to write-out to file.
+
 checkpoint68r (2023/08/02)
 o verification:
   - adjust amplitude of perturbation (grdchk_eps) to improve Fwd/Adj gradient

model/src/calc_3d_diffusivity.F

@@ -261,6 +261,7 @@ SUBROUTINE CALC_3D_DIFFUSIVITY(
 #ifndef EXCLUDE_PCELL_MIX_CODE
 # ifdef ALLOW_AUTODIFF_TAMC
 CADJ INIT loctape_3d_diff = COMMON, Nr
+CADJ STORE kappartr = loctape_3d_diff
 # endif
       IF ( interDiffKr_pCell ) THEN
 C--   This is a hack: alter vertical diffusivity (instead of changing many S/R)
@@ -310,6 +311,9 @@ SUBROUTINE CALC_3D_DIFFUSIVITY(
              ENDIF
            ENDDO
           ENDDO
+# ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE tmpFac          = loctape_3d_diff, key = k
+# endif
           IF ( mixBott.EQ.2 ) THEN
            DO j = 2-OLy, sNy+OLy
             DO i = 2-OLx, sNx+OLx
@@ -363,6 +367,9 @@ SUBROUTINE CALC_3D_DIFFUSIVITY(
              ENDIF
            ENDDO
           ENDDO
+# ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE tmpFac          = loctape_3d_diff, key = k
+# endif
           IF ( mixSurf.EQ.2 ) THEN
            DO j = 2-OLy, sNy+OLy
             DO i = 2-OLx, sNx+OLx

model/src/calc_phi_hyd.F

@@ -112,6 +112,7 @@ SUBROUTINE CALC_PHI_HYD(
 
 #ifdef ALLOW_AUTODIFF_TAMC
       tkey = bi + (bj-1)*nSx + (ikey_dynamics-1)*nSx*nSy
+      kkey = k + (tkey-1)*Nr
 #endif /* ALLOW_AUTODIFF_TAMC */
 
 C--   Initialize phiHydF to zero :
@@ -137,11 +138,8 @@ SUBROUTINE CALC_PHI_HYD(
         IF ( implicitIntGravWave .OR. myIter.LT.0 ) THEN
 C---    Calculate density
 #ifdef ALLOW_AUTODIFF_TAMC
-          kkey = k + (tkey-1)*Nr
-CADJ STORE theta(:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, byte=isbyte,
-CADJ &     kind = isbyte
-CADJ STORE salt (:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, byte=isbyte,
-CADJ &     kind = isbyte
+CADJ STORE theta(:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, kind = isbyte
+CADJ STORE salt (:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, kind = isbyte
 #endif /* ALLOW_AUTODIFF_TAMC */
           CALL FIND_RHO_2D(
      I              iMin, iMax, jMin, jMax, k,
@@ -186,6 +184,9 @@ SUBROUTINE CALC_PHI_HYD(
      I                              myTime, myIter, myThid )
         ENDIF
 #endif /* ALLOW_MOM_COMMON */
+#ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE alphaRho = comlev1_bibj_k, key = kkey, kind = isbyte
+#endif
 
 #ifdef NONLIN_FRSURF
         IF ( addSurfPhiAnom .AND.
@@ -301,22 +302,15 @@ SUBROUTINE CALC_PHI_HYD(
         IF ( implicitIntGravWave .OR. myIter.LT.0 ) THEN
 C--     Calculate density
 #ifdef ALLOW_AUTODIFF_TAMC
-          kkey = k + (tkey-1)*Nr
-CADJ STORE theta(:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, byte=isbyte,
-CADJ &     kind = isbyte
-CADJ STORE salt (:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, byte=isbyte,
-CADJ &     kind = isbyte
+CADJ STORE theta(:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, kind = isbyte
+CADJ STORE salt (:,:,k,bi,bj) = comlev1_bibj_k, key=kkey, kind = isbyte
 #endif /* ALLOW_AUTODIFF_TAMC */
           CALL FIND_RHO_2D(
      I              iMin, iMax, jMin, jMax, k,
      I              theta(1-OLx,1-OLy,k,bi,bj),
      I              salt(1-OLx,1-OLy,k,bi,bj),
      O              alphaRho,
      I              k, bi, bj, myThid )
-#ifdef ALLOW_AUTODIFF_TAMC
-CADJ STORE alphaRho (:,:) = comlev1_bibj_k, key=kkey, byte=isbyte,
-CADJ &     kind = isbyte
-#endif /* ALLOW_AUTODIFF_TAMC */
         ELSE
           DO j=jMin,jMax
            DO i=iMin,iMax
@@ -325,6 +319,9 @@ SUBROUTINE CALC_PHI_HYD(
           ENDDO
         ENDIF
 
+#ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE alphaRho = comlev1_bibj_k, key = kkey, kind = isbyte
+#endif
 C--     Calculate specific volume anomaly : alpha_prime = 1/rho - alpha_Cst
         DO j=jMin,jMax
           DO i=iMin,iMax
@@ -342,6 +339,9 @@ SUBROUTINE CALC_PHI_HYD(
      I                              myTime, myIter, myThid )
         ENDIF
 #endif /* ALLOW_MOM_COMMON */
+#ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE alphaRho = comlev1_bibj_k, key = kkey, kind = isbyte
+#endif
 
 C----  Hydrostatic pressure at cell centers
 
@@ -450,6 +450,9 @@ SUBROUTINE CALC_PHI_HYD(
      I                              myTime, myIter, myThid )
         ENDIF
 #endif /* ALLOW_MOM_COMMON */
+#ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE alphaRho = comlev1_bibj_k, key = kkey, kind = isbyte
+#endif
 
 C---    Integrate d Phi / d pi
 
@@ -605,6 +608,14 @@ SUBROUTINE CALC_PHI_HYD(
         STOP 'CALC_PHI_HYD: Bad value of buoyancyRelation !'
       ENDIF
 
+#ifdef ALLOW_AUTODIFF_TAMC
+# ifdef NONLIN_FRSURF
+CADJ STORE alphaRho = comlev1_bibj_k, key = kkey, kind = isbyte
+CADJ STORE phiHydC  = comlev1_bibj_k, key = kkey, kind = isbyte
+CADJ STORE phiHydF  = comlev1_bibj_k, key = kkey, kind = isbyte
+# endif
+#endif
+
       IF ( .NOT. useFVgradPhi ) THEN
 C--   r-coordinate and r*-coordinate cases:
 

model/src/calc_viscosity.F

@@ -1,5 +1,8 @@
 #include "PACKAGES_CONFIG.h"
 #include "CPP_OPTIONS.h"
+#ifdef ALLOW_AUTODIFF
+# include "AUTODIFF_OPTIONS.h"
+#endif
 
 CBOP
 C     !ROUTINE: CALC_VISCOSITY
@@ -24,6 +27,11 @@ SUBROUTINE CALC_VISCOSITY(
 #include "PARAMS.h"
 #include "DYNVARS.h"
 #include "GRID.h"
+#ifndef EXCLUDE_PCELL_MIX_CODE
+# ifdef ALLOW_AUTODIFF
+#  include "tamc.h"
+# endif
+#endif
 
 C     !INPUT/OUTPUT PARAMETERS:
 C     == Routine arguments ==
@@ -47,6 +55,11 @@ SUBROUTINE CALC_VISCOSITY(
       INTEGER km, mixSurf, mixBott
       _RL pC_kFac
       _RL tmpFac(1-OLx:sNx+OLx,1-OLy:sNy+OLy)
+# ifdef ALLOW_AUTODIFF_TAMC
+C     tkey :: tape key (tile dependent)
+C     kkey :: tape key (level and tile dependent)
+      INTEGER tkey, kkey
+# endif
 #endif
 CEOP
 
@@ -120,6 +133,11 @@ SUBROUTINE CALC_VISCOSITY(
       ENDDO
 
 #ifndef EXCLUDE_PCELL_MIX_CODE
+# ifdef ALLOW_AUTODIFF_TAMC
+      tkey = bi + (bj-1)*nSx + (ikey_dynamics-1)*nSx*nSy
+CADJ STORE kappaRU = comlev1_bibj, key = tkey, kind = isbyte
+CADJ STORE kappaRV = comlev1_bibj, key = tkey, kind = isbyte
+# endif
       IF ( interViscAr_pCell ) THEN
 C--   This is a hack: alter vertical viscosity (instead of changing many S/R)
 C     in order to account for missing hFac in viscous term
@@ -155,6 +173,9 @@ SUBROUTINE CALC_VISCOSITY(
        mixSurf = pCellMix_select/10
        mixBott = MOD(pCellMix_select,10)
        DO k = 2,Nr
+# ifdef ALLOW_AUTODIFF_TAMC
+         kkey = k + (tkey-1)*Nr
+# endif
          km = k - 1
          pC_kFac = 1.
          IF ( pCellMix_delR.LT.drF(k) )
@@ -195,6 +216,14 @@ SUBROUTINE CALC_VISCOSITY(
           DO j = 2-OLy, sNy+OLy
            DO i = 2-OLx, sNx+OLx
              tmpFac(i,j) = MIN( tmpFac(i,j), pCellMix_maxFac )
+# ifdef ALLOW_AUTODIFF_TAMC
+           ENDDO
+          ENDDO
+CADJ STORE tmpFac         = comlev1_bibj_k, key = kkey, kind = isbyte
+CADJ STORE kappaRU(:,:,k) = comlev1_bibj_k, key = kkey, kind = isbyte
+          DO j = 2-OLy, sNy+OLy
+           DO i = 2-OLx, sNx+OLx
+# endif
              kappaRU(i,j,k) = MAX( kappaRU(i,j,k),
      &                             pCellMix_viscAr(k)*tmpFac(i,j) )
            ENDDO
@@ -236,6 +265,14 @@ SUBROUTINE CALC_VISCOSITY(
           DO j = 2-OLy, sNy+OLy
            DO i = 2-OLx, sNx+OLx
              tmpFac(i,j) = MIN( tmpFac(i,j), pCellMix_maxFac )
+# ifdef ALLOW_AUTODIFF_TAMC
+           ENDDO
+          ENDDO
+CADJ STORE tmpFac         = comlev1_bibj_k, key = kkey, kind = isbyte
+CADJ STORE kappaRV(:,:,k) = comlev1_bibj_k, key = kkey, kind = isbyte
+          DO j = 2-OLy, sNy+OLy
+           DO i = 2-OLx, sNx+OLx
+# endif
              kappaRV(i,j,k) = MAX( kappaRV(i,j,k),
      &                             pCellMix_viscAr(k)*tmpFac(i,j) )
            ENDDO
@@ -279,9 +316,20 @@ SUBROUTINE CALC_VISCOSITY(
            ENDDO
           ENDIF
 C-    increase mixing below surface (by ~(1/hFac)^mixSurf) if too thin p-cell
+# ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE tmpFac = comlev1_bibj_k, key = kkey, kind = isbyte
+# endif
           DO j = 2-OLy, sNy+OLy
            DO i = 2-OLx, sNx+OLx
              tmpFac(i,j) = MIN( tmpFac(i,j), pCellMix_maxFac )
+# ifdef ALLOW_AUTODIFF_TAMC
+           ENDDO
+          ENDDO
+CADJ STORE tmpFac         = comlev1_bibj_k, key = kkey, kind = isbyte
+CADJ STORE kappaRU(:,:,k) = comlev1_bibj_k, key = kkey, kind = isbyte
+          DO j = 2-OLy, sNy+OLy
+           DO i = 2-OLx, sNx+OLx
+# endif
              kappaRU(i,j,k) = MAX( kappaRU(i,j,k),
      &                             pCellMix_viscAr(k)*tmpFac(i,j) )
            ENDDO
@@ -321,9 +369,20 @@ SUBROUTINE CALC_VISCOSITY(
            ENDDO
           ENDIF
 C-    increase mixing below surface (by ~(1/hFac)^mixSurf) if too thin p-cell
+# ifdef ALLOW_AUTODIFF_TAMC
+CADJ STORE tmpFac = comlev1_bibj_k, key = kkey, kind = isbyte
+# endif
           DO j = 2-OLy, sNy+OLy
            DO i = 2-OLx, sNx+OLx
              tmpFac(i,j) = MIN( tmpFac(i,j), pCellMix_maxFac )
+# ifdef ALLOW_AUTODIFF_TAMC
+           ENDDO
+          ENDDO
+CADJ STORE tmpFac         = comlev1_bibj_k, key = kkey, kind = isbyte
+CADJ STORE kappaRV(:,:,k) = comlev1_bibj_k, key = kkey, kind = isbyte
+          DO j = 2-OLy, sNy+OLy
+           DO i = 2-OLx, sNx+OLx
+# endif
              kappaRV(i,j,k) = MAX( kappaRV(i,j,k),
      &                             pCellMix_viscAr(k)*tmpFac(i,j) )
            ENDDO

model/src/do_atmospheric_phys.F

@@ -1,8 +1,8 @@
 #include "PACKAGES_CONFIG.h"
 #include "CPP_OPTIONS.h"
-#ifdef ALLOW_AUTODIFF
-# include "AUTODIFF_OPTIONS.h"
-#endif
+c#ifdef ALLOW_AUTODIFF
+c# include "AUTODIFF_OPTIONS.h"
+c#endif
 
 CBOP
 C     !ROUTINE: DO_ATMOSPHERIC_PHYS
@@ -40,9 +40,9 @@ SUBROUTINE DO_ATMOSPHERIC_PHYS(myTime, myIter, myThid)
 #include "PARAMS.h"
 #include "GRID.h"
 #include "DYNVARS.h"
-#ifdef ALLOW_AUTODIFF_TAMC
-# include "tamc.h"
-#endif /* ALLOW_AUTODIFF_TAMC */
+c#ifdef ALLOW_AUTODIFF_TAMC
+c# include "tamc.h"
+c#endif /* ALLOW_AUTODIFF_TAMC */
 
 C     !INPUT/OUTPUT PARAMETERS:
 C     == Routine arguments ==
@@ -67,10 +67,12 @@ SUBROUTINE DO_ATMOSPHERIC_PHYS(myTime, myIter, myThid)
 #endif
 
       IF ( fluidIsAir ) THEN
-#ifdef ALLOW_AUTODIFF_TAMC
-CADJ STORE theta = comlev1, key=ikey_dynamics, kind=isbyte
-CADJ STORE salt  = comlev1, key=ikey_dynamics, kind=isbyte
-#endif
+C     These store directives do not seem to be necessary, but we leave
+C     them here as a reminder.
+c#ifdef ALLOW_AUTODIFF_TAMC
+cCADJ STORE theta = comlev1, key=ikey_dynamics, kind=isbyte
+cCADJ STORE salt  = comlev1, key=ikey_dynamics, kind=isbyte
+c#endif
        DO bj=myByLo(myThid),myByHi(myThid)
         DO bi=myBxLo(myThid),myBxHi(myThid)