Merge branch 'main' into refactoring_blas_bug_fix

.github/workflows/fesom2.1.yml

@@ -12,7 +12,7 @@ jobs:
     # Containers must run in Linux based operating systems
     runs-on: ubuntu-latest
     # Docker Hub image that `container-job` executes in
-    container: koldunovn/fesom2_test:refactoring2
+    container: ghcr.io/fesom/fesom2_docker:fesom2_test_refactoring-master
 
     # Service containers to run with `gfortran_ubuntu`
     steps:

.github/workflows/fesom2_openmp.yml

@@ -12,7 +12,7 @@ jobs:
     # Containers must run in Linux based operating systems
     runs-on: ubuntu-latest
     # Docker Hub image that `container-job` executes in
-    container: koldunovn/fesom2_test:refactoring2
+    container: ghcr.io/fesom/fesom2_docker:fesom2_test_refactoring-nightly
 
     # Service containers to run with `gfortran_ubuntu`
     steps:

.gitignore

@@ -1,5 +1,7 @@
 /build
 /build.lib
+/work
+/lib64
 *.o
 *.mod
 *.x

CMakeLists.txt

@@ -15,6 +15,8 @@ set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs F
 set(CRAY OFF CACHE BOOL "compile with cray ftn")
 set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea ice column physics.")
 set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results")
+set(RECOM_COUPLED OFF CACHE BOOL "compile fesom including biogeochemistry, REcoM3")
+set(CISO_COUPLED OFF CACHE BOOL "compile ciso coupled to REcoM3. RECOM_COUPLED has to be active")
 set(USE_MULTIO OFF CACHE BOOL "Use MULTIO for IO, either grib or binary for now. This also means path to MULTIO installation has to provided using env MULTIO_INSTALL_PATH='..' and multio configuration yamls must be present to run the model with MULTIO")
 set(OASIS_WITH_YAC OFF CACHE BOOL "Useing a version of OASIS compiled with YAC instead of SCRIP for interpolation?")
 set(ASYNC_ICEBERGS ON CACHE BOOL "compile fesom with or without support for asynchronous iceberg computations")

README.md

@@ -29,12 +29,3 @@ References
 * **[Version with ICEPACK sea ice thermodynamics]** Zampieri, Lorenzo, Frank Kauker, Jörg Fröhle, Hiroshi Sumata, Elizabeth C. Hunke, and Helge Goessling. Impact of Sea-Ice Model Complexity on the Performance of an Unstructured-Mesh Sea-ice/ocean Model Under Different Atmospheric Forcings. Washington: American Geophysical Union, 2020. https://dx.doi.org/10.1002/essoar.10505308.1.
 
 * **[Version coupled with OpenIFS atmosphere]** Streffing, J., Sidorenko, D., Semmler, T., Zampieri, L., Scholz, P., Andrés-Martínez, M., et al ( 2022). AWI-CM3 coupled climate model: description and evaluation experiments for a prototype post-CMIP6 model. Geoscientific Model Development, 15. https://doi.org/10.5194/gmd-15-6399-2022
-
-Documentation for FESOM2
-************************
-
-Here lives FESOM2 documentation
-
-Rendered version: https://fesom2.readthedocs.io/en/latest/
-
-

config/bin_2p1z1d/job_albedo

@@ -0,0 +1,66 @@
+#!/bin/bash
+#SBATCH --account=nwg_hauck.maresys # edit your account
+#SBATCH --job-name=recom
+#SBATCH --partition=mpp
+#SBATCH --time=01:30:00
+####SBATCH --constraint="[rack1|rack3]"
+#SBATCH --qos=12h
+#SBATCH --nodes=9            # Number of tasks (MPI) tasks to be launched
+#SBATCH --tasks-per-node 96 #8 #4  # using all 128CPus we exceed the maxload limit (>130) of many used nodes when us parallel I/O
+#SBATCH --cpus-per-task 1
+
+#SBATCH -o fesom2o.out
+#SBATCH -e fesom2e.out
+
+# disable hyperthreading
+#SBATCH --hint=nomultithread
+
+module purge 
+source ../env/albedo/shell
+export OMP_NUM_THREADS=1
+ulimit -s unlimited
+
+# determine JOBID
+JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1`
+
+ln -s ../bin/fesom.x .           # cp -n ../bin/fesom.x
+cp -n ../config/namelist.config  .
+cp -n ../config/namelist.forcing .
+cp -n ../config/namelist.oce     .
+cp -n ../config/namelist.dyn     .
+cp -n ../config/namelist.tra     .
+cp -n ../config/namelist.ice     .
+cp -n ../config/namelist.io      .
+cp -n ../config/namelist.icepack .
+
+#___DETERMINE SLURM JOBID+OUTPUTFILE____________________________________________
+jobid=$(echo $SLURM_JOB_ID | cut -d"." -f1)
+fname="fesom2_${SLURM_JOB_NAME}_${jobid}.out"
+
+#___PUT JOB IN QUEUE____________________________________________________________
+date
+####srun --mpi=pmi2 ./fesom.x >> ${fname}
+srun --mpi=pmi2 ./fesom.x > fesom2.0.out
+date
+
+#qstat -f $PBS_JOBID
+#export EXITSTATUS=$?
+#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then
+#sbatch job_ollie
+#fi
+
+#Resultpath='/albedo/scratch/user/ogurses/ocean_only/'
+Resultpath='/albedo/scratch/user/ogurses/fesom2.5_recom_st1_vsink_vben_diags/'
+test -d $Resultpath/fesom.2019.oce.restart && exit
+
+IsInFile=$( tail -3 fesom2.0.out | grep -c timesteps)
+if (( IsInFile > 0 )); then
+# submit next #job                                                                                                     \
+ echo "submitting next job"
+ cp fesom2.0.out fesom.out.done
+ sbatch job_albedo
+else
+ echo "something is wrong, last line of fesom.out reads"
+ echo $( tail -1 fesom2.0.out)
+ echo "abnormal termination of job script"
+fi

config/bin_2p1z1d/namelist.config

@@ -0,0 +1,67 @@
+! This is the namelist file for model general configuration
+
+&modelname
+runid='fesom'
+/
+
+&timestep
+step_per_day=32 !96 !96 !72 !72 !45 !72 !96
+run_length=1 !62 !62 !62 !28
+run_length_unit='y'		! y, m, d, s
+/
+
+&clockinit              ! the model starts at
+timenew=0.0
+daynew=1
+yearnew=1958
+/
+
+&paths
+MeshPath='/albedo/work/projects/p_recompdaf/frbunsen/FESOM2/meshes/core2/'
+ClimateDataPath='/albedo/work/projects/MarESys/ogurses/input/corrected_input/' 
+ResultPath='/albedo/scratch/user/ogurses/fesom2.5_recom_st1_vsink_vben_diags/'
+/
+
+&restart_log
+restart_length=1            ! --> do netcdf restart ( only required for d,h,s cases, y, m take 1)
+restart_length_unit='y'     !output period: y, d, h, s, off
+raw_restart_length=1        ! --> do core dump restart
+raw_restart_length_unit='off' ! e.g. y, d, h, s, off
+bin_restart_length=1        ! --> do derived type binary restart 
+bin_restart_length_unit='off' ! e.g. y, d, h, s, off
+logfile_outfreq=960     !in logfile info. output frequency, # steps
+/
+
+&ale_def
+which_ALE='zstar'       ! 'linfs','zlevel', 'zstar'
+use_partial_cell=.true.
+/
+
+&geometry
+cartesian=.false.
+fplane=.false.
+cyclic_length=360       ![degree]
+rotated_grid=.true.     !option only valid for coupled model case now
+force_rotation=.true.
+alphaEuler=50.          ![degree] Euler angles, convention:
+betaEuler=15.           ![degree] first around z, then around new x,
+gammaEuler=-90.         ![degree] then around new z.
+/
+
+&calendar
+include_fleapyear=.false.
+/
+
+&run_config
+use_ice=.true.                  ! ocean+ice
+use_cavity=.false.              !
+use_cavity_partial_cell=.false. 
+use_floatice = .false.
+use_sw_pene=.true.
+flag_debug=.false.
+/
+
+&machine
+n_levels=2
+n_part= 12, 36          ! 432 number of partitions on each hierarchy level
+/

config/bin_2p1z1d/namelist.dyn

@@ -0,0 +1,24 @@
+&dynamics_visc
+visc_gamma0  = 0.003 ! [m/s],   backgroung viscosity= gamma0*len, it should be as small a s possible (keep it < 0.01 m/s).
+visc_gamma1  = 0.1   ! [nodim], for computation of the flow aware viscosity
+visc_gamma2  = 0.285 ! [s/m],   is only used in easy backscatter option
+visc_easybsreturn= 1.5
+
+opt_visc     = 5          
+! 5=Kinematic (easy) Backscatter
+! 6=Biharmonic flow aware (viscosity depends on velocity Laplacian)
+! 7=Biharmonic flow aware (viscosity depends on velocity differences)
+! 8=Dynamic Backscatter
+
+use_ivertvisc= .true.  
+/
+
+&dynamics_general
+momadv_opt   = 2       ! option for momentum advection in moment only =2
+use_freeslip = .false. ! Switch on free slip
+use_wsplit   = .false. ! Switch for implicite/explicte splitting of vert. velocity
+wsplit_maxcfl= 1.0     ! maximum allowed CFL criteria in vertical (0.5 < w_max_cfl < 1.) 
+                       ! in older FESOM it used to be w_exp_max=1.e-3
+ldiag_KE=.false.       ! activates energy diagnostics
+/
+

config/bin_2p1z1d/namelist.forcing

@@ -0,0 +1,60 @@
+! This is the namelist file for forcing
+
+&forcing_exchange_coeff
+Ce_atm_oce=0.00175 ! exchange coeff. of latent heat over open water
+Ch_atm_oce=0.00175 ! exchange coeff. of sensible heat over open water
+Cd_atm_oce=0.001  ! drag coefficient between atmosphere and water
+Ce_atm_ice=0.00175 ! exchange coeff. of latent heat over ice
+Ch_atm_ice=0.00175 ! exchange coeff. of sensible heat over ice
+Cd_atm_ice=0.0012  ! drag coefficient between atmosphere and ice 
+Swind     =0.0     ! parameterization for coupled current feedback
+/
+
+&forcing_bulk
+AOMIP_drag_coeff=.false.
+ncar_bulk_formulae=.true.
+ncar_bulk_z_wind=10.0 ! height at which wind forcing is located (CORE, JRA-do: 10m, JRA, NCEP:2m)
+ncar_bulk_z_tair=10.0 ! height at which temp forcing is located (CORE, JRA-do: 10m, JRA, NCEP:2m)
+ncar_bulk_z_shum=10.0 ! height at which humi forcing is located (CORE, JRA-do: 10m, JRA, NCEP:2m)
+
+/
+
+&land_ice
+use_landice_water=.false.
+landice_start_mon=5
+landice_end_mon=10
+/
+
+&nam_sbc
+   nm_xwind_file = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/uas.clim61'        ! name of file with winds, if nm_sbc=2
+   nm_ywind_file = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/vas.clim61'        ! name of file with winds, if nm_sbc=2
+   nm_humi_file  = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/huss.clim61'        ! name of file with humidity
+   nm_qsr_file   = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/rsds.clim61'    ! name of file with solar heat
+   nm_qlw_file   = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/rlds.clim61'    ! name of file with Long wave
+   nm_tair_file  = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/tas.clim61'        ! name of file with 2m air temperature
+   nm_prec_file  = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/prra.clim61' ! name of file with total precipitation
+   nm_snow_file  = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/prsn.clim61' ! name of file with snow  precipitation
+   nm_mslp_file  = '/albedo/work/projects/MarESys/FROM-OLLIE/forcing_JRA55-do-v1.4.0_clim61/psl.clim61'         ! air_pressure_at_sea_level
+   nm_xwind_var  = 'uas'   ! name of variable in file with wind
+   nm_ywind_var  = 'vas'   ! name of variable in file with wind
+   nm_humi_var   = 'huss'   ! name of variable in file with humidity
+   nm_qsr_var    = 'rsds'   ! name of variable in file with solar heat
+   nm_qlw_var    = 'rlds'   ! name of variable in file with Long wave
+   nm_tair_var   = 'tas'   ! name of variable in file with 2m air temperature
+   nm_prec_var   = 'prra'       ! name of variable in file with total precipitation
+   nm_snow_var   = 'prsn'       ! name of variable in file with total precipitation
+   nm_mslp_var   = 'psl'        ! name of variable in file with air_pressure_at_sea_level
+   nm_nc_iyear   = 1900
+   nm_nc_imm     = 1            ! initial month of time axis in netCDF 
+   nm_nc_idd     = 1            ! initial day of time axis in netCDF
+   nm_nc_freq    = 1            ! data points per day (i.e. 86400 if the time axis is in seconds)
+   nm_nc_tmid    = 0            ! 1 if the time stamps are given at the mid points of the netcdf file, 0 otherwise (i.e. 1 in CORE1, CORE2; 0 in JRA55)
+   y_perpetual=.true.
+   l_xwind=.true. l_ywind=.true. l_humi=.true. l_qsr=.true. l_qlw=.true. l_tair=.true. l_prec=.true. l_mslp=.true. l_cloud=.false. l_snow=.true.
+   nm_runoff_file      ='/albedo/pool/FESOM/forcing/CORE2/runoff.nc'
+   runoff_data_source ='CORE2'	!Dai09, CORE2
+   !runoff_data_source ='Dai09'  !Dai09, CORE2, JRA55
+   !runoff_climatology =.true.
+   nm_sss_data_file    ='/albedo/pool/FESOM/forcing/CORE2/PHC2_salx.nc'
+   sss_data_source    ='CORE2'
+/

config/bin_2p1z1d/namelist.ice

@@ -0,0 +1,31 @@
+! Ice namelist
+&ice_dyn
+whichEVP=0             ! 0=standart; 1=mEVP; 2=aEVP
+Pstar=30000.0          ! [N/m^2]
+ellipse=2.0
+c_pressure=20.0        ! ice concentration parameter used in ice strength computation
+delta_min=1.0e-11      ! [s^(-1)]
+evp_rheol_steps=120    ! number of EVP subcycles
+alpha_evp=250          ! constant that control numerical stability of mEVP. Adjust with resolution. 
+beta_evp=250           ! constant that control numerical stability of mEVP. Adjust with resolution.
+c_aevp=0.15            ! a tuning constant in aEVP. Adjust with resolution.
+Cd_oce_ice=0.0055      ! drag coef. oce - ice 
+ice_gamma_fct=0.5      ! smoothing parameter
+ice_diff=0.0           ! diffusion to stabilize
+theta_io=0.0           ! rotation angle
+ice_ave_steps=1        ! ice step=ice_ave_steps*oce_step
+/
+
+&ice_therm
+Sice=4.0               ! Ice salinity 3.2--5.0 ppt.
+h0=.5                  ! Lead closing parameter [m] 
+emiss_ice=0.97         ! Emissivity of Snow/Ice,
+emiss_wat=0.97         ! Emissivity of open water
+albsn=0.81             ! Albedo: frozen snow
+albsnm=0.77            !         melting snow
+albi=0.7               !         frozen ice
+albim=0.68             !         melting ice
+albw=0.1               !         open water
+con=2.1656             ! Thermal conductivities: ice; W/m/K
+consn=0.31             !                         snow
+/