From 6057ac55227a33d720e0f298f091f7deaf4256ec Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 31 Jul 2023 13:53:39 +0200 Subject: [PATCH 01/71] no more ollie --- work/job_ini_ollie | 31 ----- work/job_ollie | 34 ----- work/job_ollie_chain | 317 ------------------------------------------- 3 files changed, 382 deletions(-) delete mode 100755 work/job_ini_ollie delete mode 100755 work/job_ollie delete mode 100755 work/job_ollie_chain diff --git a/work/job_ini_ollie b/work/job_ini_ollie deleted file mode 100755 index 693296bad..000000000 --- a/work/job_ini_ollie +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0_INI -#SBATCH -p mpp -#SBATCH --ntasks=32 -#SBATCH --time=12:00:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out - -set -x - -ulimit -s unlimited - -module purge -module load intel.compiler -module load intel.mpi -module load netcdf/4.4.0_intel - -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -date -srun --mpi=pmi2 --ntasks=1 ./fesom_ini.x > "fvom_ini.out" -date - diff --git a/work/job_ollie b/work/job_ollie deleted file mode 100755 index ac2516115..000000000 --- a/work/job_ollie +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH -p mpp -#SBATCH --ntasks=288 -#SBATCH --time=00:05:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -module load intel.compiler intel.mpi netcdf/4.4.0_intel -module load centoslibs - -set -x - -ulimit -s unlimited - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.io . -cp -n ../config/namelist.icepack . - -date -srun --mpi=pmi2 ./fesom.x > "fesom2.0.out" -date - -#qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_ollie -#fi diff --git a/work/job_ollie_chain b/work/job_ollie_chain deleted file mode 100755 index 8f98f1515..000000000 --- a/work/job_ollie_chain +++ /dev/null @@ -1,317 +0,0 @@ -#!/bin/bash -#___SET SLURM OPTIONS___________________________________________________________ -#SBATCH -J chain -#SBATCH -p mpp -#SBATCH --ntasks=432 -#SBATCH --time=10:00:00 -#SBATCH --mail-type=END -#SBATCH --mail-user=Patrick.Scholz@awi.de -#SBATCH -o fesom2.0_%x_%j.out -#SBATCH -e fesom2.0_%x_%j.out - -## module load intel.compiler intel.mpi netcdf centoslibs - -#___DEFAULT INPUT_______________________________________________________________ -# how many job chains should be applied -chain_n=3 # number chain cycles -chain_s=1 # starting chain id - -# time frame of model simulation -# ___COREv2___ -year_s=1948 -year_e=2009 -# ___JRA55____ -#year_s=1958 -#year_e=2018 - -prescribe_rlen=0 # run length in namelist.config --> if 0 value from namelist.config is taken -fedit=1 - -#___HELP OUTPUT_________________________________________________________________ -script_name=job_ollie_chain -function usage { - echo "usage: $script_name [-cn ...] [-cs ...] [-ys ...] [-ye ...] [-wcl ...]" - echo " -cn number of chain cylces (default: 3)" - echo " -cs starting chain id (default: 1)" - echo " -ys starting year of model simulation (default: 1948)" - echo " -ye ending year of model simulation (default: 2009)" - echo " -h display help" - echo " -rl prescribe used run_length" - echo " -noedit no automatic editing of namelist.config" - echo - echo " --> for changing the wall-clock-time interactively use " - echo " sbatch --time=00:10:00 job_ollie_chain ..." - echo " --> for changing the number of task interactively use " - echo " sbatch --ntask=288 job_ollie_chain ..." - echo " --> for changing the job name interactively use " - echo " sbatch --job-name=whatever job_ollie_chain ..." - exit 1 -} - -#___OVERRIDE DEFAULT INPUT BY COMMANDLINE INPUT_________________________________ -while [ "$1" != "" ]; do - case $1 in - -cn | -chain_n ) shift ; chain_n=$1 ;; - -cs | -chain_s ) shift ; chain_s=$1 ;; - -ys | -year_s ) shift ; year_s=$1 ;; - -ye | -year_e ) shift ; year_e=$1 ;; - -rl | --run_length ) shift ; prescribe_rlen=$1 ;; - -noedit | --noedit ) fedit=0 ;; - -h | --help ) usage ; exit ;; - esac - shift -done - -#___EXTRACT WALL-CLOCK-TIME FROM JOBINFO________________________________________ -# either setted via #SBATCH time=... or at command line sbatch --time=... job_ollie_chain -# need here to extract to give the next job chain cycle as input -jobinfo=$(scontrol show job $SLURM_JOB_ID) -wcl=$( echo ${jobinfo##*"TimeLimit="} | cut -d " " -f 1 ) -ntasks=$( echo ${jobinfo##*"NumTasks="} | cut -d " " -f 1 ) - - -#___SET NAMELIST'S & EXECUTABLE IF NOT ALREADY EXIST____________________________ -set -x -ulimit -s unlimited -ln -s ../bin/fesom.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -#___SET CHAIN_ID________________________________________________________________ -if [ -f "file_chain_id" ]; then - chain_id=$( file_chain_id -fi - -#___PRINT INPUT INFO____________________________________________________________ -echo -e "\033[1;7;33m_____JOB CHAIN INFO_____________________________________\033[0m" -echo -e "\033[1;33m --> actual chain cycle: $chain_id \033[0m" -echo -e "\033[1;33m --> max. number of chain cycles: $chain_n \033[0m" -echo -e "\033[1;33m --> simulated time range: [ $year_s $year_e] \033[0m" -echo -e "\033[1;33m --> slurm: wall-clock-time = $wcl \033[0m" -echo -e "\033[1;33m --> slurm: ntask = $ntasks \033[0m" -if [ $prescribe_rlen -ne 0 ]; then - echo -e "\033[1;33m -->change run_length = $prescribe_rlen \033[0m" -fi - -#___CREATE SAVE DIR INFRASTRUCTURE______________________________________________ -# extract resultpath from namelist.config -dname_result_link_orig=$(grep "ResultPath=" namelist.config | grep -v '^!' | \ - cut -d "=" -f 2 | \ - cut -d "'" -f 2) -dname_result_link=$(echo ${dname_result_link_orig::-1}) - -# identify higher directory --> ResultPath -dname_result="$(dirname "$dname_result_link")/" - -# if ResultPath directory doesn't exist --> create it -if [ ! -d "${dname_result}" ]; then - echo -e "\033[33m --> ResultPath directory does not exist --> will create it \033[0m" - mkdir "${dname_result}" -fi - -# check if in namelist.config a chain path is given (that mean -# ResultPath=.../ResultDir/chain/) if not change namelist.config accordingly -check=${dname_result_link##*${dname_result}} -if [ $fedit -eq 1 ] && [ $check != "chain" ]; then - echo -e "\033[33m --> replace in namelist.config ResultPath with chain path \033[0m" - dname_result_link="${dname_result}chain" - sed -i "s|${dname_result_link_orig}|${dname_result_link}/|g" namelist.config -fi - -# identify real path in case a link is used -dname_result="$(realpath "$dname_result")/" - -# if directory for chain_id doesn't exist --> create it -if [ ! -d "${dname_result}/${chain_id}" ]; then - echo -e "\033[33m --> chain_id directory does not exist --> will create it \033[0m" - mkdir "${dname_result}/${chain_id}" -fi - -# link directory of chain_id with original linkdirectory from namelist.config -ln -sfn ${dname_result}${chain_id} $dname_result_link - -#___CHECK IF SIMULATION NEEDS TO BE INITIALISED OR CONTINUED____________________ -is_newsimul=1 -if [ -f "$dname_result_link/fesom.clock" ] ; then - aux_yr_clock=$(<${dname_result_link}/fesom.clock) - aux_yr_clock=$(echo ${aux_yr_clock} | cut -d" " -f 6) - if [ $aux_yr_clock -le $year_e ]; then is_newsimul=0 ; fi - - #___________________________________________________________________________ - if [ $fedit -eq 1 ] ; then - if [ $is_newsimul -eq 0 ] ; then - year_d=$(( $year_e - $aux_yr_clock + 1 )) - rlen=$(grep "run_length=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1) - # do not prescribe run length in job fle - if [ $prescribe_rlen -eq 0 ] ; then - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $year_d \033[0m" - fi - # prescribe run length in job fle - else - aux_year_d=$prescribe_rlen - if [ $year_d -lt $aux_year_d ] ; then aux_year_d=$year_d ; fi - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$aux_year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $aux_year_d \033[0m" - fi - fi - fi - fi -else - #___________________________________________________________________________ - # set model run length in job_script and change namelist.config accordingly - # to match - if [ $fedit -eq 1 ] ; then - year_d=$(( $year_e - $year_s + 1 )) - rlen=$(grep "run_length=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1) - # do not prescribe run length in job fle - if [ $prescribe_rlen -eq 0 ] ; then - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $year_d \033[0m" - fi - # prescribe run length in job file - else - aux_year_d=$prescribe_rlen - if [ $year_d -lt $aux_year_d ] ; then aux_year_d=$year_d ; fi - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$aux_year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $aux_year_d \033[0m" - fi - fi - fi -fi - -#___CREATE CLOCK & RESTART INFRASTRUCTURE FOR COLD/WARM START___________________ -# only touch clock file when a new simulation is supposed to start, if an old one -# should be continued dont touch it -if [ $is_newsimul -eq 1 ] ; then - - # --> make cold start - if [ $chain_id -eq 1 ] ; then - #_______________________________________________________________________ - # create cold start clock file - echo -e "\033[33m --> create cold start clock file \033[0m" - printf "0 1 ${year_s}\n0 1 ${year_s}" > $dname_result_link/fesom.clock - - #_______________________________________________________________________ - # in case yearnew in namelist.config was changed from 1948 - yearnew=$(grep "yearnew=" namelist.config | cut -d "=" -f 2) - if [ $yearnew -ne $year_s ]; then - sed -i " s/yearnew=$yearnew/yearnew=$year_s/" namelist.config - fi - - #___BACKUP NAMELIST.* FILES INTO RESULT DIRECTORY_______________________ - cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.cvmix namelist.tra namelist.dyn ${dname_result}/. - cp fesom.x ${dname_result}/. - - #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________ - if [ ! -d "${dname_result}/src" ]; then mkdir "${dname_result}/src" ; fi - cp ../src/*.F90 ${dname_result}/src/. - - - # --> make warm start - else - #_______________________________________________________________________ - # create warm start clock file - stepperday=$(grep "step_per_day=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1 ) - aux_sec=$(( 86400 - 86400 / $stepperday )) - aux_day=365 - aux_yr=$(( $year_s - 1 )) - echo -e "\033[33m --> create warm start clock file \033[0m" - printf "${aux_sec} ${aux_day} ${aux_yr}\n0 1 ${year_s}" > $dname_result_link/fesom.clock - - #_______________________________________________________________________ - # chain id from previous spinup cycle - prev_chain_id=$(( $chain_id - 1 )) - - #_______________________________________________________________________ - # copy restart ocean files/directories from previous spinup cycle - prev_rfile=${dname_result}${prev_chain_id}/fesom.${year_e}.oce.restart - if [ -d "${prev_rfile}" ]; then - cp -r ${prev_rfile} ${dname_result_link}/fesom.${aux_yr}.oce.restart - elif [ -f "${prev_rfile}.nc" ]; then - cp ${prev_rfile}.nc ${dname_result_link}/fesom.${aux_yr}.oce.restart.nc - else - echo -e "\033[1;31m --> ERROR: could not find ocean restart file \033[0m" - exit - fi - - # copy restart ice files/files/directories from previous spinup cycle - prev_rfile=${dname_result}${prev_chain_id}/fesom.${year_e}.ice.restart - if [ -d "${prev_rfile}" ]; then - cp -r ${prev_rfile} ${dname_result_link}/fesom.${aux_yr}.ice.restart - elif [ -f "${prev_rfile}.nc" ]; then - cp ${prev_rfile}.nc ${dname_result_link}/fesom.${aux_yr}.ice.restart.nc - else - echo -e "\033[1;31m --> ERROR: could not find ice restart file \033[0m" - exit - fi - - #_______________________________________________________________________ - # adapt year new in namelist.config otherwise fesom is not doing a - # restart - aux=$(grep "yearnew=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1 ) - sed -i " s/yearnew=$aux/yearnew=$aux_yr/" namelist.config - - #_______________________________________________________________________ - # backup namelist.* & fesom.x in case they dont exist - if [ ! -f "${dname_result}/namelist.config" ]; then - cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.cvmix namelist.tra namelist.dyn ${dname_result}/. - fi - if [ ! -f "${dname_result}/fesom.x" ]; then - cp fesom.x ${dname_result}/. - fi - #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________ - if [ ! -d "${dname_result}/src" ]; then mkdir "${dname_result}/src" ; fi - cp ../src/*.F90 ${dname_result}/src/. - fi -fi - -#___DETERMINE SLURM JOBID+OUTPUTFILE____________________________________________ -jobid=$(echo $SLURM_JOB_ID | cut -d"." -f1) -fname="fesom2.0_${SLURM_JOB_NAME}_${jobid}.out" - -#___PUT JOB IN QUEUE____________________________________________________________ -date -srun --mpi=pmi2 ./fesom.x >> ${fname} -err_srun=$? -echo -e "\033[33m --> err_srun=${err_srun} \033[0m" -date - -#___SHOW JOB INFORMATION________________________________________________________ -scontrol show job $SLURM_JOB_ID - -#___SETUP JOBCHAIN______________________________________________________________ -# check if complete cycle is finished only than increase chain_id -aux_yr_clock=$(<${dname_result_link}/fesom.clock) -aux_yr_clock=$(echo ${aux_yr_clock} | cut -d" " -f 6) - -# setup next chain job via dependence -if [ ${err_srun} -eq 0 ]; then - if [ $aux_yr_clock -lt $year_e ] || [ ${chain_id} -lt ${chain_n} ]; then - # aftercorr:job_id --> A task of this job array can begin execution after - # the corresponding task ID in the specified job has completed successfully - # (ran to completion with an exit code of zero). - echo -e "\033[33m --> setup next chain cycle \033[0m" - sbatch --time=$wcl --ntasks=$ntasks --job-name=${SLURM_JOB_NAME} --dependency=aftercorr:$SLURM_JOB_ID $script_name \ - -cn $chain_n -cs $chain_s -ys $year_s -ye $year_e -rl $prescribe_rlen - fi -fi - -#___CHECK FOR COMPLETNES________________________________________________________ -# check if complete cycle is finished only than increase chain_id -if [ $aux_yr_clock -gt $year_e ] && [ ${chain_id} -lt ${chain_n} ] ; then - chain_id=$(( $chain_id + 1 )) - echo $chain_id > file_chain_id -fi - From 5d3a6ea4359084bc33653bbd3d99821fb5bef0af Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 31 Jul 2023 15:38:15 +0200 Subject: [PATCH 02/71] Update getting_started.rst --- docs/getting_started/getting_started.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index af6b00035..715dc5f8c 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -8,7 +8,7 @@ This chapter describes several ways of getting started with FESOM2. First we sho TL;DR version for supported HPC systems ======================================= -Supported systems are: generic ``ubuntu``, ``ollie`` at AWI, ``mistral`` at DKRZ, ``JURECA`` at JSC, ``HLRN``, ``Hazel Hen``, ``Marinostrum 4`` at BSC. During configuration the system will be recognised and apropriate environment variables and compiler options should be used. +Supported systems are: generic ``ubuntu``, ``ollie`` at AWI, ``mistral`` at DKRZ, ``JURECA`` at JSC, ``HLRN``, ``Hazel Hen``, ``MareNostrum 4`` at BSC. During configuration the system will be recognised and apropriate environment variables and compiler options should be used. :: git clone https://github.com/FESOM/fesom2.git @@ -47,7 +47,7 @@ Clone the GitHub repository with a git command: git clone https://github.com/FESOM/fesom2.git -The repository contains model code and two additional libraries: `Metis` (domain partitioner) and `Parms` (solver), necessary to run FESOM2. To build FESOM2 executable one have to compile Parms library and the code of the model (`src` folder). In order to build executable that is used for model domain partitioning (distribution of the model mesh between CPUs) one have to compile `Metis` library and also some code located in the src directory (see :ref:`partitioning`). Building of the model executable and the partitioner is usually done automatically with the use of CMake. If you going to build the code not on one of the supported platforms (ollie, DKRZ, HLRN, and HAZELHEN, general Ubuntu), you might need to do some (usually small) modifications described in `Adding new platform for compilation`_ section. +The repository contains model code and two additional libraries: `Metis` (domain partitioner) and `Parms` (solver), necessary to run FESOM2. To build FESOM2 executable one have to compile Parms library and the code of the model (`src` folder). In order to build executable that is used for model domain partitioning (distribution of the model mesh between CPUs) one have to compile `Metis` library and also some code located in the src directory (see :ref:`partitioning`). Building of the model executable and the partitioner is usually done automatically with the use of CMake. If you going to build the code not on one of the supported platforms (ollie, DKRZ, HLRN, HAZELHEN, and BSC, general Ubuntu), you might need to do some (usually small) modifications described in `Adding new platform for compilation`_ section. Change to the `fesom2` folder and execute: @@ -57,7 +57,7 @@ Change to the `fesom2` folder and execute: In the best case scenario, your platform will be recognized and the Parms library and model executable will be built and copied to the bin directory. If something went wrong have a look at Troubleshooting_ section. -If you would like to select platform manually (which is nessesary in the case of Ubuntu, for eample), type: +If you would like to select platform manually (which is necessary in the case of Ubuntu, for example), type: :: @@ -67,7 +67,7 @@ If you would like to select platform manually (which is nessesary in the case of Data and mesh files ------------------- -The FESOM2 repository contains only very small example meshes and data (in the ``test`` directory, see the note below). However, if you want to run realistic simulations, you ether have to have them on your system, or download an archive with sample data. THere is a chance that your system already have some of the necesseary files, you can check it in the ``setups/paths.yml`` file. If not, the easiest way to start is to download example set from `DKRZ cloud`_ (12 Gb) by executing: +The FESOM2 repository contains only very small example meshes and data (in the ``test`` directory, see the note below). However, if you want to run realistic simulations, you ether have to have them on your system, or download an archive with sample data. There is a chance that your system already has some of the necesseary files, you can check it in the ``setups/paths.yml`` file. If not, the easiest way to start is to download example set from `DKRZ cloud`_ (12 Gb) by executing: :: @@ -100,7 +100,7 @@ You have to do several basic things in order to prepare the run. First, create a mkdir results -you might make a link to some other directory located on the part of the system where you have a lot of storage. In the results directory, you have to create ``fesom.clock`` file (NOTE, if you change ``runid`` in ``namelist.config`` to something like ``runid=mygreatrun``, the file will be named ``mygreatrun.clock``). Inside the file you have to put two identical lines: +You might make a link to some other directory located on the part of the system where you have a lot of storage. In the results directory, you have to create ``fesom.clock`` file (NOTE, if you change ``runid`` in ``namelist.config`` to something like ``runid=mygreatrun``, the file will be named ``mygreatrun.clock``). Inside the file you have to put two identical lines: :: From 2dd4ca12555784eab79a82d64cfa27683e5db821 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 10 Aug 2023 14:42:11 +0200 Subject: [PATCH 03/71] incorporating Razvan's implementation of MULTIO. Required Cmakefiles still need to be adjusted. --- src/MOD_PARTIT.F90 | 5 +- src/ifs_interface/ifs_interface.F90 | 38 +++ src/ifs_interface/ifs_notused.F90 | 40 --- src/ifs_interface/iom.F90 | 411 ++++++++++++++++++++++++++++ src/ifs_interface/mpp_io.F90 | 181 ++++++++++++ src/io_meandata.F90 | 85 +++++- src/oce_mesh.F90 | 36 +-- 7 files changed, 726 insertions(+), 70 deletions(-) create mode 100644 src/ifs_interface/iom.F90 create mode 100644 src/ifs_interface/mpp_io.F90 diff --git a/src/MOD_PARTIT.F90 b/src/MOD_PARTIT.F90 index 5c8a598af..fb4a88542 100644 --- a/src/MOD_PARTIT.F90 +++ b/src/MOD_PARTIT.F90 @@ -47,10 +47,10 @@ module MOD_PARTIT integer :: eDim_nod2D integer, allocatable, dimension(:) :: myList_nod2D - integer :: myDim_elem2D + integer :: myDim_elem2D, myDim_elem2D_shrinked integer :: eDim_elem2D integer :: eXDim_elem2D - integer, allocatable, dimension(:) :: myList_elem2D + integer, allocatable, dimension(:) :: myList_elem2D, myInd_elem2D_shrinked integer :: myDim_edge2D integer :: eDim_edge2D @@ -75,6 +75,7 @@ module MOD_PARTIT integer, allocatable :: s_mpitype_nod3D(:,:,:), r_mpitype_nod3D(:,:,:) integer :: MPIERR + !!! remPtr_* are constructed during the runtime and shall not be dumped!!! integer, allocatable :: remPtr_nod2D(:), remList_nod2D(:) integer, allocatable :: remPtr_elem2D(:), remList_elem2D(:) diff --git a/src/ifs_interface/ifs_interface.F90 b/src/ifs_interface/ifs_interface.F90 index 03aba0b48..f3d738342 100644 --- a/src/ifs_interface/ifs_interface.F90 +++ b/src/ifs_interface/ifs_interface.F90 @@ -9,6 +9,44 @@ MODULE nemogcmcoup_steps INTEGER :: substeps !per IFS timestep END MODULE nemogcmcoup_steps +SUBROUTINE nemogcmcoup_init_ioserver( icomm, lnemoioserver, irequired, iprovided, lmpi1) + + ! Initialize the NEMO mppio server + USE mpp_io + + IMPLICIT NONE + INTEGER :: icomm + LOGICAL :: lnemoioserver + INTEGER :: irequired, iprovided + LOGICAL :: lmpi1 + + CALL mpp_io_init(icomm, lnemoioserver, irequired, iprovided, lmpi1) +END SUBROUTINE nemogcmcoup_init_ioserver + + +SUBROUTINE nemogcmcoup_init_ioserver_2( icomm ) + ! Initialize the NEMO mppio server + USE mpp_io + + IMPLICIT NONE + INTEGER :: icomm + + CALL mpp_io_init_2( icomm ) + IF (lioserver) THEN + ! IO server finished, clean-up multio objects + CALL mpp_stop() + ENDIF +END SUBROUTINE nemogcmcoup_init_ioserver_2 + +SUBROUTINE nemogcmcoup_end_ioserver + ! Function is only called for the IO client. + USE mpp_io + + IMPLICIT NONE + + CALL mpp_stop() + END SUBROUTINE nemogcmcoup_end_ioserver + SUBROUTINE nemogcmcoup_init( mype, icomm, inidate, initime, itini, itend, zstp, & & lwaveonly, iatmunit, lwrite ) diff --git a/src/ifs_interface/ifs_notused.F90 b/src/ifs_interface/ifs_notused.F90 index bc711a8c6..0beda3670 100644 --- a/src/ifs_interface/ifs_notused.F90 +++ b/src/ifs_interface/ifs_notused.F90 @@ -3,33 +3,6 @@ ! ! -Original code by Kristian Mogensen, ECMWF. -SUBROUTINE nemogcmcoup_init_ioserver( icomm, lnemoioserver ) - - ! Initialize the NEMO mppio server - - IMPLICIT NONE - INTEGER :: icomm - LOGICAL :: lnemoioserver - - WRITE(*,*)'No mpp_ioserver' - CALL abort - -END SUBROUTINE nemogcmcoup_init_ioserver - - -SUBROUTINE nemogcmcoup_init_ioserver_2( icomm ) - - ! Initialize the NEMO mppio server - - IMPLICIT NONE - INTEGER :: icomm - - WRITE(*,*)'No mpp_ioserver' - CALL abort - -END SUBROUTINE nemogcmcoup_init_ioserver_2 - - SUBROUTINE nemogcmcoup_mlflds_get( mype, npes, icomm, & & nlev, nopoints, pgt3d, pgs3d, pgu3d, pgv3d ) @@ -332,16 +305,3 @@ SUBROUTINE nemogcmcoup_wam_update_stress( mype, npes, icomm, npoints, & END SUBROUTINE nemogcmcoup_wam_update_stress -SUBROUTINE nemogcmcoup_end_ioserver - - ! Close io servers - - IMPLICIT NONE - INTEGER :: icomm - LOGICAL :: lnemoioserver - - WRITE(*,*)'No mpp_ioserver' - CALL abort - -END SUBROUTINE nemogcmcoup_end_ioserver - diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 new file mode 100644 index 000000000..57783098e --- /dev/null +++ b/src/ifs_interface/iom.F90 @@ -0,0 +1,411 @@ +!===================================================== +! Input/Output manager : Library to write output files +! +! -Original code for NEMOv40 by ECMWF. +! -Adapted to FESOM2 by Razvan Aguridan, ECMWF, 2023. +!----------------------------------------------------- + +MODULE iom + USE multio_api + USE, INTRINSIC :: iso_fortran_env, only: real64 + + IMPLICIT NONE + PRIVATE + + TYPE(multio_handle) :: mio_handle + INTEGER(8), PRIVATE :: mio_parent_comm + + PUBLIC iom_initialize, iom_init_server, iom_finalize + PUBLIC iom_send_fesom_domains + PUBLIC iom_field_request, iom_send_fesom_data + + PRIVATE ctl_stop + !!---------------------------------------------------------------------- + !! NEMO/OCE 4.0 , NEMO Consortium (2018) + !! $Id: iom.F90 13297 2020-07-13 08:01:58Z andmirek $ + !! Software governed by the CeCILL license (see ./LICENSE) + !!---------------------------------------------------------------------- + + TYPE iom_field_request + CHARACTER(100) :: name = REPEAT(" ", 100) + CHARACTER(5) :: gridType = REPEAT(" ", 5) + REAL(real64), DIMENSION(:), POINTER :: values => NULL() + INTEGER :: globalSize = 0 + INTEGER :: level = 0 + INTEGER :: step = 0 + END TYPE + +CONTAINS + + SUBROUTINE multio_custom_error_handler(context, err) + USE mpi + + IMPLICIT NONE + INTEGER(8), INTENT(INOUT) :: context ! Use mpi communicator as context + INTEGER, INTENT(IN) :: err + INTEGER :: mpierr + + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop( 'MULTIO ERROR: ', multio_error_string(err)) + IF (context /= MPI_UNDEFINED) THEN + CALL mpi_abort(int(context), MPI_ERR_OTHER, mpierr) + context = MPI_UNDEFINED + ENDIF + ENDIF + END SUBROUTINE + + + SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) + USE mpi + + IMPLICIT NONE + CHARACTER(LEN=*), INTENT(IN) :: client_id + INTEGER,INTENT(IN), OPTIONAL :: local_comm + INTEGER,INTENT(OUT), OPTIONAL :: return_comm + INTEGER,INTENT(IN), OPTIONAL :: global_comm + TYPE(multio_configuration) :: conf_ctx + INTEGER :: err + CHARACTER(len=16) :: err_str + + mio_parent_comm = mpi_comm_world + + err = multio_initialise() + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('Initializing multio failed: ', multio_error_string(err)) + END IF + + IF (PRESENT(global_comm)) THEN + mio_parent_comm = global_comm + ENDIF + + ! Prepare context and check errors explicitly until everything is set up - then failure handler is used + BLOCK + CHARACTER(:), allocatable :: config_file + INTEGER :: config_file_length + + CALL get_environment_variable('MULTIO_FESOM_CONFIG_FILE', length=config_file_length) + IF (config_file_length == 0) THEN + call ctl_stop('The fesom plan file is not correctly set!') + err = conf_ctx%new() + ELSE + ALLOCATE(character(len=config_file_length + 1) :: config_file) + + CALL get_environment_variable('MULTIO_FESOM_CONFIG_FILE', config_file) + err = conf_ctx%new(config_file) + + DEALLOCATE(config_file) + ENDIF + END BLOCK + + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('Creating multio configuration context failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%mpi_allow_world_default_comm(.FALSE._1) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%mpi_client_id(client_id) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('conf_ctx%mpi_client_id(', TRIM(client_id),') failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%mpi_return_client_comm(return_comm) + IF (err /= MULTIO_SUCCESS) THEN + WRITE (err_str, "(I)") return_comm + CALL ctl_stop('conf_ctx%mpi_return_client_comm(', err_str,') failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%mpi_parent_comm(int(mio_parent_comm)) + IF (err /= MULTIO_SUCCESS) THEN + WRITE (err_str, "(I)") mio_parent_comm + CALL ctl_stop('conf_ctx%mpi_parent_comm(', err_str,') failed: ', multio_error_string(err)) + END IF + + err = mio_handle%new(conf_ctx) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('mio_handle%new(conf_ctx) failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%delete() + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('conf_ctx%delete() failed: ', multio_error_string(err)) + END IF + + ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface + err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) + END IF + + err = mio_handle%open_connections(); + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('mio_handle%open_connections failed: ', multio_error_string(err)) + END IF + END SUBROUTINE iom_initialize + + SUBROUTINE iom_finalize() + IMPLICIT NONE + INTEGER :: err + + err = mio_handle%close_connections(); + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('mio_handle%close_connections failed: ', multio_error_string(err)) + END IF + + err = mio_handle%delete(); + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('mio_handle%delete failed: ', multio_error_string(err)) + END IF + END SUBROUTINE iom_finalize + + SUBROUTINE iom_init_server(server_comm) + IMPLICIT NONE + INTEGER, INTENT(IN) :: server_comm + type(multio_configuration) :: conf_ctx + INTEGER :: err + CHARACTER(len=16) :: err_str + + mio_parent_comm = server_comm + + err = multio_initialise() + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('Initializing multio failed: ', multio_error_string(err)) + END IF + + ! Prepare context and check errors explicitly until everything is set up - then failure handler is used + + BLOCK + CHARACTER(:), allocatable :: config_file + INTEGER :: config_file_length + + CALL get_environment_variable('MULTIO_FESOM_CONFIG_FILE', length=config_file_length) + IF (config_file_length == 0) THEN + err = conf_ctx%new() + ELSE + ALLOCATE(character(len=config_file_length + 1) :: config_file) + + CALL get_environment_variable('MULTIO_FESOM_CONFIG_FILE', config_file) + err = conf_ctx%new(config_file) + + DEALLOCATE(config_file) + ENDIF + END BLOCK + + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('Creating multio server configuration context failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%mpi_allow_world_default_comm(.FALSE._1) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%mpi_parent_comm(int(mio_parent_comm)) + IF (err /= MULTIO_SUCCESS) THEN + WRITE (err_str, "(I)") mio_parent_comm + CALL ctl_stop('conf_ctx%mpi_parent_comm(', err_str,') failed: ', multio_error_string(err)) + END IF + + ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface + ! Set handler before invoking blocking start server call + err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) + END IF + + ! Blocking call + err = multio_start_server(conf_ctx) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('multio_start_server(conf_ctx) failed: ', multio_error_string(err)) + END IF + + err = conf_ctx%delete() + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('conf_ctx%delete() failed: ', multio_error_string(err)) + END IF + END SUBROUTINE iom_init_server + + SUBROUTINE iom_send_fesom_domains(partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + + IMPLICIT NONE + + TYPE(multio_metadata) :: md + INTEGER :: cerr + INTEGER :: elem, elnodes(3), aux + TYPE(t_partit), INTENT(IN), TARGET :: partit + TYPE(t_mesh), intent(in), TARGET :: mesh + INTEGER, DIMENSION(:), POINTER :: temp + +#include "../associate_part_def.h" +#include "../associate_mesh_def.h" +#include "../associate_part_ass.h" +#include "../associate_mesh_ass.h" + + cerr = md%new() + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%new() failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("name", "ngrid") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%set_string(name) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("category", "fesom-domain-nodemap") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%set_string(category) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("representation", "unstructured") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%set_string(representation) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("globalSize", mesh%nod2D) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%set_int(globalSize) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_bool("toAllServers", .TRUE._1) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%set_bool(toAllServers) failed: ', multio_error_string(cerr)) + END IF + + temp => partit%myList_nod2D(1:partit%myDim_nod2D) + cerr = mio_handle%write_domain(md, temp - 1) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, mio_handle%write_domain() failed: ', multio_error_string(cerr)) + END IF + + cerr = md%delete() + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: ngrid, md%delete() failed: ', multio_error_string(cerr)) + END IF + + !declare grid at elements + cerr = md%new() + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%new() failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("name", "egrid") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%set_string(name) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("category", "fesom-domain-elemmap") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%set_string(category) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("representation", "unstructured") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%set_string(representation) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("globalSize", mesh%elem2D) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%set_int(globalSize) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_bool("toAllServers", .TRUE._1) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%set_bool(toAllServers) failed: ', multio_error_string(cerr)) + END IF + + cerr = mio_handle%write_domain(md, partit%myList_elem2D(partit%myInd_elem2D_shrinked) - 1) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, mio_handle%write_domain() failed: ', multio_error_string(cerr)) + END IF + + cerr = md%delete() + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_domains: egrid, md%delete() failed: ', multio_error_string(cerr)) + END IF + END SUBROUTINE iom_send_fesom_domains + + SUBROUTINE iom_send_fesom_data(data) + IMPLICIT NONE + + TYPE(iom_field_request), INTENT(INOUT) :: data + INTEGER :: cerr + TYPE(multio_metadata) :: md + + cerr = md%new() + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%new() failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("category", "fesom-grid-output") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_string(category) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("globalSize", data%globalSize) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_int(globalSize) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("level", data%level) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_int(level) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_bool("toAllServers", .FALSE._1) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_bool(toAllServers) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("name", trim(data%name)) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_string(name) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("gridSubType", data%gridType) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_string(gridSubType) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("domain", data%gridType) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_string(domain) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("step", data%step) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_int(step) failed: ', multio_error_string(cerr)) + END IF + + cerr = mio_handle%write_field(md, data%values) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: mio_handle%write_field failed: ', multio_error_string(cerr)) + END IF + + cerr = md%delete() + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%delete failed: ', multio_error_string(cerr)) + END IF + END SUBROUTINE + + SUBROUTINE ctl_stop(m1, m2, m3, m4) + USE mpi + + IMPLICIT NONE + CHARACTER(len=*), INTENT(in), OPTIONAL :: m1, m2, m3, m4 + INTEGER :: dummy + + IF ( PRESENT(m1) ) WRITE(*,*) m1 + IF ( PRESENT(m2) ) WRITE(*,*) m2 + IF ( PRESENT(m3) ) WRITE(*,*) m3 + IF ( PRESENT(m4) ) WRITE(*,*) m4 + + CALL mpi_abort(mpi_comm_world, 1, dummy) + END SUBROUTINE ctl_stop + + !!====================================================================== + END MODULE iom + \ No newline at end of file diff --git a/src/ifs_interface/mpp_io.F90 b/src/ifs_interface/mpp_io.F90 new file mode 100644 index 000000000..95a939049 --- /dev/null +++ b/src/ifs_interface/mpp_io.F90 @@ -0,0 +1,181 @@ +!===================================================== +! Ocean output intialisation. +! +! -Original code for NEMOv40 by Kristian Mogensen, ECMWF. +! -Adapted to FESOM2 by Razvan Aguridan, ECMWF, 2023. +!----------------------------------------------------- + +MODULE mpp_io + USE iom + IMPLICIT NONE + PRIVATE + + PUBLIC & + & mpp_io_init, & + & mpp_io_init_2, & + & mpp_stop + + INTEGER :: ntask_multio = 0 + INTEGER :: ntask_xios = 0 + LOGICAL, PUBLIC :: lioserver, lmultioserver, lmultiproc + INTEGER :: ntask_notio + INTEGER, SAVE :: mppallrank, mppallsize, mppiorank, mppiosize + INTEGER, SAVE :: mppmultiorank, mppmultiosize + INTEGER, SAVE :: mppcomprank, mppcompsize + INTEGER, SAVE :: pcommworld, pcommworldmultio + + CONTAINS + + SUBROUTINE mpp_io_init( iicomm, lio, irequired, iprovided, lmpi1 ) + + INCLUDE "mpif.h" + INTEGER, INTENT(INOUT) :: iicomm + LOGICAL, INTENT(INOUT) :: lio + INTEGER, INTENT(INOUT) :: irequired, iprovided + LOGICAL, INTENT(IN) :: lmpi1 + + INTEGER :: icode, ierr, icolor + LOGICAL :: mpi_called + CHARACTER(len=128) :: cdlogfile + INTEGER :: ji + NAMELIST/namio/ntask_multio,ntask_xios + + CALL mpi_initialized( mpi_called, icode ) + IF ( icode /= MPI_SUCCESS ) THEN + WRITE(*,*)' mpp_io_init: Error in routine mpi_initialized' + CALL mpi_abort( mpi_comm_world, icode, ierr ) + ENDIF + + IF( mpi_called ) THEN + WRITE(*,*)' mpi_io_init assumes that it is initialising MPI' + CALL mpi_abort( mpi_comm_world, 1, ierr ) + ENDIF + + IF (lmpi1) THEN + CALL mpi_init( icode ) + ELSE +#ifdef MPI1 + WRITE(0,*)'mpp_io_init:' + WRITE(0,*)'MPI1 defined but lmpi1 is false' + CALL abort +#else + CALL mpi_init_thread(irequired,iprovided,icode) +#endif + ENDIF + + IF ( icode /= MPI_SUCCESS ) THEN + WRITE(*,*)' mpp_io_init: Error in routine mpi_init' + CALL mpi_abort( mpi_comm_world, icode, ierr ) + ENDIF + + CALL mpi_comm_rank( mpi_comm_world, mppallrank, ierr ) + CALL mpi_comm_size( mpi_comm_world, mppallsize, ierr ) + + OPEN(10,file='namio.in') + READ(10,namio) + WRITE(*,namio) + CLOSE(10) + + IF ( ntask_xios + ntask_multio == 0 ) THEN + iicomm = mpi_comm_world + lio=.FALSE. + RETURN + ENDIF + + ntask_notio = mppallsize - ntask_xios - ntask_multio + IF ((mppallrank+1)<=ntask_notio) THEN + icolor=1 + lioserver=.FALSE. + lmultioserver=.FALSE. + ELSE + icolor=3 + lioserver=.TRUE. + lmultioserver=.TRUE. + ENDIF + lio=lioserver + + CALL mpi_comm_split( mpi_comm_world, icolor, 0, iicomm, icode ) + IF ( icode /= MPI_SUCCESS ) THEN + WRITE(*,*)' mpp_io_init: Error in routine mpi_comm_split' + CALL mpi_abort( mpi_comm_world, icode, ierr ) + ENDIF + IF (lioserver) THEN + CALL mpi_comm_rank( iicomm, mppiorank, ierr ) + CALL mpi_comm_size( iicomm, mppiosize, ierr ) + WRITE(cdlogfile,'(A,I4.4,A)')'nemo_io_server.',mppiorank,'.log' + ELSE + mppiorank=0 + mppiosize=0 + ENDIF + lio=lioserver + + END SUBROUTINE mpp_io_init + + SUBROUTINE mpp_io_init_2( iicomm ) + + INTEGER, INTENT(INOUT) :: iicomm + + INTEGER :: icode, ierr, icolor, iicommx, iicommm, iicommo + INTEGER :: ji,inum + LOGICAL :: lcompp + INCLUDE "mpif.h" + + ! Construct multio server communicator + + IF (lmultioserver.OR..NOT.lioserver) THEN + icolor=12 + ELSE + icolor=13 + ENDIF + + CALL mpi_comm_split( iicomm, icolor, 0, pcommworldmultio, icode ) + IF ( icode /= MPI_SUCCESS ) THEN + WRITE(*,*)' mpp_io_init2: Error in routine mpi_comm_split' + CALL mpi_abort( mpi_comm_world, icode, ierr ) + ENDIF + + CALL mpi_comm_rank( pcommworldmultio, mppmultiorank, ierr ) + CALL mpi_comm_size( pcommworldmultio, mppmultiosize, ierr ) + + ! Construct compute communicator + + IF (.NOT.lioserver) THEN + icolor=14 + lcompp=.TRUE. + ELSE + icolor=15 + lcompp=.FALSE. + ENDIF + + CALL mpi_comm_split( iicomm, icolor, 0, iicommo, icode ) + IF ( icode /= MPI_SUCCESS ) THEN + WRITE(*,*)' mpp_io_init2: Error in routine mpi_comm_split' + CALL mpi_abort( mpi_comm_world, icode, ierr ) + ENDIF + + CALL mpi_comm_rank( iicommo, mppcomprank, ierr ) + CALL mpi_comm_size( iicommo, mppcompsize, ierr ) + + IF (.NOT.lioserver) THEN + CALL iom_initialize( "for_xios_mpi_id", return_comm=iicommm, global_comm = pcommworldmultio ) ! nemo local communicator given by xios + ELSE + ! For io-server tasks start an run the right server + CALL iom_init_server( server_comm = pcommworldmultio ) + ENDIF + + ! Return to the model with iicomm being compute only tasks + iicomm = iicommo + + END SUBROUTINE mpp_io_init_2 + + SUBROUTINE mpp_stop + INTEGER :: ierr + + IF (.NOT.lioserver) THEN + call iom_finalize() + ENDIF + + CALL mpi_finalize( ierr ) + END SUBROUTINE mpp_stop + +END MODULE mpp_io \ No newline at end of file diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 1edb8067c..6ff0dce80 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -20,6 +20,8 @@ module io_MEANDATA type(t_partit), pointer :: p_partit integer :: ndim integer :: glsize(2) + integer :: shrinked_size + integer, allocatable, dimension(:) :: shrinked_indx integer :: accuracy real(real64), allocatable, dimension(:,:) :: local_values_r8 real(real32), allocatable, dimension(:,:) :: local_values_r4 @@ -1127,6 +1129,9 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) use MOD_ICE use mod_tracer use io_gather_module +#if defined(__MULTIO) + use iom +#endif #if defined (__icepack) use icedrv_main, only: init_io_icepack #endif @@ -1142,11 +1147,10 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) type(t_tracer), intent(in) , target :: tracers type(t_dyn) , intent(in) , target :: dynamics type(t_ice) , intent(inout), target :: ice - - character(:), allocatable :: filepath - real(real64) :: rtime !timestamp of the record + character(:), allocatable :: filepath + real(real64) :: rtime !timestamp of the record - ctime=timeold+(dayold-1.)*86400 +ctime=timeold+(dayold-1.)*86400 !___________________________________________________________________________ if (lfirst) then @@ -1164,14 +1168,18 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ !PS if (partit%flag_debug .and. partit%mype==0) print *, achar(27)//'[33m'//' -I/O-> call update_means'//achar(27)//'[0m' call update_means - !___________________________________________________________________________ ! loop over defined streams do n=1, io_NSTREAMS !_______________________________________________________________________ ! make pointer for entry onto io_stream object entry=>io_stream(n) - +!#if defined(__MULTIO) +! call mio_write_nod(mio, entry) +! lfirst=.false. +! return +!#endif + !_______________________________________________________________________ !check whether output will be written based on event frequency do_output=.false. @@ -1196,6 +1204,7 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) ! if its time for output --> do_output==.true. if (do_output) then if (vec_autorotate) call io_r2g(n, partit, mesh) ! automatically detect if a vector field and rotate if makes sense! +#if !defined(__MULTIO) if(entry%thread_running) call entry%thread%join() entry%thread_running = .false. @@ -1248,7 +1257,7 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) entry%rec_count=max(entry%rec_count, 1) write(*,*) trim(entry%name)//': current mean I/O counter = ', entry%rec_count end if ! --> if(partit%mype == entry%root_rank) then - +#endif !___________________________________________________________________ ! write double precision output if (entry%accuracy == i_real8) then @@ -1273,17 +1282,22 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) END DO ! --> DO J=1, size(entry%local_values_r4,dim=2) !$OMP END PARALLEL DO end if ! --> if (entry%accuracy == i_real8) then - !___________________________________________________________________ entry%addcounter = 0 ! clean_meanarrays entry%ctime_copy = ctime - + +#if defined(__MULTIO) +! if (n==1) then + entry%rec_count = istep + call send_data_to_multio(entry) +! end if +#else !___________________________________________________________________ ! this is where the magic happens --> here do_output_callback is ! triggered as a method of the io_stream object --> call write_mean(...) call entry%thread%run() entry%thread_running = .true. - +#endif endif ! --> if (do_output) then end do ! --> do n=1, io_NSTREAMS lfirst=.false. @@ -1561,8 +1575,16 @@ subroutine def_stream_after_dimension_specific(entry, name, description, units, !___________________________________________________________________________ if(entry%glsize(1)==mesh%nod2D .or. entry%glsize(2)==mesh%nod2D) then entry%is_elem_based = .false. + entry%shrinked_size=partit%myDim_nod2D else if(entry%glsize(1)==mesh%elem2D .or. entry%glsize(2)==mesh%elem2D) then entry%is_elem_based = .true. + entry%shrinked_size=partit%myDim_elem2D_shrinked + allocate(entry%shrinked_indx(entry%shrinked_size)) + entry%shrinked_indx=partit%myInd_elem2D_shrinked +! write(*,*) partit%mype, partit%myDim_elem2D, partit%myDim_elem2D_shrinked, partit%myDim_elem2D-partit%myDim_elem2D_shrinked +! entry_index=0 +! call MPI_AllREDUCE(partit%myDim_elem2D_shrinked, entry_index, 1, MPI_INTEGER, MPI_SUM, partit%MPI_COMM_FESOM, err) +! write(*,*) 'total elem=', mesh%elem2D, entry_index else if(partit%mype == 0) print *,"can not determine if ",trim(name)," is node or elem based" stop @@ -1699,4 +1721,47 @@ subroutine io_r2g(n, partit, mesh) !$OMP END PARALLEL DO END IF end subroutine + +#if defined(__MULTIO) +SUBROUTINE send_data_to_multio(entry) + USE iom + USE multio_api + + IMPLICIT NONE + + TYPE(Meandata), TARGET, INTENT(INOUT) :: entry + TYPE(iom_field_request) :: request + REAL(real64), DIMENSION(SIZE(entry%shrinked_indx)), TARGET :: temp + INTEGER :: numLevels, globalSize, lev, i + + numLevels = entry%glsize(1) + globalSize = entry%glsize(2) + + request%name = trim(entry%name) + IF (.NOT. entry%is_elem_based) THEN + request%gridType = "ngrid" + ELSE + request%gridType = "egrid" + END IF + request%globalSize = globalSize + request%step = entry%rec_count + + ! loop over vertical layers --> do gather 3d variables layerwise in 2d slices + DO lev=1, numLevels + request%level = lev + + IF (.NOT. entry%is_elem_based) THEN + request%values => entry%local_values_r8_copy(lev, 1:entry%shrinked_size) + ELSE + DO i = 1, SIZE(entry%shrinked_indx) + temp(i) = entry%local_values_r8_copy(lev, entry%shrinked_indx(i)) + END DO + + request%values => temp + END IF + + CALL iom_send_fesom_data(request) + END DO +END SUBROUTINE +#endif end module diff --git a/src/oce_mesh.F90 b/src/oce_mesh.F90 index 653785313..1843e345b 100755 --- a/src/oce_mesh.F90 +++ b/src/oce_mesh.F90 @@ -842,28 +842,28 @@ SUBROUTINE read_mesh(partit, mesh) n=com_elem2D_full%sptr(com_elem2D_full%sPEnum+1)-1 ALLOCATE(com_elem2D_full%slist(n)) read(fileID,*) com_elem2D_full%slist - -!!$ read(fileID,*) com_edge2D%rPEnum -!!$ ALLOCATE(com_edge2D%rPE(com_edge2D%rPEnum)) -!!$ read(fileID,*) com_edge2D%rPE -!!$ ALLOCATE(com_edge2D%rptr(com_edge2D%rPEnum+1)) -!!$ read(fileID,*) com_edge2D%rptr -!!$ ALLOCATE(com_edge2D%rlist(eDim_edge2D)) -!!$ read(fileID,*) com_edge2D%rlist -!!$ -!!$ read(fileID,*) com_edge2D%sPEnum -!!$ ALLOCATE(com_edge2D%sPE(com_edge2D%sPEnum)) -!!$ read(fileID,*) com_edge2D%sPE -!!$ ALLOCATE(com_edge2D%sptr(com_edge2D%sPEnum+1)) -!!$ read(fileID,*) com_edge2D%sptr -!!$ n=com_edge2D%sptr(com_edge2D%sPEnum+1)-1 -!!$ ALLOCATE(com_edge2D%slist(n)) -!!$ read(fileID,*) com_edge2D%slist close(fileID) + if (mype==0) write(*,*) 'communication arrays are read' deallocate(rbuff, ibuff) deallocate(mapping) - + +! necessary for MULTIO auxuary data: +! one element might belong to several processes hence we unify the element partition +! such that sum(myDim_elem2D_shrinked) over all processors will give elem2D + partit%myDim_elem2D_shrinked=0 + DO n=1, myDim_elem2D + if (mesh%elem2D_nodes(1, n) > myDim_nod2D) cycle + partit%myDim_elem2D_shrinked=partit%myDim_elem2D_shrinked+1 + END DO + allocate(partit%myInd_elem2D_shrinked(partit%myDim_elem2D_shrinked)) +! fill the respective indicies + nn=1 + DO n=1, myDim_elem2D + if (mesh%elem2D_nodes(1, n) > myDim_nod2D) cycle + partit%myInd_elem2D_shrinked(nn)=n + nn=nn+1 + END DO ! no checksum for now, execute_command_line is failing too often. if you think it is important, please drop me a line and I will try to revive it: jan.hegewald@awi.de mesh%representative_checksum = '' From 3720b95ff7cc61e96aad6a99cbac2971e3944ab2 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 10 Aug 2023 14:47:46 +0200 Subject: [PATCH 04/71] adjusted format IO from Suvi --- src/ifs_interface/iom.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index 57783098e..fa45a177b 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -113,13 +113,13 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) err = conf_ctx%mpi_return_client_comm(return_comm) IF (err /= MULTIO_SUCCESS) THEN - WRITE (err_str, "(I)") return_comm + WRITE (err_str, "(I10)") return_comm CALL ctl_stop('conf_ctx%mpi_return_client_comm(', err_str,') failed: ', multio_error_string(err)) END IF err = conf_ctx%mpi_parent_comm(int(mio_parent_comm)) IF (err /= MULTIO_SUCCESS) THEN - WRITE (err_str, "(I)") mio_parent_comm + WRITE (err_str, "(I10)") mio_parent_comm CALL ctl_stop('conf_ctx%mpi_parent_comm(', err_str,') failed: ', multio_error_string(err)) END IF @@ -204,7 +204,7 @@ SUBROUTINE iom_init_server(server_comm) err = conf_ctx%mpi_parent_comm(int(mio_parent_comm)) IF (err /= MULTIO_SUCCESS) THEN - WRITE (err_str, "(I)") mio_parent_comm + WRITE (err_str, "(I10)") mio_parent_comm CALL ctl_stop('conf_ctx%mpi_parent_comm(', err_str,') failed: ', multio_error_string(err)) END IF @@ -408,4 +408,4 @@ END SUBROUTINE ctl_stop !!====================================================================== END MODULE iom - \ No newline at end of file + From eaa1a64bbe0d93fb8249e01a45ca97da5adf64d6 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 10 Aug 2023 15:12:12 +0200 Subject: [PATCH 05/71] forgot to define FESOM domains in fesom_module.F90 --- src/fesom_module.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index 422aada79..0f17732b2 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -243,6 +243,10 @@ subroutine fesom_init(fesom_total_nsteps) write(*,*) '============================================' endif +#if defined(__MULTIO) + call iom_send_fesom_domains(f%partit, f%mesh) +#endif + ! f%dump_dir='DUMP/' ! INQUIRE(file=trim(f%dump_dir), EXIST=f%L_EXISTS) ! if (.not. f%L_EXISTS) call system('mkdir '//trim(f%dump_dir)) From 9a1e0fc256478187ceec1075ec5f354777ffa332 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 11 Aug 2023 10:14:31 +0200 Subject: [PATCH 06/71] forgot to nclude 'use iom' in fesom_module --- src/fesom_module.F90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index 0f17732b2..fea998944 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -89,6 +89,9 @@ module fesom_module subroutine fesom_init(fesom_total_nsteps) use fesom_main_storage_module +#if defined(__MULTIO) + use iom +#endif integer, intent(out) :: fesom_total_nsteps ! EO parameters logical mpi_is_initialized From 8f402d82fa523dde8eceb3415bbc9040eb97c22b Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 11 Aug 2023 11:40:06 +0200 Subject: [PATCH 07/71] MULTIO implimentation of Razvan is ready to merge. CMakeFiles still need to be adjusted (we use Razvans currently) to run it with IFS. --- src/ifs_interface/ifs_interface.F90 | 2 ++ src/ifs_interface/iom.F90 | 5 +++-- src/ifs_interface/mpp_io.F90 | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ifs_interface/ifs_interface.F90 b/src/ifs_interface/ifs_interface.F90 index f3d738342..1fdfd7779 100644 --- a/src/ifs_interface/ifs_interface.F90 +++ b/src/ifs_interface/ifs_interface.F90 @@ -9,6 +9,7 @@ MODULE nemogcmcoup_steps INTEGER :: substeps !per IFS timestep END MODULE nemogcmcoup_steps +#if defined(__MULTIO) SUBROUTINE nemogcmcoup_init_ioserver( icomm, lnemoioserver, irequired, iprovided, lmpi1) ! Initialize the NEMO mppio server @@ -46,6 +47,7 @@ SUBROUTINE nemogcmcoup_end_ioserver CALL mpp_stop() END SUBROUTINE nemogcmcoup_end_ioserver +#endif SUBROUTINE nemogcmcoup_init( mype, icomm, inidate, initime, itini, itend, zstp, & & lwaveonly, iatmunit, lwrite ) diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index fa45a177b..27ec77942 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -6,6 +6,7 @@ !----------------------------------------------------- MODULE iom +#if defined(__MULTIO) USE multio_api USE, INTRINSIC :: iso_fortran_env, only: real64 @@ -407,5 +408,5 @@ SUBROUTINE ctl_stop(m1, m2, m3, m4) END SUBROUTINE ctl_stop !!====================================================================== - END MODULE iom - +#endif +END MODULE iom diff --git a/src/ifs_interface/mpp_io.F90 b/src/ifs_interface/mpp_io.F90 index 95a939049..eda8feae7 100644 --- a/src/ifs_interface/mpp_io.F90 +++ b/src/ifs_interface/mpp_io.F90 @@ -6,6 +6,7 @@ !----------------------------------------------------- MODULE mpp_io +#if defined(__MULTIO) USE iom IMPLICIT NONE PRIVATE @@ -177,5 +178,5 @@ SUBROUTINE mpp_stop CALL mpi_finalize( ierr ) END SUBROUTINE mpp_stop - -END MODULE mpp_io \ No newline at end of file +#endif +END MODULE mpp_io From 64093df4454183c62cee4acff88ba0fdded24d6e Mon Sep 17 00:00:00 2001 From: Sebastian Beyer Date: Tue, 22 Aug 2023 11:11:03 +0200 Subject: [PATCH 08/71] remove unused TODO block for currents coupling --- src/ifs_interface/ifs_interface.F90 | 46 +---------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/src/ifs_interface/ifs_interface.F90 b/src/ifs_interface/ifs_interface.F90 index 1fdfd7779..f27aac06d 100644 --- a/src/ifs_interface/ifs_interface.F90 +++ b/src/ifs_interface/ifs_interface.F90 @@ -575,50 +575,6 @@ SUBROUTINE nemogcmcoup_lim2_get( mype, npes, icomm, & nfield = nfield + 1 pgvcur(:) = zrecvnf(:,nfield) - ! Pack u(v) surface currents on elements - !zsendnfUV(:,1)=fesom%dynamics%UV(1,1,1:myDim_elem2D) - !zsendnfUV(:,2)=fesom%dynamics%UV(2,1,1:myDim_elem2D) !UV includes eDim, leave those away here - !nfielduv = 2 - ! - !do elem=1, myDim_elem2D - ! - ! ! compute element midpoints - ! elnodes=elem2D_nodes(:,elem) - ! rlon=sum(coord_nod2D(1,elnodes))/3.0_wpIFS - ! rlat=sum(coord_nod2D(2,elnodes))/3.0_wpIFS - ! - ! ! Rotate vectors to geographical coordinates (r2g) - ! CALL vector_r2g(zsendnfUV(elem,1), zsendnfUV(elem,2), rlon, rlat, 0) ! 0-flag for rot. coord - ! - !end do - -#ifdef FESOM_TODO - - ! We need to sort out the non-unique global index before we - ! can couple currents - - ! Interpolate: 'pgucur' and 'pgvcur' on Gaussian grid. - IF (lparintmultatm) THEN - CALL parinter_fld_mult( nfielduv, mype, npes, icomm, UVtogauss, & - & myDim_nod2D, zsendnfUV, & - & nopoints, zrecvnfUV ) - ELSE - DO jf = 1, nfielduv - CALL parinter_fld( mype, npes, icomm, UVtogauss, & - & myDim_nod2D, zsendnfUV(:,jf), & - & nopoints, zrecvnfUV(:,jf) ) - ENDDO - ENDIF - pgucur(:) = zrecvnfUV(:,1) - pgvcur(:) = zrecvnfUV(:,2) - -#else - - !pgucur(:) = 0.0 - !pgvcur(:) = 0.0 - -#endif - END SUBROUTINE nemogcmcoup_lim2_get @@ -1187,4 +1143,4 @@ SUBROUTINE nemogcmcoup_final endif CALL fesom_finalize -END SUBROUTINE nemogcmcoup_final +END SUBROUTINE nemogcmcoup_final \ No newline at end of file From 224e4680e65a6043414c233bd581c87bf568ba5a Mon Sep 17 00:00:00 2001 From: Sebastian Beyer Date: Tue, 22 Aug 2023 11:26:40 +0200 Subject: [PATCH 09/71] remove some "done" comments --- src/ifs_interface/ifs_interface.F90 | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/ifs_interface/ifs_interface.F90 b/src/ifs_interface/ifs_interface.F90 index f27aac06d..847fa60ae 100644 --- a/src/ifs_interface/ifs_interface.F90 +++ b/src/ifs_interface/ifs_interface.F90 @@ -818,24 +818,20 @@ SUBROUTINE nemogcmcoup_lim2_update( mype, npes, icomm, & ! Sort out incoming arrays from the IFS and put them on the ocean grid ! TODO - shortwave(:)=0. ! Done, updated below. What to do with shortwave over ice?? - !longwave(:)=0. ! Done. Only used in stand-alone mode. - prec_rain(:)=0. ! Done, updated below. - prec_snow(:)=0. ! Done, updated below. - evap_no_ifrac=0. ! Done, updated below. This is evap over ocean, does this correspond to evap_tot? - sublimation=0. ! Done, updated below. + shortwave(:)=0. + !longwave(:)=0. + prec_rain(:)=0. + prec_snow(:)=0. + evap_no_ifrac=0. + sublimation=0. ! - ice_heat_flux=0. ! Done. This is qns__ice currently. Is this the non-solar heat flux? ! non solar heat fluxes below ! (qns) - oce_heat_flux=0. ! Done. This is qns__oce currently. Is this the non-solar heat flux? + ice_heat_flux=0. + oce_heat_flux=0. ! - !runoff(:)=0. ! not used apparently. What is runoffIN, ocerunoff? - !evaporation(:)=0. - !ice_thermo_cpl.F90: !---- total evaporation (needed in oce_salt_balance.F90) - !ice_thermo_cpl.F90: evaporation = evap_no_ifrac*(1.-a_ice) + sublimation*a_ice - stress_atmice_x=0. ! Done, taux_ice - stress_atmice_y=0. ! Done, tauy_ice - stress_atmoce_x=0. ! Done, taux_oce - stress_atmoce_y=0. ! Done, tauy_oce + stress_atmice_x=0. + stress_atmice_y=0. + stress_atmoce_x=0. + stress_atmoce_y=0. ! =================================================================== ! ! Pack all arrays From d376f748ea49a9bb63fa1897808b0b5c3f01b712 Mon Sep 17 00:00:00 2001 From: Sebastian Beyer Date: Tue, 22 Aug 2023 11:56:20 +0200 Subject: [PATCH 10/71] fix misleading message about fesom abort --- src/ifs_interface/ifs_notused.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ifs_interface/ifs_notused.F90 b/src/ifs_interface/ifs_notused.F90 index 0beda3670..1cedbc82c 100644 --- a/src/ifs_interface/ifs_notused.F90 +++ b/src/ifs_interface/ifs_notused.F90 @@ -175,7 +175,7 @@ SUBROUTINE nemogcmcoup_update_add( mype, npes, icomm, & ! Local variables if(fesom%mype==0) then - WRITE(0,*)'nemogcmcoup_update_add should not be called when coupling to fesom. Commented ABORT. Proceeding...' + WRITE(0,*)'In nemogcmcoup_update_add FESOM dummy routine. Proceeding...' !CALL abort endif @@ -304,4 +304,3 @@ SUBROUTINE nemogcmcoup_wam_update_stress( mype, npes, icomm, npoints, & CALL abort END SUBROUTINE nemogcmcoup_wam_update_stress - From 9ffffeb255b51aabbb3a79a260be384a5c71b937 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 21 Aug 2023 11:06:14 +0200 Subject: [PATCH 11/71] Add same information from GH readme to RTD docs --- docs/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 8dfe72d41..28af8b1fb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,6 +6,15 @@ FESOM2 documentation ==================== +The Finite volumE Sea Ice-Ocean Model (FESOM2). + +Multi-resolution ocean general circulation model that solves +the equations of motion describing the ocean and sea ice using +finite-volume methods on unstructured computational grids. The +model is developed and supported by researchers at the Alfred +Wegener Institute, Helmholtz Centre for Polar and Marine +Research (AWI), in Bremerhaven, Germany. + Authors ------- From de9c28673558b457415c3059941be5896acad937 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Wed, 30 Aug 2023 13:04:10 +0200 Subject: [PATCH 12/71] added a subroutine (node_contours) to be used for description of scalar control volumes for YAC, OASIS, ATLAS etc. --- src/node_contour_boundary.h | 28 ++++++ src/node_contour_inner.h | 25 ++++++ src/node_contours.F90 | 174 ++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+) create mode 100644 src/node_contour_boundary.h create mode 100644 src/node_contour_inner.h create mode 100644 src/node_contours.F90 diff --git a/src/node_contour_boundary.h b/src/node_contour_boundary.h new file mode 100644 index 000000000..938ce8d87 --- /dev/null +++ b/src/node_contour_boundary.h @@ -0,0 +1,28 @@ +flag=1 +nn=0 +do while (flag==1) + if (elem2D_nodes(1,elem)==n) then + edge_left=elem_edges(3,elem) + edge_right=elem_edges(2,elem) + elseif (elem2D_nodes(2,elem)==n) then + edge_left=elem_edges(1,elem) + edge_right=elem_edges(3,elem) + else + edge_left=elem_edges(2,elem) + edge_right=elem_edges(1,elem) + end if + nn=nn+1 + nedges(nn)=edge_left + nelems(nn)=elem + el=edge_tri(:,edge_right) + if (el(2)>0) then + if (el(1)==elem) then + elem=el(2) + else + elem=el(1) + end if + else !the last element + nedges(nn+1)=edge_right + flag=0 + end if +end do \ No newline at end of file diff --git a/src/node_contour_inner.h b/src/node_contour_inner.h new file mode 100644 index 000000000..1e4555f89 --- /dev/null +++ b/src/node_contour_inner.h @@ -0,0 +1,25 @@ +nn=0 +do while (nnedge2D_in) then + bEdge_left=bEdge_left+1 + belem_left(bEdge_left)=elem + end if + if (myList_edge2D(edge_right)>edge2D_in) then + bEdge_right=bEdge_right+1 + belem_right(bEdge_right)=elem + end if + end do + +! now we have three cases + if (bEdge_left==0) then ! inner contour + elem=nod_in_elem2D(1, n) ! we can start from any + allocate(nedges(nod_in_elem2D_num(n))) + nedges=0 + allocate(nelems(nod_in_elem2D_num(n))) + nelems=0 + !!!!!!! inner_node_contour +#include "node_contour_inner.h" + do nn=1, nod_in_elem2D_num(n) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + do nn=nod_in_elem2D_num(n)+1, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)) + my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)) + end do + deallocate(nedges, nelems) + end if + + if (bEdge_left==1) then ! standard boundary node + elem=belem_left(1) + allocate(nedges(nod_in_elem2D_num(n)+1)) + nedges=0 + allocate(nelems(nod_in_elem2D_num(n))) + nelems=0 + !!!!!!!boundary_node_contour +#include "node_contour_boundary.h" + do nn=1, nod_in_elem2D_num(n) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + nn=nod_in_elem2D_num(n)+1 + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + nn=nod_in_elem2D_num(n)+2 + my_x_corners(n, nn)=coord_nod2D(1,n) + my_y_corners(n, nn)=coord_nod2D(2,n) + do nn=nod_in_elem2D_num(n)+3, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)+2) + my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)+2) + end do + !!!!!!! + deallocate(nedges, nelems) + end if + + if (bEdge_left==2) then ! strange boundary node + elem=belem_left(1) + allocate(nedges (nod_in_elem2D_num(n)+1)) + allocate(nedges1(nod_in_elem2D_num(n)+1)) + nedges =0 + nedges1=0 + allocate(nelems (nod_in_elem2D_num(n))) + allocate(nelems1(nod_in_elem2D_num(n))) + nelems=0 + nelems1=0 + !!!!!!!boundary_node_contour +#include "node_contour_boundary.h" + where (nedges>0) + nedges1=nedges + end where + where (nelems>0) + nelems1=nelems + end where + nn1=nn + do nn=1, nn1 + call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems1(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + nn=nn1+1 + call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + nn=nn1+2 + my_x_corners(n, nn)=coord_nod2D(1,n) + my_y_corners(n, nn)=coord_nod2D(2,n) + !!!!!!! + elem=belem_left(2) + allocate(nedges2(nod_in_elem2D_num(n)+1)) + nedges =0 + nedges2=0 + allocate(nelems2(nod_in_elem2D_num(n))) + nelems =0 + nelems2=0 + !!!!!!!boundary_node_contour +#include "node_contour_boundary.h" + where (nedges>0) + nedges2=nedges + end where + where (nelems>0) + nelems2=nelems + end where + nn2=nn + do nn=nn1+3, nn1+nn2+2 + call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems2(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + nn=nn1+nn2+3 + call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + nn=nn1+nn2+4 + my_x_corners(n, nn)=coord_nod2D(1,n) + my_y_corners(n, nn)=coord_nod2D(2,n) + do nn=nn1+nn2+5, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nn1+nn2+4) + my_y_corners(n, nn)=my_y_corners(n, nn1+nn2+4) + end do + !!!!!!! + deallocate(nedges, nelems, nedges1, nelems1, nedges2, nelems2) + end if +end do +do n=1, myDim_nod2D + do nn=1, size(my_x_corners, 2) + this_x_coord=my_x_corners(n, nn) + this_y_coord=my_y_corners(n, nn) + call r2g(my_x_corners(n, nn), my_y_corners(n, nn), this_x_coord, this_y_coord) + end do +end do +my_x_corners=my_x_corners/rad +my_y_corners=my_y_corners/rad +end subroutine node_contours \ No newline at end of file From 96a95deff3df6ac4fcb06861fab8d51a438c3d23 Mon Sep 17 00:00:00 2001 From: mandresm Date: Mon, 4 Sep 2023 16:50:02 +0200 Subject: [PATCH 13/71] adds the possibility of specifying the compiler as the input of configure --- configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.sh b/configure.sh index 86d7eb02a..211b50e79 100755 --- a/configure.sh +++ b/configure.sh @@ -2,7 +2,7 @@ set -e -source env.sh # source this from your run script too +source env.sh $1 # source this from your run script too mkdir build || true # make sure not to commit this to svn or git cd build cmake .. $@ -DCMAKE_BUILD_TYPE=Debug # not required when re-compiling From a07ddd28b97ed02d1c46098bcd200348d0ae7883 Mon Sep 17 00:00:00 2001 From: suvarchal Date: Tue, 5 Sep 2023 22:17:06 +0200 Subject: [PATCH 14/71] add support for using + in machine spec --- env.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/env.sh b/env.sh index a073ad208..fbf5da384 100755 --- a/env.sh +++ b/env.sh @@ -32,8 +32,17 @@ if [[ -z "$1" ]] || [[ "$1" =~ ^- ]]; then # no argument given LOGINHOST="$(hostname -f)" else - LOGINHOST=$1 # 1st arg exists and doesn't start with -, meaning it is machine specification + MACHINESPEC=$1 # 1st arg exists and doesn't start with -, meaning it is machine specification shift # pop the argument as we already stored it, remaining arguments are passed to cmake + + # check if given machine spec has + in it. if so save the later part as compilerid + if [[ $LOGINHOST == *"+"* ]]; then + LOGINHOST="${MACHINESPEC%%+*}" # Everything before the '+' + COMPILERID="${MACHINESPEC#*+}" # Everything after the '+' + else + LOGINHOST="$MACHINESPEC" + COMPILERID="" + fi fi @@ -58,9 +67,9 @@ elif [[ $LOGINHOST =~ \.hww\.de$ ]] || [[ $LOGINHOST =~ ^nid[0-9]{5}$ ]]; then STRATEGY="hazelhen.hww.de" elif [[ $LOGINHOST =~ \.jureca$ ]]; then STRATEGY="jureca" -elif [[ $LOGINHOST = ubuntu ]]; then +elif [[ $LOGINHOST =~ ^ubuntu ]]; then STRATEGY="ubuntu" -elif [[ $LOGINHOST = bsc ]]; then +elif [[ $LOGINHOST =~ ^bsc ]]; then STRATEGY="bsc" elif [[ $LOGINHOST =~ ^juwels[0-9][0-9].ib.juwels.fzj.de$ ]]; then STRATEGY="juwels" From f9ab2472c54d543b4365ec57d6fae1a398806483 Mon Sep 17 00:00:00 2001 From: suvarchal Date: Tue, 5 Sep 2023 22:17:50 +0200 Subject: [PATCH 15/71] remove redundant compilerid regex for levante --- env.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/env.sh b/env.sh index fbf5da384..d63771c68 100755 --- a/env.sh +++ b/env.sh @@ -50,11 +50,6 @@ if [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then STRATEGY="mistral.dkrz.de" elif [[ $LOGINHOST =~ ^levante ]] || [[ $LOGINHOST =~ ^l[:alnum:]+\.lvt\.dkrz\.de$ ]]; then STRATEGY="levante.dkrz.de" - # following regex only matches if input is 2 word like levante.nvhpc, this enables using different shells for a machine directly - compid_regex="^([[:alnum:]]+)\.([[:alnum:]]+)$" - if [[ $LOGINHOST =~ $compid_regex ]]; then - COMPILERID="${BASH_REMATCH[2]}" - fi elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then STRATEGY="ollie" elif [[ $LOGINHOST =~ ^albedo[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then From 92090fe75b6cc4fcb7aed7a50a4dcba6ff2e784d Mon Sep 17 00:00:00 2001 From: suvarchal Date: Tue, 5 Sep 2023 22:19:06 +0200 Subject: [PATCH 16/71] remove passed to env.sh from PR, all args passed --- configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.sh b/configure.sh index 211b50e79..86d7eb02a 100755 --- a/configure.sh +++ b/configure.sh @@ -2,7 +2,7 @@ set -e -source env.sh $1 # source this from your run script too +source env.sh # source this from your run script too mkdir build || true # make sure not to commit this to svn or git cd build cmake .. $@ -DCMAKE_BUILD_TYPE=Debug # not required when re-compiling From 2d0268cdb789d3c23ef2bcc9bed9076032314b7e Mon Sep 17 00:00:00 2001 From: suvarchal Date: Tue, 5 Sep 2023 22:25:46 +0200 Subject: [PATCH 17/71] minor bug in if for compilerid detection --- env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/env.sh b/env.sh index d63771c68..5ac127a22 100755 --- a/env.sh +++ b/env.sh @@ -36,7 +36,7 @@ else shift # pop the argument as we already stored it, remaining arguments are passed to cmake # check if given machine spec has + in it. if so save the later part as compilerid - if [[ $LOGINHOST == *"+"* ]]; then + if [[ $MACHINESPEC == *"+"* ]]; then LOGINHOST="${MACHINESPEC%%+*}" # Everything before the '+' COMPILERID="${MACHINESPEC#*+}" # Everything after the '+' else @@ -98,6 +98,7 @@ else export FESOM_PLATFORM_STRATEGY=$STRATEGY SHELLFILE="${DIR}/env/${STRATEGY}/shell" if [[ -n ${COMPILERID} ]]; then + echo "Compiler ID for shell is: ${COMPILERID}" SHELLFILE="${SHELLFILE}.${COMPILERID}" fi if [[ ! -e ${SHELLFILE} ]]; then From a647cedd865ebe80741e692d8d241f83aaae7260 Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Wed, 6 Sep 2023 13:21:09 +0200 Subject: [PATCH 18/71] fix that ensures we dont check for salinity <= 3 inside ice shelf --- src/write_step_info.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write_step_info.F90 b/src/write_step_info.F90 index e9b601b98..858dd1812 100644 --- a/src/write_step_info.F90 +++ b/src/write_step_info.F90 @@ -417,7 +417,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) end if ! --> if ( .not. trim(which_ALE)=='linfs' .and. ... - do nz=1,nlevels_nod2D(n)-1 + do nz=ulevels_nod2D(n),nlevels_nod2D(n)-1 !_______________________________________________________________ ! check temp if ( (tracers%data(1)%values(nz, n) /= tracers%data(1)%values(nz, n)) .or. & From 098a486f86f204adadb87323a927c677b6470cd3 Mon Sep 17 00:00:00 2001 From: suvarchal Date: Tue, 19 Sep 2023 23:17:13 +0200 Subject: [PATCH 19/71] add MULTIO cmake-modulefor MULTIO discovery, compile flags for MULTIO and generic cmakelist changes to declutter it a bit --- CMakeLists.txt | 1 + cmake/FindMULTIO.cmake | 50 ++++++++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 40 ++++++++++++++++++++++++++++----- 3 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 cmake/FindMULTIO.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 60a65d66f..2e9ba6ff9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs F set(CRAY OFF CACHE BOOL "compile with cray ftn") set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea ice column physics.") set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results") +set(USE_MULTIO OFF CACHE BOOL "Use MULTIO for IO, either grib or binary for now. This also means path to MULTIO installation has to provided using env MULTIO_INSTALL_PATH='..' and multio configuration yamls must be present to run the model with MULTIO") #set(VERBOSE OFF CACHE BOOL "toggle debug output") #add_subdirectory(oasis3-mct/lib/psmile) diff --git a/cmake/FindMULTIO.cmake b/cmake/FindMULTIO.cmake new file mode 100644 index 000000000..5c86c30b8 --- /dev/null +++ b/cmake/FindMULTIO.cmake @@ -0,0 +1,50 @@ +# FindMULTIO.cmake + +include(FindPackageHandleStandardArgs) + +# Use the environment variable as a hint +set(MULTIO_HINT_PATH $ENV{MULTIO_INSTALL_PATH}) + +# Try to find the library +find_library(MULTIO_FAPI_LIBRARY + NAMES multio-fapi # Adjust this if the library name is different + HINTS ${MULTIO_HINT_PATH}/lib +) + +# Try to find the dependency library +find_library(MULTIO_API_LIBRARY + NAMES multio-api + HINTS ${MULTIO_HINT_PATH}/lib +) + +# Try to find the dependency library +find_library(MULTIO_LIBRARY + NAMES multio + HINTS ${MULTIO_HINT_PATH}/lib +) + +# Try to find the Fortran module path +find_path(MULTIO_MODULE_PATH + NAMES multio_api.mod # Replace with an actual module name you expect to find + HINTS ${MULTIO_HINT_PATH}/module ${MULTIO_HINT_PATH}/multio/module +) + + +# Aggregate the libraries for easier linking +set(MULTIO_LIBRARIES ${MULTIO_FAPI_LIBRARY} ${MULTIO_API_LIBRARY} ${MULTIO_LIBRARY}) + + +# Handle the results +find_package_handle_standard_args(MULTIO + REQUIRED_VARS MULTIO_LIBRARIES MULTIO_MODULE_PATH + FOUND_VAR MULTIO_FOUND +) + +# If found, set the MULTIO_LIBRARIES and MULTIO_INCLUDE_DIRS variables for easy use +if(MULTIO_FOUND) + set(MULTIO_INCLUDE_DIRS ${MULTIO_MODULE_PATH}) +endif() + +# Mark variables as advanced +mark_as_advanced(MULTIO_LIBRARY MULTIO_MODULE_PATH) + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 79307ff0b..d3e291314 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.9) project(fesom C Fortran) +find_package(MPI REQUIRED) + +#TODO: these machine specific changes must be done using cmake toolchain then here otherwise will make cmake another raps. if(DEFINED ENV{FESOM_PLATFORM_STRATEGY}) set(FESOM_PLATFORM_STRATEGY $ENV{FESOM_PLATFORM_STRATEGY} CACHE STRING "switch to platform specific compile settings, this is usually determined via the env.sh script") else() @@ -19,6 +22,7 @@ else() option(ALBEDO_INTELMPI_WORKAROUNDS "workaround for performance issues on albedo" OFF) endif() +#TODO: these machine specific changes must be done using cmake toolchain then here otherwise will make cmake another raps. if(ALEPH_CRAYMPICH_WORKAROUNDS) # todo: enable these options only for our targets @@ -35,6 +39,7 @@ if(ALBEDO_INTELMPI_WORKAROUNDS) add_compile_options(-DDISABLE_PARALLEL_RESTART_READ) endif() +#TODO: this machine specific block can easyly go in cmake toolchain. if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) message(STATUS "multithreading disabled for Levante") # multithreading suddenly produces an error, disable it until a fix is found. issue #413 option(DISABLE_MULTITHREADING "disable asynchronous operations" ON) @@ -72,9 +77,15 @@ elseif(${BUILD_FESOM_AS_LIBRARY}) else() file(GLOB sources_Fortran ${src_home}/*.F90) endif() + +if(${USE_MULTIO}) + list(APPEND sources_Fortran ${src_home}/ifs_interface/iom.F90 ${src_home}/ifs_interface/mpp_io.F90) +endif() + + #list(REMOVE_ITEM sources_Fortran ${src_home}/fesom_partition_init.F90) file(GLOB sources_C ${src_home}/*.c) -list(REMOVE_ITEM sources_C ${src_home}/psolve_feom.c) # does the file still exist? +#list(REMOVE_ITEM sources_C ${src_home}/psolve_feom.c) # this file doesn't exist! please check and remove this # generate a custom file from fesom_version_info.F90 which includes the current git SHA set(FESOM_ORIGINAL_VERSION_FILE ${src_home}/fesom_version_info.F90) @@ -101,7 +112,10 @@ add_subdirectory(../lib/parms ${PROJECT_BINARY_DIR}/parms) add_subdirectory(async_threads_cpp) -include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindNETCDF.cmake) +set(CMAKE_MODULE_PATH ${CURRENT_LIST_DIR}../cmake) + +find_package(NETCDF REQUIRED) +#include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindNETCDF.cmake) if(${BUILD_FESOM_AS_LIBRARY}) add_library(${PROJECT_NAME}_C ${sources_C}) @@ -146,6 +160,7 @@ endif() # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel) if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) if(${BUILD_FESOM_AS_LIBRARY}) + # TODO: not the best idea to use -xHost will stop from cross compiling tests in CI target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -init=zero -no-wrap-margin -fpe0) # add -fpe0 for RAPS environment else() target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin) @@ -193,6 +208,8 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) endif() endif() if(${BUILD_FESOM_AS_LIBRARY}) + + set(CMAKE_POSITION_INDEPENDENT_CODE ON) # not sure below are needed anymore target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) target_compile_options(${PROJECT_NAME}_C PRIVATE -fPIC) endif() @@ -201,7 +218,14 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTO target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} async_threads_cpp) +target_link_libraries(${PROJECT_NAME} async_threads_cpp) + +# MULTIO +if(${USE_MULTIO}) + find_package(MULTIO REQUIRED) + target_link_libraries(${PROJECT_NAME} ${MULTIO_LIBRARIES}) + target_include_directories(${PROJECT_NAME} PRIVATE ${MULTIO_INCLUDE_DIRS}) +endif() set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) target_compile_options(${PROJECT_NAME} PRIVATE ${OpenMP_Fortran_FLAGS}) # currently we only have OpenMP in the Fortran part @@ -213,9 +237,13 @@ set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "director if(${BUILD_FESOM_AS_LIBRARY}) install(TARGETS ${PROJECT_NAME} DESTINATION "${FESOM_INSTALL_PREFIX}/lib") install(TARGETS ${PROJECT_NAME}_C DESTINATION "${FESOM_INSTALL_PREFIX}/lib") + # below may be needed if we want to use fesom in ec-bundle + #install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") + #install(TARGETS ${PROJECT_NAME}_C EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") + else() set(FESOM_INSTALL_FILEPATH "${FESOM_INSTALL_PREFIX}/bin/fesom.x") -get_filename_component(FESOM_INSTALL_PATH ${FESOM_INSTALL_FILEPATH} DIRECTORY) -get_filename_component(FESOM_INSTALL_NAME ${FESOM_INSTALL_FILEPATH} NAME) -install(PROGRAMS ${PROJECT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${FESOM_INSTALL_PATH} RENAME ${FESOM_INSTALL_NAME}) + get_filename_component(FESOM_INSTALL_PATH ${FESOM_INSTALL_FILEPATH} DIRECTORY) + get_filename_component(FESOM_INSTALL_NAME ${FESOM_INSTALL_FILEPATH} NAME) + install(PROGRAMS ${PROJECT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${FESOM_INSTALL_PATH} RENAME ${FESOM_INSTALL_NAME}) endif() From 41e541bd716ba87b7a3e3577ffbf40b585b665cf Mon Sep 17 00:00:00 2001 From: suvarchal Date: Wed, 20 Sep 2023 12:33:27 +0200 Subject: [PATCH 20/71] changes to support fesom MIO standalone --- src/CMakeLists.txt | 5 ++- src/MOD_PARTIT.F90 | 1 + src/fesom_module.F90 | 13 ++++++-- src/gen_modules_partitioning.F90 | 44 +++++++++++++++++++++----- src/ifs_interface/iom.F90 | 36 ++++++++++----------- src/ifs_interface/mpp_io.F90 | 54 +++++++++++++++++++++++++++++--- 6 files changed, 120 insertions(+), 33 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d3e291314..b38ebc9dc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -71,7 +71,7 @@ if(${USE_ICEPACK}) file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/icepack_drivers/*.F90 ${src_home}/icepack_drivers/Icepack/columnphysics/*.F90) -elseif(${BUILD_FESOM_AS_LIBRARY}) +elseif(${BUILD_FESOM_AS_LIBRARY}) # TODO: assumes multio is always on file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/ifs_interface/*.F90) # ICEPACK + LIBRARY NOT SUPPORTED (YET) else() @@ -135,6 +135,9 @@ else() add_executable(${PROJECT_NAME} ${sources_Fortran} ${src_home}/fesom_main.F90) endif() target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243) +if(${USE_MULTIO}) + target_compile_definitions(${PROJECT_NAME} PRIVATE __MULTIO) +endif() if(${DISABLE_MULTITHREADING}) target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_MULTITHREADING) endif() diff --git a/src/MOD_PARTIT.F90 b/src/MOD_PARTIT.F90 index fb4a88542..3a76761bf 100644 --- a/src/MOD_PARTIT.F90 +++ b/src/MOD_PARTIT.F90 @@ -58,6 +58,7 @@ module MOD_PARTIT integer :: pe_status = 0 ! if /=0 then something is wrong integer :: MPI_COMM_FESOM ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD) + integer :: MPI_COMM_WORLD ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD) ! MPI Datatypes for interface exchange ! Element fields (2D; 2D integer; 3D with nl-1 or nl levels, 1 - 4 values) diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index fea998944..65e3cd400 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -45,7 +45,7 @@ module fesom_main_storage_module integer :: n, from_nstep, offset, row, i, provided integer :: which_readr ! read which restart files (0=netcdf, 1=core dump,2=dtype) integer :: total_nsteps - integer, pointer :: mype, npes, MPIerr, MPI_COMM_FESOM + integer, pointer :: mype, npes, MPIerr, MPI_COMM_FESOM, MPI_COMM_WORLD real(kind=WP) :: t0, t1, t2, t3, t4, t5, t6, t7, t8, t0_ice, t1_ice, t0_frc, t1_frc real(kind=WP) :: rtime_fullice, rtime_write_restart, rtime_write_means, rtime_compute_diag, rtime_read_forcing real(kind=real32) :: rtime_setup_mesh, rtime_setup_ocean, rtime_setup_forcing @@ -111,7 +111,7 @@ subroutine fesom_init(fesom_total_nsteps) !OIFS-FESOM2 coupling: does not require MPI_INIT here as this is done by OASIS call MPI_Initialized(mpi_is_initialized, f%i) if(.not. mpi_is_initialized) then - ! do not initialize MPI here if it has been initialized already, e.g. via IFS when fesom is called as library (__ifsinterface is defined) + ! TODO: do not initialize MPI here if it has been initialized already, e.g. via IFS when fesom is called as library (__ifsinterface is defined) call MPI_INIT_THREAD(MPI_THREAD_MULTIPLE, f%provided, f%i) f%fesom_did_mpi_init = .true. end if @@ -128,6 +128,7 @@ subroutine fesom_init(fesom_total_nsteps) f%mype =>f%partit%mype f%MPIerr =>f%partit%MPIerr f%MPI_COMM_FESOM=>f%partit%MPI_COMM_FESOM + f%MPI_COMM_WORLD=>f%partit%MPI_COMM_WORLD f%npes =>f%partit%npes if(f%mype==0) then write(*,*) @@ -406,6 +407,10 @@ subroutine fesom_runloop(current_nsteps) subroutine fesom_finalize() use fesom_main_storage_module +#if defined(__MULTIO) + use iom + use mpp_io +#endif ! EO parameters real(kind=real32) :: mean_rtime(15), max_rtime(15), min_rtime(15) @@ -448,6 +453,10 @@ subroutine fesom_finalize() ! OpenIFS coupled version has to call oasis_terminate through par_ex call par_ex(f%partit%MPI_COMM_FESOM, f%partit%mype) #endif + +#ifdef __MULTIO && (ifndef __ifsinterface || ifndef __oasis) + call mpp_stop +#endif if(f%fesom_did_mpi_init) call par_ex(f%partit%MPI_COMM_FESOM, f%partit%mype) ! finalize MPI before FESOM prints its stats block, otherwise there is sometimes output from other processes from an earlier time in the programm AFTER the starts block (with parastationMPI) if (f%mype==0) then 41 format (a35,a10,2a15) !Format for table heading diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90 index edf441e73..c296a0f85 100644 --- a/src/gen_modules_partitioning.F90 +++ b/src/gen_modules_partitioning.F90 @@ -42,17 +42,41 @@ subroutine par_init(partit) ! initializes MPI USE o_PARAM USE MOD_PARTIT USE MOD_PARSUP +#ifdef __MULTIO + USE iom + USE mpp_io +#endif + implicit none type(t_partit), intent(inout), target :: partit integer :: i integer :: provided_mpi_thread_support_level character(:), allocatable :: provided_mpi_thread_support_level_name - -#if defined __oasis || defined __ifsinterface +#ifdef __oasis || defined __ifsinterface ! use comm from coupler or ifs -#else - partit%MPI_COMM_FESOM=MPI_COMM_WORLD ! use global comm if not coupled (e.g. no __oasis or __ifsinterface) -#endif + ! TODO: multio with __ifsinterface is magically handled by IFS by using same module + ! names and routine names as in src/ifs_interface, that is not elegant. +#else +#ifdef __MULTIO + ! fesom standalone with MULTIO + character(len=255) :: oce_npes_str, mio_npes_str + integer :: oce_npes_int, mio_npes_int, oce_status, mio_status + CALL MPI_Comm_Size(MPI_COMM_WORLD, partit%npes, i) + CALL MPI_Comm_Rank(MPI_COMM_WORLD, partit%mype, i) + partit%MPI_COMM_FESOM=MPI_COMM_WORLD + partit%MPI_COMM_WORLD=MPI_COMM_WORLD + !TODO: dont need both env variables can do with just one and subtract from npes + CALL get_environment_variable('OCE_NPES', oce_npes_str, status=oce_status) + CALL get_environment_variable('MIO_NPES', mio_npes_str, status=mio_status) + read(oce_npes_str,*,iostat=oce_status) oce_npes_int + read(mio_npes_str,*,iostat=mio_status) mio_npes_int + call mpp_io_init_2(partit%MPI_COMM_FESOM) +#else + partit%MPI_COMM_FESOM=MPI_COMM_WORLD ! use global comm if not coupled (e.g. no __oasis or __ifsinterface or IO server) +#endif + +#endif + call MPI_Comm_Size(partit%MPI_COMM_FESOM,partit%npes,i) call MPI_Comm_Rank(partit%MPI_COMM_FESOM,partit%mype,i) @@ -104,16 +128,22 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI ! For standalone runs we directly call the MPI_barrier and MPI_finalize !--------------------------------------------------------------- +!TODO: logic is convoluted here, not defined oasis and model needs to abort doesn't happen using par_ex #ifndef __oasis if (present(abort)) then if (mype==0) write(*,*) 'Run finished unexpectedly!' call MPI_ABORT(COMM, 1 ) else + ! TODO: this is where fesom standalone, ifsinterface etc get to + !1. there no abort actually even when model calls abort, and barrier may hang + !2. when using fesom as lib using finalize is bad here as there may + ! be other MPI tasks running in calling library like IFS, better + ! better practice in that case would be to free the communicator. call MPI_Barrier(COMM, error) call MPI_Finalize(error) endif -#else ! standalone - +#else ! standalone +! TODO logic below is also convoluted really not really for standalone ! From here on the two coupled options !------------------------------------- #if defined (__oifs) diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index 27ec77942..7166efe8f 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -106,11 +106,11 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) END IF - - err = conf_ctx%mpi_client_id(client_id) - IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('conf_ctx%mpi_client_id(', TRIM(client_id),') failed: ', multio_error_string(err)) - END IF +! TODO: mpi_client_id not in multio main fapi only in dummy api +! err = conf_ctx%mpi_client_id(client_id) +! IF (err /= MULTIO_SUCCESS) THEN +! CALL ctl_stop('conf_ctx%mpi_client_id(', TRIM(client_id),') failed: ', multio_error_string(err)) +! END IF err = conf_ctx%mpi_return_client_comm(return_comm) IF (err /= MULTIO_SUCCESS) THEN @@ -133,12 +133,12 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('conf_ctx%delete() failed: ', multio_error_string(err)) END IF - + ! TODO: not in multio main fapi only in dummy api ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface - err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) - IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) - END IF +! err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) +! IF (err /= MULTIO_SUCCESS) THEN +! CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) +! END IF err = mio_handle%open_connections(); IF (err /= MULTIO_SUCCESS) THEN @@ -208,13 +208,13 @@ SUBROUTINE iom_init_server(server_comm) WRITE (err_str, "(I10)") mio_parent_comm CALL ctl_stop('conf_ctx%mpi_parent_comm(', err_str,') failed: ', multio_error_string(err)) END IF - + ! TODO: not in multio main fapi only in dummy api ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface ! Set handler before invoking blocking start server call - err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) - IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) - END IF +! err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) +! IF (err /= MULTIO_SUCCESS) THEN +! CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) +! END IF ! Blocking call err = multio_start_server(conf_ctx) @@ -246,7 +246,7 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) #include "../associate_part_ass.h" #include "../associate_mesh_ass.h" - cerr = md%new() + cerr = md%new(mio_handle) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: ngrid, md%new() failed: ', multio_error_string(cerr)) END IF @@ -288,7 +288,7 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) END IF !declare grid at elements - cerr = md%new() + cerr = md%new(mio_handle) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: egrid, md%new() failed: ', multio_error_string(cerr)) END IF @@ -336,7 +336,7 @@ SUBROUTINE iom_send_fesom_data(data) INTEGER :: cerr TYPE(multio_metadata) :: md - cerr = md%new() + cerr = md%new(mio_handle) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%new() failed: ', multio_error_string(cerr)) END IF diff --git a/src/ifs_interface/mpp_io.F90 b/src/ifs_interface/mpp_io.F90 index eda8feae7..c9d86e627 100644 --- a/src/ifs_interface/mpp_io.F90 +++ b/src/ifs_interface/mpp_io.F90 @@ -113,15 +113,49 @@ SUBROUTINE mpp_io_init( iicomm, lio, irequired, iprovided, lmpi1 ) END SUBROUTINE mpp_io_init SUBROUTINE mpp_io_init_2( iicomm ) + INCLUDE "mpif.h" INTEGER, INTENT(INOUT) :: iicomm INTEGER :: icode, ierr, icolor, iicommx, iicommm, iicommo INTEGER :: ji,inum LOGICAL :: lcompp - INCLUDE "mpif.h" +#if defined(__MULTIO) && !defined(__ifsinterface) && !defined(__oasis) + ! Construct multio server communicator in FESOM standalone + INTEGER :: commsize, myrank + character(len=255) :: oce_npes_str, mio_npes_str + integer :: oce_npes_int, mio_npes_int, oce_status, mio_status + ! fesom standalone with MULTIO + ! both below envs are not needed as one can be deduced from other by subtracting + ! from global total pes + CALL get_environment_variable('OCE_NPES', oce_npes_str, status=oce_status) + CALL get_environment_variable('MIO_NPES', mio_npes_str, status=mio_status) + CALL MPI_Comm_Size(MPI_COMM_WORLD, commsize, ierr) + CALL MPI_Comm_Rank(MPI_COMM_WORLD, myrank, ierr) + if ((oce_status/=0) .or. (mio_status/=0)) then + if (oce_status/=0) then + if (myrank==0) write(*,*) '$OCE_NPES variable is not set!' + end if + if (mio_status/=0) then + if (myrank==0) write(*,*) '$MIO_NPES variable is not set!' + end if + !call par_ex(MPI_COMM_WORLD, myrank, abort=.true.) ! TODO:doesn't work see par_ex + call mpi_abort(mpi_comm_world) + stop + end if + read(oce_npes_str,*,iostat=oce_status) oce_npes_int + read(mio_npes_str,*,iostat=mio_status) mio_npes_int + if (myrank==0) write(*,*) 'Total number of processes: ', commsize + if (myrank==0) write(*,*) 'FESOM runs on ', oce_npes_int, ' processes' + if (myrank==0) write(*,*) 'MULTIO Server runs on ', mio_npes_int, ' processes' + + ! note: in case of ifsinterface lmultioserver and lioserver are init in mpp_init + if (myrank > oce_npes_int-1) then + lmultioserver = .true. + lioserver = .true. + ENDIF +#endif - ! Construct multio server communicator IF (lmultioserver.OR..NOT.lioserver) THEN icolor=12 @@ -158,15 +192,23 @@ SUBROUTINE mpp_io_init_2( iicomm ) CALL mpi_comm_size( iicommo, mppcompsize, ierr ) IF (.NOT.lioserver) THEN - CALL iom_initialize( "for_xios_mpi_id", return_comm=iicommm, global_comm = pcommworldmultio ) ! nemo local communicator given by xios + CALL iom_initialize( "feosom client", return_comm=iicommm, global_comm = MPI_COMM_WORLD ) ! nemo local communicator given by xios + !iicomm = iicommm ELSE ! For io-server tasks start an run the right server - CALL iom_init_server( server_comm = pcommworldmultio ) + CALL iom_init_server( server_comm = iicomm ) ENDIF ! Return to the model with iicomm being compute only tasks iicomm = iicommo - +#if defined(__MULTIO) && !defined(__ifsinterface) && !defined(__oasis) + IF (lioserver) THEN + ! TODO: unless we fix partit and segregate compute communicators from IO + ! communicators it is hard to elegantly close either FESOM or MULTIO + ! currently force stopping multio after all connections from clients are closed + stop "Exiting MIO server" + END IF +#endif END SUBROUTINE mpp_io_init_2 SUBROUTINE mpp_stop @@ -176,7 +218,9 @@ SUBROUTINE mpp_stop call iom_finalize() ENDIF +#ifdef __ifsinterface CALL mpi_finalize( ierr ) +#endif END SUBROUTINE mpp_stop #endif END MODULE mpp_io From 3166498ff979e7307949bf2998c4aeb645d84d5c Mon Sep 17 00:00:00 2001 From: suvarchal Date: Wed, 20 Sep 2023 13:37:08 +0200 Subject: [PATCH 21/71] channge bad _MULTIO define and redundant reading of MIO_PES --- src/fesom_module.F90 | 2 +- src/gen_modules_partitioning.F90 | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index 65e3cd400..fed4f0698 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -454,7 +454,7 @@ subroutine fesom_finalize() call par_ex(f%partit%MPI_COMM_FESOM, f%partit%mype) #endif -#ifdef __MULTIO && (ifndef __ifsinterface || ifndef __oasis) +#if defined(__MULTIO) && !defined(__ifsinterface) && !defined(__oasis) call mpp_stop #endif if(f%fesom_did_mpi_init) call par_ex(f%partit%MPI_COMM_FESOM, f%partit%mype) ! finalize MPI before FESOM prints its stats block, otherwise there is sometimes output from other processes from an earlier time in the programm AFTER the starts block (with parastationMPI) diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90 index c296a0f85..0366d9f4c 100644 --- a/src/gen_modules_partitioning.F90 +++ b/src/gen_modules_partitioning.F90 @@ -58,18 +58,10 @@ subroutine par_init(partit) ! initializes MPI ! names and routine names as in src/ifs_interface, that is not elegant. #else #ifdef __MULTIO - ! fesom standalone with MULTIO - character(len=255) :: oce_npes_str, mio_npes_str - integer :: oce_npes_int, mio_npes_int, oce_status, mio_status CALL MPI_Comm_Size(MPI_COMM_WORLD, partit%npes, i) CALL MPI_Comm_Rank(MPI_COMM_WORLD, partit%mype, i) partit%MPI_COMM_FESOM=MPI_COMM_WORLD partit%MPI_COMM_WORLD=MPI_COMM_WORLD - !TODO: dont need both env variables can do with just one and subtract from npes - CALL get_environment_variable('OCE_NPES', oce_npes_str, status=oce_status) - CALL get_environment_variable('MIO_NPES', mio_npes_str, status=mio_status) - read(oce_npes_str,*,iostat=oce_status) oce_npes_int - read(mio_npes_str,*,iostat=mio_status) mio_npes_int call mpp_io_init_2(partit%MPI_COMM_FESOM) #else partit%MPI_COMM_FESOM=MPI_COMM_WORLD ! use global comm if not coupled (e.g. no __oasis or __ifsinterface or IO server) From 0b933277a1a0186ed4d5e2c9eb3dc392be80aaa1 Mon Sep 17 00:00:00 2001 From: suvarchal Date: Thu, 21 Sep 2023 00:48:54 +0200 Subject: [PATCH 22/71] parms doesn't necessarily need blas, if cmake doesn't find it then ignore blas --- lib/parms/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/parms/CMakeLists.txt b/lib/parms/CMakeLists.txt index 3cb7b48cf..0fedf9d16 100644 --- a/lib/parms/CMakeLists.txt +++ b/lib/parms/CMakeLists.txt @@ -14,12 +14,19 @@ if(${BUILD_FESOM_AS_LIBRARY}) else() add_library(${PROJECT_NAME} ${all_sources}) endif() + +if(${BLAS_FOUND}) target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) +target_link_libraries(${PROJECT_NAME} INTERFACE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) +else() +target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8) +target_link_libraries(${PROJECT_NAME} INTERFACE $ENV{UBUNTU_BLAS_LIBRARY}) +endif() + target_include_directories(${PROJECT_NAME} PRIVATE ${src_home}/src/../include ${src_home}/src/include INTERFACE ${src_home}/src/../include ) -target_link_libraries(${PROJECT_NAME} INTERFACE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel") target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) From 80db75a367775e32d41041bb167ba5fc641b2b03 Mon Sep 17 00:00:00 2001 From: suvarchal Date: Thu, 21 Sep 2023 01:13:32 +0200 Subject: [PATCH 23/71] skip blas when not found and use standard discovery for BLAS as default --- lib/parms/CMakeLists.txt | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/parms/CMakeLists.txt b/lib/parms/CMakeLists.txt index 0fedf9d16..7f85d5eef 100644 --- a/lib/parms/CMakeLists.txt +++ b/lib/parms/CMakeLists.txt @@ -6,7 +6,10 @@ project(parms C) set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt file(GLOB all_sources ${src_home}/src/*.c ${src_home}/src/DDPQ/*.c) -include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/FindBLAS.cmake") +find_package(BLAS) # standard way to find blas +if( NOT BLAS_FOUND) # try older way to find blas + include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/FindBLAS.cmake") +endif() # create our library (set its name to name of this project) if(${BUILD_FESOM_AS_LIBRARY}) @@ -16,11 +19,13 @@ else() endif() if(${BLAS_FOUND}) -target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) -target_link_libraries(${PROJECT_NAME} INTERFACE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) + message("BLAS FOUND ---------------------------") + target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) + target_link_libraries(${PROJECT_NAME} INTERFACE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) else() -target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8) -target_link_libraries(${PROJECT_NAME} INTERFACE $ENV{UBUNTU_BLAS_LIBRARY}) + message("BLAS NOT FOUND ***********************") + target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8) + target_link_libraries(${PROJECT_NAME} INTERFACE $ENV{UBUNTU_BLAS_LIBRARY}) endif() target_include_directories(${PROJECT_NAME} @@ -28,7 +33,8 @@ target_include_directories(${PROJECT_NAME} INTERFACE ${src_home}/src/../include ) if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel") - target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) + target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) + if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) endif() From d0bcad86d69d76b9e323d3e5da3203b517c2ce9d Mon Sep 17 00:00:00 2001 From: Fernanda Matos Date: Wed, 27 Sep 2023 14:17:31 +0200 Subject: [PATCH 24/71] updating documentation for fesom standalone and job_levante to allow for it on levante --- docs/getting_started/getting_started.rst | 133 ++++++++++++++++++----- work/job_levante | 4 + 2 files changed, 109 insertions(+), 28 deletions(-) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index 715dc5f8c..97a19d09d 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -8,25 +8,19 @@ This chapter describes several ways of getting started with FESOM2. First we sho TL;DR version for supported HPC systems ======================================= -Supported systems are: generic ``ubuntu``, ``ollie`` at AWI, ``mistral`` at DKRZ, ``JURECA`` at JSC, ``HLRN``, ``Hazel Hen``, ``MareNostrum 4`` at BSC. During configuration the system will be recognised and apropriate environment variables and compiler options should be used. +Supported systems are: generic ``ubuntu``, ``albedo`` at AWI, ``levante`` at DKRZ, ``JURECA`` at JSC, ``HLRN``, ``Hazel Hen``, ``MareNostrum 4`` at BSC. During configuration the system will be recognised and apropriate environment variables and compiler options should be used. :: git clone https://github.com/FESOM/fesom2.git cd fesom2 - bash -l configure.sh -Create file fesom.clock in the output directory with the following content (if you plan to run with COREII foring): - -:: +If one intends to run simulations on ``albedo`` or ``levante``, one first needs to change to the ``refactoring`` branch (note that this part is only needed while the ``refactoring`` is not yet merged into ``master``: + + git checkout refactoring - 0 1 1948 - 0 1 1948 +After confirming that the right FESOM2 branch is being used, compile the model with: -after that one has to adjust the run script for the target sustem and run it: -:: - - cd work - sbatch job_ollie + bash -l ./configure.sh Detailed steps of compiling and runing the code =============================================== @@ -51,6 +45,24 @@ The repository contains model code and two additional libraries: `Metis` (domain Change to the `fesom2` folder and execute: +:: + + cd fesom2 + +If one intends to run simulations on ``albedo`` or ``levante``, one first needs to change to the ``refactoring`` branch (note that this part is only needed while the ``refactoring`` branch is not yet merged into ``master``. + +:: + + git checkout refactoring + +As a good practice, if one wants to make modifications to the source code or any of the files, it is advisable to create a branch from refactoring: + +:: + + git checkout -b refactoring + +After confirming that the right FESOM2 branch is being used, compile the model with: + :: bash -l ./configure.sh @@ -94,20 +106,34 @@ The ``input`` folder contains files with initial conditions (``phc3.0``) and atm Preparing the run ------------------ -You have to do several basic things in order to prepare the run. First, create a directory where results will be stored. Usually, it is created in the model root directory: +You have to do several basic things in order to prepare the run. + +First, be aware of the files you need to modify according to your run configurations. Normally, those are: + +- ``namelist.config``: inside of the ``config`` folder. In this file you can set several configurations, such as the path to your mesh, climatology and results, as well as run length, units and start year of your run. + +- ``namelist.forcing``: inside of the ``config`` folder. In this file you can set the path to your forcing files. + +- ``job_``: inside of the ``work`` folder. In this file you can set other important configurations, such as the time, tasks and tasks per node you allocate to your run. + +The exact changes necessary to those file are indicated later in this documentation. Before doing so, create a directory to store your output. Usually, it is created in the model root directory: :: mkdir results -You might make a link to some other directory located on the part of the system where you have a lot of storage. In the results directory, you have to create ``fesom.clock`` file (NOTE, if you change ``runid`` in ``namelist.config`` to something like ``runid=mygreatrun``, the file will be named ``mygreatrun.clock``). Inside the file you have to put two identical lines: +You might make a link to some other directory located on the part of the system where you have a lot of storage. + +In your results directory, create a file named ``fesom.clock`` (NOTE: if you change ``runid`` in ``namelist.config`` to something like ``runid=mygreatrun``, the file will be named ``mygreatrun.clock``). + +Inside the file you have to put two identical lines: :: 0 1 1958 0 1 1958 -This is initial date of the model run, or the time of the `cold start` of your model. More detailed explanation of the clock file will be given in the `The clock file`_ section. +This is initial date of the model run, or the time of the `cold start` of your model. In case you want to start your run with a specific forcing from a specific year, substitute 1958 to the desired year. More detailed explanation of the clock file will be given in the `The clock file`_ section. The next step is to make some changes in the model configuration. All runtime options can be set in the namelists that are located in the config directory: @@ -115,14 +141,40 @@ The next step is to make some changes in the model configuration. All runtime op cd ../config/ -There are several configuration files, but we are only interested in the ``namelist.config`` for now. The options that you might want to change for your first FESOM2 run are: +As mentioned before, in this directory, you will normally have to change two files: ``namelist.config`` and ``namelist.forcing``. Both of these files ask for paths to initial conditions. Normally, these paths can be found under ``./setups/paths.yml``. + +Changing namelist.config +======================== + +In ``namelist.config``, the options that you might want to change for your first FESOM2 run are: -- ``run_length`` length of the model run in run_length_unit (see below). -- ``run_length_unit`` units of the run_length. Can be ``y`` (year), ``m`` (month), ``d`` (days), ``s`` (model steps). -- ``MeshPath`` - path to the mesh you would like to use (e.g. ``/youdir/FESOM2_one_year_input/mesh/pi/``, slash at the end is important!) -- ``ClimateDataPath`` - path to the folder with the file with model temperature and salinity initial conditions (e.g. ``/youdir/FESOM2_one_year_input/input/phc3.0/``). The name of the file with initial conditions is defined in `namelist.oce`, but during first runs you probably don't want to change it. +- ``run_length``: length of the model run in run_length_unit (see below). -More detailed explination of options in the ``namelist.config`` is in the section :ref:`chap_general_configuration`. +- ``run_length_unit``: units of the run_length. Can be ``y`` (year), ``m`` (month), ``d`` (days), ``s`` (model steps). + +.. note:: you will need to adjust the run time to the length of your run. So, if you set ``run_length`` to 10 and ``run_length_unit`` to ``y``, for example, the run time needs to be enough for a 10-year run at once. For 4 nodes, for example, that would take about four hours to run. +- ``yearnew``: define the same as the year in your ``fesom.clock``; + +- ``MeshPath``: path to the mesh you would like to use (e.g. ``/youdir/FESOM2_one_year_input/mesh/pi/``, slash at the end is important!); + +- ``ClimateDataPath``: path to the folder with the file with model temperature and salinity initial conditions (e.g. ``/youdir/FESOM2_one_year_input/input/phc3.0/``). The name of the file with initial conditions is defined in `namelist.oce`, but during first runs you probably don't want to change it; + +- ``ResultPath``: path to your results folder. The output of the model will be stored there. + +More detailed explanation of options in the ``namelist.config`` is in the section :ref:`chap_general_configuration`. + +Changing namelist.forcing +========================= + +In ``namelist.forcing``, the options you need to change for your first FESOM2 run depends on the forcing you decide to use to initialize your experiment. Please note that the year you initialize your experiment with needs to be included in the forcing data files. + +In section ``&nam_sbc``, change the path of all the files to the path to the forcing you have chosen. For example, if you want to initialize your experiment with JRA55 forcing on ``levante``, the path to each fiel will be: + +:: + +'/pool/data/AWICM/FESOM2/FORCING/JRA55-do-v1.4.0/' + +More detailed explanation of options in the ``namelist.forcing`` is in the section :ref:`chap_forcing_configuration`. Running the model ----------------- @@ -130,18 +182,28 @@ Running the model Change to the ``work`` directory. You should find several batch scripts that are used to submit model jobs to different HPC machines. The scripts also link ``fesom.x`` executable to the ``work`` directory and copy namelists with configurations from config folder. .. note:: - Model executable, namelists and job script have to be located in the same directory (usually ``work``). + Model executable, namelists and job script will be located in the same directory (usually ``work``). -If you are working on AWI's ``ollie`` supercomputer, you have to use ``job_ollie``, in other case use the job script for your specific platform, or try to modify one of the existing ones. +If you are working on AWI's ``albedo`` supercomputer, you have to use ``job_albedo``, in other case use the job script for your specific platform, or try to modify one of the existing ones. -.. note:: - One thing you might need to adjust in the job files is the number of cores, you would like to run the model on. For example, for SLURM it will be adjusting ``#SBATCH --ntasks=288`` value, and for simple ``mpirun`` command, that we have for ``job_ubuntu`` it will be argument for the ``-n`` option. It is necessary, that your mesh has the corresponding partitioning (``dist_xx`` folder, where ``xx`` is the number of cores). +In the job file, the changes are done based on the HPC you are using. For ``levante``, you should adapt for example: -On ``ollie`` the submission of your job is done by executing the following command: +- ``#SBATCH --job-name``: name of your experiment; e.g. myexperiment_001; + +- ``#SBATCH --ntasks-per-node``: number of cores per node. This number has to be divisible by the number of tasks. If you choose the ``ntasks``/4, for example, you will run your experiment with 4 nodes; + +- ``#SBATCH --ntasks``: number of cores. This number has to be the same of your desired mesh partitioning. It is the ``xx`` number in your ``dist_xx`` mesh folder; + +- ``#SBATCH --time``: be generous with your run time, in case you are running a longer simulation and the job is not being resubmmited after each time step; + +- ``#SBATCH -A ``: define your project account. + + +On ``levante`` the submission of your job is done by executing the following command: :: - sbatch job_ollie + sbatch job_levante The job is then submitted. In order to check the status of your job on ollie you can execute: @@ -149,7 +211,22 @@ The job is then submitted. In order to check the status of your job on ollie you squeue -u yourusername -Results of the model run should appear in the ``results`` directory that you have specified in the ``namelist.config``. After the run is finished the ``fesom.clock`` file (or if you change your runid, ``runid.clock``) will be updated with information about the time of your run's end, that allows running the next time portion of the model experiment by just resubmitting the job with ``sbatch job_ollie``. +The output of the model run should appear in the ``results`` directory that you have specified in the ``namelist.config``. After the run is finished the ``fesom.clock`` file (or if you change your runid, ``runid.clock``) will be updated with information about the time of your run's end, that allows running the next time portion of the model experiment by just resubmitting the job with ``sbatch job_ollie``. + +Some files will also be stored on the work folder. Those are + +- A file containing information about errors during job preparation and submission, usually containing ``err.out`` in its name; + +- A file containing information about the job itself, such as duration, folders, etc, usually contining ``out.out`` in its name; + +- A file containing information about the simulation, usually called ``fesom2-0.out``; + +- A binary file ``fesom.x`` specific to that simulation; + +- A copy of the namelists used to define the configurations of your run. + +In case your simulation crashes, usually the job error file or ``fesom2-0.out`` contain valuable information to either fix the issue causing the crash or to give the developers an idea of what can be done to help you. + Other things you need to know earlier on ======================================== diff --git a/work/job_levante b/work/job_levante index 721dc361d..824e233a7 100755 --- a/work/job_levante +++ b/work/job_levante @@ -28,6 +28,10 @@ cp -n ../config/namelist.forcing . cp -n ../config/namelist.oce . cp -n ../config/namelist.ice . cp -n ../config/namelist.icepack . +cp -n ../config/namelist.tra . +cp -n ../config/namelist.io . +cp -n ../config/namelist.cvmix . +cp -n ../config/namelist.dyn . date srun -l fesom.x > "fesom2.0.out" From 1ca6ef18a853b1ac1d3fc247ed37ef918d0ded69 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 28 Sep 2023 16:29:27 +0200 Subject: [PATCH 25/71] added runoff mapper --- src/gen_modules_read_NetCDF.F90 | 243 +++++++++++++++++++++++++++++++- src/gen_surface_forcing.F90 | 3 +- 2 files changed, 242 insertions(+), 4 deletions(-) diff --git a/src/gen_modules_read_NetCDF.F90 b/src/gen_modules_read_NetCDF.F90 index 8e459d1db..4e11e7c5a 100755 --- a/src/gen_modules_read_NetCDF.F90 +++ b/src/gen_modules_read_NetCDF.F90 @@ -205,7 +205,7 @@ subroutine read_surf_hydrography_NetCDF(file, vari, itime, model_2Darray, partit integer :: i, j, n, num integer :: itime, latlen, lonlen integer :: status, ncid, varid - integer :: lonid, latid + integer :: lonid, latid, drain_num integer :: istart(4), icount(4) real(real64) :: x, y, miss real(real64), allocatable :: lon(:), lat(:) @@ -363,7 +363,244 @@ subroutine read_2ddata_on_grid_NetCDF(file, vari, itime, model_2Darray, partit, call MPI_BCast(ncdata, nod2D, MPI_DOUBLE_PRECISION, 0, MPI_COMM_FESOM, ierror) model_2Darray=ncdata(myList_nod2D) end subroutine read_2ddata_on_grid_NetCDF - -end module g_read_other_NetCDF +subroutine read_runoff_mapper(file, vari, R, partit, mesh) + ! 1. Read arrival points from the runoff mapper + ! 2. Create conservative remapping A*X=runoff: + ! A=remapping operator; X=runoff into drainage basins (in Sv); runoff= runoff im [m/s] to be put into the ocean +! use, intrinsic :: ISO_FORTRAN_ENV + use g_config + use o_param + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE g_forcing_arrays, only: runoff + use g_support + implicit none + +#include "netcdf.inc" + character(*), intent(in) :: file + character(*), intent(in) :: vari + real(real64), intent(in) :: R + type(t_mesh), intent(in), target :: mesh + type(t_partit), intent(inout), target :: partit + integer :: i, j, n, num, cnt, number_arrival_points, offset + real(real64) :: entry, dist, delta_lat, delta_lon + integer :: itime, latlen, lonlen + integer :: status, ncid, varid + integer :: lonid, latid, drain_num + integer :: istart(2), icount(2) + real(real64), allocatable :: lon(:), lat(:) + real(real64) :: W + integer, allocatable :: ncdata(:,:) + real(real64), allocatable :: lon_sparse(:), lat_sparse(:), dist_min(:), dist_min_glo(:) + real(real64), allocatable :: arrival_area(:) + integer, allocatable :: data_sparse(:), dist_ind(:) + logical :: check_dummy, do_onvert + integer :: ierror ! return error code + type(sparse_matrix) :: RUNOFF_MAPPER + +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + + if (mype==0) write(*,*) 'building RUNOFF MAPPER with radius of smoothing= ', R*1.e-3, ' km' + if (mype==0) then + ! open file + status=nf_open(trim(file), nf_nowrite, ncid) + end if + + call MPI_BCast(status, 1, MPI_INTEGER, 0, MPI_COMM_FESOM, ierror) + if (status.ne.nf_noerr)then + print*,'ERROR: CANNOT READ 2D netCDF FILE CORRECTLY !!!!!' + print*,'Error in opening netcdf file '//file + call par_ex(partit%MPI_COMM_FESOM, partit%mype) + stop + endif + + if (mype==0) then + ! lat + status=nf_inq_dimid(ncid, 'lat', latid) + status=nf_inq_dimlen(ncid, latid, latlen) + ! lon + status=nf_inq_dimid(ncid, 'lon', lonid) + status=nf_inq_dimlen(ncid, lonid, lonlen) + end if + call MPI_BCast(latlen, 1, MPI_INTEGER, 0, MPI_COMM_FESOM, ierror) + call MPI_BCast(lonlen, 1, MPI_INTEGER, 0, MPI_COMM_FESOM, ierror) + + ! lat + if (mype==0) then + allocate(lat(latlen)) + status=nf_inq_varid(ncid, 'lat', varid) + status=nf_get_vara_double(ncid,varid,1,latlen,lat) + end if + + ! lon + if (mype==0) then + allocate(lon(lonlen)) + status=nf_inq_varid(ncid, 'lon', varid) + status=nf_get_vara_double(ncid,varid,1,lonlen,lon) + ! make sure range 0. - 360. + do n=1,lonlen + if (lon(n)<0.0_WP) then + lon(n)=lon(n)+360._WP + end if + end do + end if + if (mype==0) then + allocate(ncdata(lonlen,latlen)) + ncdata = 0.0_WP + ! data + status=nf_inq_varid(ncid, trim(vari), varid) + istart = (/1,1/) + icount= (/lonlen,latlen/) + status=nf_get_vara_int(ncid,varid,istart,icount,ncdata) + ! close file + status=nf_close(ncid) + number_arrival_points=0 + do i=1, lonlen + do j=1, latlen + if (ncdata(i,j)>0) then + number_arrival_points=number_arrival_points+1 + end if + end do + end do + end if + + call MPI_BCast(number_arrival_points, 1, MPI_INTEGER, 0, MPI_COMM_FESOM, ierror) + allocate(lon_sparse(number_arrival_points), lat_sparse(number_arrival_points)) + allocate(dist_min(number_arrival_points), dist_min_glo(number_arrival_points), dist_ind(number_arrival_points)) + allocate(data_sparse(number_arrival_points)) + + if (mype==0) then + cnt=1 + do i=1, lonlen + do j=1, latlen + if (ncdata(i,j)>0) then + lon_sparse(cnt)=lon(i) + lat_sparse(cnt)=lat(j) + data_sparse(cnt)=ncdata(i,j) + cnt=cnt+1 + end if + end do + end do + deallocate(ncdata, lon, lat) + end if + call MPI_BCast(lon_sparse, number_arrival_points, MPI_DOUBLE_PRECISION, 0, MPI_COMM_FESOM, ierror) + call MPI_BCast(lat_sparse, number_arrival_points, MPI_DOUBLE_PRECISION, 0, MPI_COMM_FESOM, ierror) + call MPI_BCast(data_sparse, number_arrival_points, MPI_INTEGER, 0, MPI_COMM_FESOM, ierror) + drain_num=maxval(data_sparse) + lon_sparse=lon_sparse-360.0_WP + lon_sparse=lon_sparse*rad + lat_sparse=lat_sparse*rad + + do n=1, number_arrival_points + do i=1, myDim_nod2d+eDim_nod2D + delta_lon=abs(geo_coord_nod2D(1,i)-lon_sparse(n)) + if (delta_lon > cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length + delta_lat=(geo_coord_nod2D(2,i)-lat_sparse(n)) + entry = sin(delta_lat/2.0)**2 + cos(geo_coord_nod2D(2,i)) * cos(lat_sparse(n)) * sin(delta_lon/2.0)**2 + dist=2.0 * atan2(sqrt(entry), sqrt(1.0 - entry))*r_earth + if (i==1) then + dist_min(n)=dist + dist_ind(n)=1 + end if + if (dist cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length + delta_lat=(geo_coord_nod2D(2,n)-lat_sparse(i)) + entry = sin(delta_lat/2.0)**2 + cos(geo_coord_nod2D(2,n)) * cos(lat_sparse(i)) * sin(delta_lon/2.0)**2 + dist=2.0 * atan2(sqrt(entry), sqrt(1.0 - entry))*r_earth + if (dist<=R) cnt=cnt+1 + END DO + RUNOFF_MAPPER%rowptr(n+1)=RUNOFF_MAPPER%rowptr(n)+max(cnt,1) + END DO + + ! write(*,*) 'rowptr check:', mype, RUNOFF_MAPPER%rowptr(RUNOFF_MAPPER%dim+1) + + ALLOCATE(RUNOFF_MAPPER%colind(RUNOFF_MAPPER%rowptr(RUNOFF_MAPPER%dim+1)-1)) + ALLOCATE(RUNOFF_MAPPER%values(RUNOFF_MAPPER%rowptr(RUNOFF_MAPPER%dim+1)-1)) + ALLOCATE(arrival_area(drain_num)) + status=0 + arrival_area=0.0_WP ! will be used further to normalize the total flux + DO n=1, myDim_nod2d+eDim_nod2D + offset=RUNOFF_MAPPER%rowptr(n) + cnt=0 + W=areasvol(ulevels_nod2D(n),n) + DO i=1, number_arrival_points + j=data_sparse(i) + delta_lon=abs(geo_coord_nod2D(1,n)-lon_sparse(i)) + if (delta_lon > cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length + delta_lat=(geo_coord_nod2D(2,n)-lat_sparse(i)) + entry = sin(delta_lat/2.0)**2 + cos(geo_coord_nod2D(2,n)) * cos(lat_sparse(i)) * sin(delta_lon/2.0)**2 + dist=2.0 * atan2(sqrt(entry), sqrt(1.0 - entry))*r_earth + entry=(1.0-dist/R) + if (entry > 0.) then + RUNOFF_MAPPER%values(offset+cnt)=entry + RUNOFF_MAPPER%colind(offset+cnt)=j + if (n<=myDim_nod2d) then + arrival_area(j)=arrival_area(j)+entry*W + end if + cnt=cnt+1 + end if + END DO + if (cnt==0) then + RUNOFF_MAPPER%values(offset)=0.0_WP + RUNOFF_MAPPER%colind(offset)=1 + end if + END DO + +! write(*,*) 'status', mype, status + call MPI_AllREDUCE(MPI_IN_PLACE , arrival_area, drain_num, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) + DO i=1, drain_num + if (mype==0) write(*,*) 'arrival_area=', i, arrival_area(i) + where (RUNOFF_MAPPER%colind==i) + RUNOFF_MAPPER%values=RUNOFF_MAPPER%values/arrival_area(i) + end where + END DO + + deallocate(lon_sparse, lat_sparse, dist_min, dist_min_glo) + deallocate(arrival_area) + deallocate(data_sparse, dist_ind) + +do n=1, myDim_nod2D+eDim_nod2D + i=RUNOFF_MAPPER%rowptr(n) + j=RUNOFF_MAPPER%rowptr(n+1)-1 + runoff(n)=sum(RUNOFF_MAPPER%values(i:j)) +end do + +call integrate_nod(runoff, entry, partit, mesh) + +if (mype==0) write(*,*) 'RUNOFF MAPPER check (input of 1Sv from each basin results in runoff of):', entry, ' Sv' +runoff=runoff*1.e-2 +end subroutine read_runoff_mapper +end module g_read_other_NetCDF diff --git a/src/gen_surface_forcing.F90 b/src/gen_surface_forcing.F90 index 967f217f8..2ffb26c43 100644 --- a/src/gen_surface_forcing.F90 +++ b/src/gen_surface_forcing.F90 @@ -44,7 +44,7 @@ MODULE g_sbf USE g_config, only: dummy, ClimateDataPath, dt USE g_clock, only: timeold, timenew, dayold, daynew, yearold, yearnew, cyearnew USE g_forcing_arrays, only: runoff, chl - USE g_read_other_NetCDF, only: read_other_NetCDF, read_2ddata_on_grid_netcdf + USE g_read_other_NetCDF, only: read_other_NetCDF, read_2ddata_on_grid_netcdf, read_runoff_mapper IMPLICIT NONE include 'netcdf.inc' @@ -1100,6 +1100,7 @@ SUBROUTINE sbc_ini(partit, mesh) if (mype==0) write(*,*) "DONE: Ocean forcing inizialization." if (mype==0) write(*,*) 'Parts of forcing data (only constant in time fields) are read' +! call read_runoff_mapper("/p/project/chhb19/streffing1/input/runoff-mapper/runoff_maps_D3.nc", "arrival_point_id", 1.0_WP, partit, mesh) END SUBROUTINE sbc_ini SUBROUTINE sbc_do(partit, mesh) From a7a2504d1bef92b97b79e895c819c71cab2ed274 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 29 Sep 2023 11:00:00 +0200 Subject: [PATCH 26/71] improved realization of the RUNOFF MAPPER. --- src/gen_modules_read_NetCDF.F90 | 122 +++++++++++++++++++------------- 1 file changed, 71 insertions(+), 51 deletions(-) diff --git a/src/gen_modules_read_NetCDF.F90 b/src/gen_modules_read_NetCDF.F90 index 4e11e7c5a..8118fa78a 100755 --- a/src/gen_modules_read_NetCDF.F90 +++ b/src/gen_modules_read_NetCDF.F90 @@ -368,7 +368,6 @@ subroutine read_runoff_mapper(file, vari, R, partit, mesh) ! 1. Read arrival points from the runoff mapper ! 2. Create conservative remapping A*X=runoff: ! A=remapping operator; X=runoff into drainage basins (in Sv); runoff= runoff im [m/s] to be put into the ocean -! use, intrinsic :: ISO_FORTRAN_ENV use g_config use o_param USE MOD_MESH @@ -381,22 +380,20 @@ subroutine read_runoff_mapper(file, vari, R, partit, mesh) #include "netcdf.inc" character(*), intent(in) :: file character(*), intent(in) :: vari - real(real64), intent(in) :: R + real(kind=WP), intent(in):: R type(t_mesh), intent(in), target :: mesh type(t_partit), intent(inout), target :: partit integer :: i, j, n, num, cnt, number_arrival_points, offset - real(real64) :: entry, dist, delta_lat, delta_lon + real(kind=WP) :: dist, W integer :: itime, latlen, lonlen integer :: status, ncid, varid integer :: lonid, latid, drain_num integer :: istart(2), icount(2) - real(real64), allocatable :: lon(:), lat(:) - real(real64) :: W + real(kind=WP), allocatable :: lon(:), lat(:) integer, allocatable :: ncdata(:,:) - real(real64), allocatable :: lon_sparse(:), lat_sparse(:), dist_min(:), dist_min_glo(:) - real(real64), allocatable :: arrival_area(:) + real(kind=WP), allocatable :: lon_sparse(:), lat_sparse(:), dist_min(:), dist_min_glo(:) + real(kind=WP), allocatable :: arrival_area(:) integer, allocatable :: data_sparse(:), dist_ind(:) - logical :: check_dummy, do_onvert integer :: ierror ! return error code type(sparse_matrix) :: RUNOFF_MAPPER @@ -493,39 +490,53 @@ subroutine read_runoff_mapper(file, vari, R, partit, mesh) call MPI_BCast(lat_sparse, number_arrival_points, MPI_DOUBLE_PRECISION, 0, MPI_COMM_FESOM, ierror) call MPI_BCast(data_sparse, number_arrival_points, MPI_INTEGER, 0, MPI_COMM_FESOM, ierror) drain_num=maxval(data_sparse) + ALLOCATE(arrival_area(drain_num)) + arrival_area=0.0_WP ! will be used further to normalize the total flux lon_sparse=lon_sparse-360.0_WP lon_sparse=lon_sparse*rad lat_sparse=lat_sparse*rad do n=1, number_arrival_points - do i=1, myDim_nod2d+eDim_nod2D - delta_lon=abs(geo_coord_nod2D(1,i)-lon_sparse(n)) - if (delta_lon > cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length - delta_lat=(geo_coord_nod2D(2,i)-lat_sparse(n)) - entry = sin(delta_lat/2.0)**2 + cos(geo_coord_nod2D(2,i)) * cos(lat_sparse(n)) * sin(delta_lon/2.0)**2 - dist=2.0 * atan2(sqrt(entry), sqrt(1.0 - entry))*r_earth + do i=1, myDim_nod2d + dist=distance_on_sphere(lon_sparse(n), lat_sparse(n), geo_coord_nod2D(1,i), geo_coord_nod2D(2,i)) if (i==1) then dist_min(n)=dist dist_ind(n)=1 end if - if (dist cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length - delta_lat=(geo_coord_nod2D(2,n)-lat_sparse(i)) - entry = sin(delta_lat/2.0)**2 + cos(geo_coord_nod2D(2,n)) * cos(lat_sparse(i)) * sin(delta_lon/2.0)**2 - dist=2.0 * atan2(sqrt(entry), sqrt(1.0 - entry))*r_earth - if (dist<=R) cnt=cnt+1 + dist=distance_on_sphere(lon_sparse(i), lat_sparse(i), geo_coord_nod2D(1,n), geo_coord_nod2D(2,n)) + if (dist < R) cnt=cnt+1 END DO RUNOFF_MAPPER%rowptr(n+1)=RUNOFF_MAPPER%rowptr(n)+max(cnt,1) END DO - ! write(*,*) 'rowptr check:', mype, RUNOFF_MAPPER%rowptr(RUNOFF_MAPPER%dim+1) - ALLOCATE(RUNOFF_MAPPER%colind(RUNOFF_MAPPER%rowptr(RUNOFF_MAPPER%dim+1)-1)) ALLOCATE(RUNOFF_MAPPER%values(RUNOFF_MAPPER%rowptr(RUNOFF_MAPPER%dim+1)-1)) - ALLOCATE(arrival_area(drain_num)) - status=0 - arrival_area=0.0_WP ! will be used further to normalize the total flux DO n=1, myDim_nod2d+eDim_nod2D offset=RUNOFF_MAPPER%rowptr(n) cnt=0 W=areasvol(ulevels_nod2D(n),n) DO i=1, number_arrival_points j=data_sparse(i) - delta_lon=abs(geo_coord_nod2D(1,n)-lon_sparse(i)) - if (delta_lon > cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length - delta_lat=(geo_coord_nod2D(2,n)-lat_sparse(i)) - entry = sin(delta_lat/2.0)**2 + cos(geo_coord_nod2D(2,n)) * cos(lat_sparse(i)) * sin(delta_lon/2.0)**2 - dist=2.0 * atan2(sqrt(entry), sqrt(1.0 - entry))*r_earth - entry=(1.0-dist/R) - if (entry > 0.) then - RUNOFF_MAPPER%values(offset+cnt)=entry + if ((j<0) .OR. (j>drain_num)) then + if (mype==0) then + write(*,*) 'RUNOFF MAPPER ERROR: arrival point has an index outside of permitted range', j, drain_num + write(*,*) 'two different grid points have same distance to a target point!' + end if + call par_ex(partit%MPI_COMM_FESOM, partit%mype) + STOP + end if + dist=distance_on_sphere(lon_sparse(i), lat_sparse(i), geo_coord_nod2D(1,n), geo_coord_nod2D(2,n)) + if (dist < R) then + RUNOFF_MAPPER%values(offset+cnt)=(1.0-dist/R) RUNOFF_MAPPER%colind(offset+cnt)=j if (n<=myDim_nod2d) then - arrival_area(j)=arrival_area(j)+entry*W + arrival_area(j)=arrival_area(j)+(1.0-dist/R)*W end if cnt=cnt+1 end if @@ -579,10 +583,9 @@ subroutine read_runoff_mapper(file, vari, R, partit, mesh) end if END DO -! write(*,*) 'status', mype, status - call MPI_AllREDUCE(MPI_IN_PLACE , arrival_area, drain_num, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) + call MPI_AllREDUCE(MPI_IN_PLACE , arrival_area, drain_num, MPI_DOUBLE, MPI_SUM, MPI_COMM_FESOM, MPIerr) + DO i=1, drain_num - if (mype==0) write(*,*) 'arrival_area=', i, arrival_area(i) where (RUNOFF_MAPPER%colind==i) RUNOFF_MAPPER%values=RUNOFF_MAPPER%values/arrival_area(i) end where @@ -598,9 +601,26 @@ subroutine read_runoff_mapper(file, vari, R, partit, mesh) runoff(n)=sum(RUNOFF_MAPPER%values(i:j)) end do -call integrate_nod(runoff, entry, partit, mesh) +call integrate_nod(runoff, W, partit, mesh) -if (mype==0) write(*,*) 'RUNOFF MAPPER check (input of 1Sv from each basin results in runoff of):', entry, ' Sv' +if (mype==0) write(*,*) 'RUNOFF MAPPER check (total amount of basins):', drain_num +if (mype==0) write(*,*) 'RUNOFF MAPPER check (input of 1Sv from each basin results in runoff of):', W, ' Sv' runoff=runoff*1.e-2 end subroutine read_runoff_mapper -end module g_read_other_NetCDF + +real(kind=WP) function distance_on_sphere(lon1, lat1, lon2, lat2) +! use, intrinsic :: ISO_FORTRAN_ENV + use o_param + use g_config + implicit none +!lons & lats are in radians + real(kind=WP), intent(in) :: lon1, lat1, lon2, lat2 + real(kind=WP) :: r, delta_lon, delta_lat + + delta_lon=abs(lon1-lon2) + if (delta_lon > cyclic_length/2.0_WP) delta_lon=delta_lon-cyclic_length + delta_lat=(lat1-lat2) + r = sin(delta_lat/2.0)**2 + cos(lat1) * cos(lat2) * sin(delta_lon/2.0)**2 + distance_on_sphere=2.0 * atan2(sqrt(r), sqrt(1.0 - r))*r_earth +end function distance_on_sphere +end module g_read_other_NetCDF \ No newline at end of file From 7da741ac13d1f3a72b646c208fa134754c3b072f Mon Sep 17 00:00:00 2001 From: Fernanda Matos Date: Mon, 2 Oct 2023 10:21:18 +0200 Subject: [PATCH 27/71] Correcting for PR comments --- docs/getting_started/getting_started.rst | 31 ++++++++++++------------ 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index 97a19d09d..75c0b989b 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -9,18 +9,25 @@ TL;DR version for supported HPC systems ======================================= Supported systems are: generic ``ubuntu``, ``albedo`` at AWI, ``levante`` at DKRZ, ``JURECA`` at JSC, ``HLRN``, ``Hazel Hen``, ``MareNostrum 4`` at BSC. During configuration the system will be recognised and apropriate environment variables and compiler options should be used. -:: +:: git clone https://github.com/FESOM/fesom2.git cd fesom2 - -If one intends to run simulations on ``albedo`` or ``levante``, one first needs to change to the ``refactoring`` branch (note that this part is only needed while the ``refactoring`` is not yet merged into ``master``: - git checkout refactoring + bash -l ./configure.sh -After confirming that the right FESOM2 branch is being used, compile the model with: +Create file ``fesom.clock`` in the output directory with the following content (if you plan to run with COREII forcing): + +:: + 0 1 1958 + 0 1 1958 + +after that, one has to adjust the run script for the target system and run it: + +:: + cd work + sbatch job_albedo - bash -l ./configure.sh Detailed steps of compiling and runing the code =============================================== @@ -37,7 +44,6 @@ Build model executable with Cmake Clone the GitHub repository with a git command: :: - git clone https://github.com/FESOM/fesom2.git @@ -46,25 +52,17 @@ The repository contains model code and two additional libraries: `Metis` (domain Change to the `fesom2` folder and execute: :: - cd fesom2 - -If one intends to run simulations on ``albedo`` or ``levante``, one first needs to change to the ``refactoring`` branch (note that this part is only needed while the ``refactoring`` branch is not yet merged into ``master``. - -:: - git checkout refactoring As a good practice, if one wants to make modifications to the source code or any of the files, it is advisable to create a branch from refactoring: :: - git checkout -b refactoring After confirming that the right FESOM2 branch is being used, compile the model with: :: - bash -l ./configure.sh In the best case scenario, your platform will be recognized and the Parms library and model executable will be built and copied to the bin directory. If something went wrong have a look at Troubleshooting_ section. @@ -152,7 +150,8 @@ In ``namelist.config``, the options that you might want to change for your first - ``run_length_unit``: units of the run_length. Can be ``y`` (year), ``m`` (month), ``d`` (days), ``s`` (model steps). -.. note:: you will need to adjust the run time to the length of your run. So, if you set ``run_length`` to 10 and ``run_length_unit`` to ``y``, for example, the run time needs to be enough for a 10-year run at once. For 4 nodes, for example, that would take about four hours to run. +.. note:: you might need to adjust the run time to the length of your run. In some setups and/or for some machines, if you set ``run_length`` to 10 and ``run_length_unit`` to ``y``, for example, the run time needs to be enough for a 10-year run at once. + - ``yearnew``: define the same as the year in your ``fesom.clock``; - ``MeshPath``: path to the mesh you would like to use (e.g. ``/youdir/FESOM2_one_year_input/mesh/pi/``, slash at the end is important!); From a3bdfaffd7a8681100baaf48026f2e38a1133a21 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 5 Oct 2023 14:15:54 +0200 Subject: [PATCH 28/71] some updates important for mio functionality --- src/ifs_interface/iom.F90 | 64 +++++++++++++++++++++++++++++++++------ src/io_meandata.F90 | 15 ++++----- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index 27ec77942..f4d5bbb50 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -28,7 +28,8 @@ MODULE iom !!---------------------------------------------------------------------- TYPE iom_field_request - CHARACTER(100) :: name = REPEAT(" ", 100) + CHARACTER(100) :: name = REPEAT(" ", 100) + CHARACTER(100) :: category = REPEAT(" ", 100) CHARACTER(5) :: gridType = REPEAT(" ", 5) REAL(real64), DIMENSION(:), POINTER :: values => NULL() INTEGER :: globalSize = 0 @@ -83,17 +84,17 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) BLOCK CHARACTER(:), allocatable :: config_file INTEGER :: config_file_length - + CALL get_environment_variable('MULTIO_FESOM_CONFIG_FILE', length=config_file_length) IF (config_file_length == 0) THEN call ctl_stop('The fesom plan file is not correctly set!') err = conf_ctx%new() ELSE ALLOCATE(character(len=config_file_length + 1) :: config_file) - + CALL get_environment_variable('MULTIO_FESOM_CONFIG_FILE', config_file) err = conf_ctx%new(config_file) - + DEALLOCATE(config_file) ENDIF END BLOCK @@ -135,11 +136,12 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) END IF ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface +#if defined __ifsinterface err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) + CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) END IF - +#endif err = mio_handle%open_connections(); IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('mio_handle%open_connections failed: ', multio_error_string(err)) @@ -211,11 +213,12 @@ SUBROUTINE iom_init_server(server_comm) ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface ! Set handler before invoking blocking start server call +#if defined __ifsinterface err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) + CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) END IF - +#endif ! Blocking call err = multio_start_server(conf_ctx) IF (err /= MULTIO_SUCCESS) THEN @@ -246,7 +249,11 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) #include "../associate_part_ass.h" #include "../associate_mesh_ass.h" +#if defined __ifsinterface cerr = md%new() +#else + cerr = md%new(mio_handle) +#endif IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: ngrid, md%new() failed: ', multio_error_string(cerr)) END IF @@ -288,7 +295,11 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) END IF !declare grid at elements +#if defined __ifsinterface cerr = md%new() +#else + cerr = md%new(mio_handle) +#endif IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: egrid, md%new() failed: ', multio_error_string(cerr)) END IF @@ -330,18 +341,23 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) END SUBROUTINE iom_send_fesom_domains SUBROUTINE iom_send_fesom_data(data) + USE g_clock IMPLICIT NONE TYPE(iom_field_request), INTENT(INOUT) :: data INTEGER :: cerr TYPE(multio_metadata) :: md +#if defined __ifsinterface cerr = md%new() +#else + cerr = md%new(mio_handle) +#endif IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%new() failed: ', multio_error_string(cerr)) END IF - cerr = md%set_string("category", "fesom-grid-output") + cerr = md%set_string("category", data%category) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%set_string(category) failed: ', multio_error_string(cerr)) END IF @@ -366,11 +382,21 @@ SUBROUTINE iom_send_fesom_data(data) CALL ctl_stop('send_fesom_data: md%set_string(name) failed: ', multio_error_string(cerr)) END IF - cerr = md%set_string("gridSubType", data%gridType) + cerr = md%set_string("gridSubtype", "undefined") IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%set_string(gridSubType) failed: ', multio_error_string(cerr)) END IF + cerr = md%set_string("grid-type", "undefined") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_string(grid-type) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_string("operation", "average") + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_string(operation) failed: ', multio_error_string(cerr)) + END IF + cerr = md%set_string("domain", data%gridType) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%set_string(domain) failed: ', multio_error_string(cerr)) @@ -381,6 +407,24 @@ SUBROUTINE iom_send_fesom_data(data) CALL ctl_stop('send_fesom_data: md%set_int(step) failed: ', multio_error_string(cerr)) END IF + cerr = md%set_int("stepInHours", data%step*24) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_int(stepInHours) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("timeSpanInHours", 24) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_int(timeSpanInHours) failed: ', multio_error_string(cerr)) + END IF + + cerr = md%set_int("currentDate", yearnew * 10000 + month * 100 + day_in_month) + cerr = md%set_int("currentTime", INT(INT(timenew / 3600) * 10000 + (INT(timenew / 60) - INT(timenew / 3600) * 60) * 100 + (timenew-INT(timenew / 60) * 60))) + cerr = md%set_int("startDate", 2020 * 10000 + 01 * 100 + 20) + cerr = md%set_int("startTime", 0) + IF (cerr /= MULTIO_SUCCESS) THEN + CALL ctl_stop('send_fesom_data: md%set_int(date) failed: ', multio_error_string(cerr)) + END IF + cerr = mio_handle%write_field(md, data%values) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: mio_handle%write_field failed: ', multio_error_string(cerr)) diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 6ff0dce80..03af7c5d7 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -649,9 +649,9 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ ! output Monin-Obukov (TB04) mixing length - if (use_momix) then - call def_stream(nod2D, myDim_nod2D, 'momix_length', 'Monin-Obukov mixing length', 'm', mixlength(:), 1, 'm', i_real4, partit, mesh) - end if + !if (use_momix) then + ! call def_stream(nod2D, myDim_nod2D, 'momix_length', 'Monin-Obukov mixing length', 'm', mixlength(:), 1, 'm', i_real4, partit, mesh) + !end if !___________________________________________________________________________ if (ldiag_curl_vel3) then @@ -1745,21 +1745,22 @@ SUBROUTINE send_data_to_multio(entry) END IF request%globalSize = globalSize request%step = entry%rec_count - + if (numLevels==1) then + request%category="ocean-2d" + else + request%category="ocean-3d" + end if ! loop over vertical layers --> do gather 3d variables layerwise in 2d slices DO lev=1, numLevels request%level = lev - IF (.NOT. entry%is_elem_based) THEN request%values => entry%local_values_r8_copy(lev, 1:entry%shrinked_size) ELSE DO i = 1, SIZE(entry%shrinked_indx) temp(i) = entry%local_values_r8_copy(lev, entry%shrinked_indx(i)) END DO - request%values => temp END IF - CALL iom_send_fesom_data(request) END DO END SUBROUTINE From 040ea1c43003838e56d72bf1375bb16cb1e4ed73 Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Tue, 10 Oct 2023 15:50:04 +0200 Subject: [PATCH 29/71] add readthedocs.yaml --- .readthedocs.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..4a4f35297 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file From 79ccbab69196938a9b9d0b2a405ec38e7c69386a Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Tue, 10 Oct 2023 16:10:42 +0200 Subject: [PATCH 30/71] change to python 3.9 --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 4a4f35297..54d438de3 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.9" # You can also specify other tool versions: # nodejs: "20" # rust: "1.70" From b29024666599fec7217faf24507e2cc5dda09a0a Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Tue, 10 Oct 2023 16:35:25 +0200 Subject: [PATCH 31/71] install theme --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index d4b6b966a..302b80753 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx==4.1.2 sphinxcontrib-bibtex==2.3.0 +sphinx_rtd_theme From 4f875204528d2d9086c823a31bbcdb50df013c3c Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Tue, 10 Oct 2023 17:23:03 +0200 Subject: [PATCH 32/71] small fix for docs --- docs/getting_started/getting_started.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index 75c0b989b..24bc94988 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -11,6 +11,7 @@ TL;DR version for supported HPC systems Supported systems are: generic ``ubuntu``, ``albedo`` at AWI, ``levante`` at DKRZ, ``JURECA`` at JSC, ``HLRN``, ``Hazel Hen``, ``MareNostrum 4`` at BSC. During configuration the system will be recognised and apropriate environment variables and compiler options should be used. :: + git clone https://github.com/FESOM/fesom2.git cd fesom2 git checkout refactoring @@ -19,12 +20,14 @@ Supported systems are: generic ``ubuntu``, ``albedo`` at AWI, ``levante`` at DKR Create file ``fesom.clock`` in the output directory with the following content (if you plan to run with COREII forcing): :: + 0 1 1958 0 1 1958 after that, one has to adjust the run script for the target system and run it: :: + cd work sbatch job_albedo @@ -44,6 +47,7 @@ Build model executable with Cmake Clone the GitHub repository with a git command: :: + git clone https://github.com/FESOM/fesom2.git @@ -52,17 +56,20 @@ The repository contains model code and two additional libraries: `Metis` (domain Change to the `fesom2` folder and execute: :: + cd fesom2 git checkout refactoring As a good practice, if one wants to make modifications to the source code or any of the files, it is advisable to create a branch from refactoring: :: + git checkout -b refactoring After confirming that the right FESOM2 branch is being used, compile the model with: :: + bash -l ./configure.sh In the best case scenario, your platform will be recognized and the Parms library and model executable will be built and copied to the bin directory. If something went wrong have a look at Troubleshooting_ section. From 04fe01301e5583aaa31eee7f0f4b4a471836e203 Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Wed, 11 Oct 2023 14:19:14 +0200 Subject: [PATCH 33/71] update docker image to refactoring --- docs/getting_started/getting_started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index 24bc94988..f390d6a1f 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -461,12 +461,12 @@ The best way to run the model locally is to use Docker container. You obviously - Get the image:: - docker pull koldunovn/fesom2_test:fesom2.1 + docker pull koldunovn/fesom2_test:refactoring2 - Go to the folder with your version of fesom2 folder (NOT inside fesom2 folder, one up, the one you run ``git clone https://github.com/FESOM/fesom2.git`` in). - Run:: - docker run -it -v "$(pwd)"/fesom2:/fesom/fesom2 koldunovn/fesom2_test:fesom2.1 /bin/bash + docker run -it -v "$(pwd)"/fesom2:/fesom/fesom2 koldunovn/fesom2_test:refactoring2 /bin/bash - This should get you inside the container. You now can edit the files in your fesom2 folder (on host system), but run compule and run the model inside the container. - When inside the container, to compile do: From e3d9280de5907e9f2be2c3ce2989160f8044f5db Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Wed, 11 Oct 2023 14:38:30 +0200 Subject: [PATCH 34/71] update docker image to refactoring again, sorry suvi :) --- docs/getting_started/getting_started.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index f390d6a1f..227fd00de 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -474,6 +474,7 @@ The best way to run the model locally is to use Docker container. You obviously :: cd fesom2 + git checkout refactoring bash -l configure.sh ubuntu - To prepare the run (this will do the test with pi mesh):: From fa5d477dc5371d2a7c1709a249c8df8d237a9eae Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 13:34:50 +0300 Subject: [PATCH 35/71] Added workflow for sync to lumi and a first version of a gitlab CI workflow --- .github/workflows/lumi-sync.yaml | 26 ++++++++++++++++++++++++++ .gitlab-ci.yml | 13 +++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/workflows/lumi-sync.yaml create mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml new file mode 100644 index 000000000..3121338ac --- /dev/null +++ b/.github/workflows/lumi-sync.yaml @@ -0,0 +1,26 @@ +name: Run CICD on Lumi (using .gitlab-ci.yml) + +on: push + branches: + - refactoring + +jobs: + gitlabsync: + runs-on: ubuntu-latest + steps: + - name: "Check out code" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Gitlab mirror and CI" + uses: "tiggi/gitlab-mirror-and-ci-action@tiggi/fixes" + with: + args: "https://gitlab.com/lumi-cicd/fesom2.git" + env: + FOLLOW_TAGS: "true" + FORCE_PUSH: "true" + GITLAB_HOSTNAME: "gitlab.com" + GITLAB_USERNAME: "tiggi" + GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }} + GITLAB_PROJECT_ID: "51374059" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..bdd7339e1 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,13 @@ +fesom-job: + stage: build + variables: + SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=16G -t 00:30:00 -A project_462000376" + tags: + - lumi + artifacts: + paths: + - fesom_build.log + script: + - echo "building fesom branch" + - bash -l configure.sh lumi + - touch fesom_build.log From 33155cc84bfe2bb50fe3e36854997f9ff4735e31 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 13:45:55 +0300 Subject: [PATCH 36/71] Added lumi_gpu_evatali branch to workflow --- .github/workflows/lumi-sync.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml index 3121338ac..859c0a2eb 100644 --- a/.github/workflows/lumi-sync.yaml +++ b/.github/workflows/lumi-sync.yaml @@ -3,6 +3,7 @@ name: Run CICD on Lumi (using .gitlab-ci.yml) on: push branches: - refactoring + - lumi_gpu_evatali jobs: gitlabsync: From 4a3ea6f97dac6d83003dc2f7dc0b16361aad5517 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:03:06 +0300 Subject: [PATCH 37/71] Fixed bad YAML in Lumi CICD workflow --- .github/workflows/lumi-sync.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml index 859c0a2eb..11ca2531c 100644 --- a/.github/workflows/lumi-sync.yaml +++ b/.github/workflows/lumi-sync.yaml @@ -1,9 +1,10 @@ name: Run CICD on Lumi (using .gitlab-ci.yml) -on: push - branches: - - refactoring - - lumi_gpu_evatali +on: + push: + branches: + - refactoring + - lumi_gpu_evatali jobs: gitlabsync: From 17b396e3019a4ad48b9e75f2333866146fab6675 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:16:25 +0300 Subject: [PATCH 38/71] Add a partition for the Lumi CICD job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd7339e1..78c5607bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ fesom-job: stage: build variables: - SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=16G -t 00:30:00 -A project_462000376" + SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi artifacts: From 62098b4235e049f9a123e8d9b6ab9c7fb7ba20d5 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:24:28 +0300 Subject: [PATCH 39/71] Remove the artifact from gitlab-ci, it's better to have it printed out so that github also gets it --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78c5607bf..d2dd6cf2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,10 +4,9 @@ fesom-job: SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi - artifacts: - paths: - - fesom_build.log + # artifacts: + #paths: + # - fesom_build.log script: - echo "building fesom branch" - bash -l configure.sh lumi - - touch fesom_build.log From 473de2a93baf73fb4684861253341d72d21d2f64 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:27:11 +0300 Subject: [PATCH 40/71] Remove the artifact from gitlab-ci, it's better to have it printed out so that github also gets it #2 --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2dd6cf2c..9822c32bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,9 +4,9 @@ fesom-job: SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi - # artifacts: - #paths: - # - fesom_build.log +# artifacts: +# paths: +# - fesom_build.log script: - echo "building fesom branch" - bash -l configure.sh lumi From 62d3e6a1213e188b682ddb267963a5b94d7bc52d Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 26 Oct 2023 14:36:53 +0200 Subject: [PATCH 41/71] an OMP bux fix in tracer FCT --- src/oce_adv_tra_fct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oce_adv_tra_fct.F90 b/src/oce_adv_tra_fct.F90 index 013a64aff..24a43bef2 100644 --- a/src/oce_adv_tra_fct.F90 +++ b/src/oce_adv_tra_fct.F90 @@ -104,7 +104,7 @@ subroutine oce_tra_adv_fct(dt, ttf, lo, adf_h, adf_v, fct_ttf_min, fct_ttf_max, #include "associate_mesh_ass.h" !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, nz, k, elem, enodes, num, el, nl1, nl2, nu1, nu2, nl12, nu12, edge, & -!$OMP flux, ae,tvert_max, tvert_min) +!$OMP flux, ae) ! -------------------------------------------------------------------------- ! ttf is the tracer field on step n ! del_ttf is the increment From ef9e91f5d8ddfff34cdfb2e380a1ac169435ffc0 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 22 Aug 2023 10:22:37 +0200 Subject: [PATCH 42/71] Add forward declarations in parms for standard compliance --- lib/parms/src/parms_pc_schurras.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/parms/src/parms_pc_schurras.c b/lib/parms/src/parms_pc_schurras.c index 979acdfe9..505dfc1c2 100755 --- a/lib/parms/src/parms_pc_schurras.c +++ b/lib/parms/src/parms_pc_schurras.c @@ -10,6 +10,11 @@ #endif #endif +/* Forward declarations */ +int parms_OperatorGetU(parms_Operator self, void **mat); +int parms_MatGetOffDiag(parms_Mat self, void **mat); +int parms_CommGetOdvlist(parms_Comm self, int **odvlist); + typedef struct schurras_data { parms_Operator op_out,op_in; From 6298b59867ca8caa1c7808610acc874d56ae78f5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:19:49 +0200 Subject: [PATCH 43/71] Fix warning: format string is not a string literal (potentially insecure) --- lib/parms/src/DDPQ/sets.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 1cfb68b68..8e3db4e56 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -19,7 +19,7 @@ void parms_errexit( char *f_str, ... ) sprintf(out2, "Error! %s\n", out1); - fprintf(stdout, out2); + fprintf(stdout, "%s", out2); fflush(stdout); exit( -1 ); From 66eaaafa8a8d80170de11c328e0a902da5222e31 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:24:12 +0200 Subject: [PATCH 44/71] Fix warning: add explicit braces to avoid dangling else --- lib/parms/src/parms_ilu_vcsr.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/parms/src/parms_ilu_vcsr.c b/lib/parms/src/parms_ilu_vcsr.c index 116aa51ba..17169e210 100755 --- a/lib/parms/src/parms_ilu_vcsr.c +++ b/lib/parms/src/parms_ilu_vcsr.c @@ -1683,14 +1683,15 @@ int parms_ilu_update(parms_Mat self, parms_FactParam param, void *mat, if(ABS_VALUE(t1) < DBL_EPSILON) continue; if( ii+start < schur_start ){ - for(jj = 1; jj < nnz; jj++) { - iw = jw[rowjj[jj]]; - if(iw != -1) - if(iw < ii+start) - rowm[iw] -= t1*rowmm[jj]; - else - data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; - } + for(jj = 1; jj < nnz; jj++) { + iw = jw[rowjj[jj]]; + if(iw != -1) { + if(iw < ii+start) + rowm[iw] -= t1*rowmm[jj]; + else + data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; + } + } } else { for(jj = 1; jj < nnz; jj++){ From 74f796e555bc21516d6574a495669a68ed6d1342 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:43:16 +0200 Subject: [PATCH 45/71] Fix warning: incompatible function pointer types initializing... TODO: Assign 'getu' (#512) --- lib/parms/src/DDPQ/arms2.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/parms/src/DDPQ/arms2.c b/lib/parms/src/DDPQ/arms2.c index d7a4ce88d..5dad20138 100755 --- a/lib/parms/src/DDPQ/arms2.c +++ b/lib/parms/src/DDPQ/arms2.c @@ -227,14 +227,15 @@ static int parms_arms_getssize_vcsr(parms_Operator self) } static struct parms_Operator_ops parms_arms_sol_vptr = { - parms_arms_sol_vcsr, - parms_arms_lsol_vcsr, - parms_arms_invs_vcsr, - parms_arms_ascend_vcsr, - parms_arms_getssize_vcsr, - parms_arms_nnz, - arms_free_vcsr, - arms_view_vcsr + parms_arms_sol_vcsr, /* apply */ + parms_arms_lsol_vcsr, /* lsol */ + parms_arms_invs_vcsr, /* invs */ + NULL, /* getu !!! WARNING, UNASSIGNED !!! */ + parms_arms_ascend_vcsr, /* ascend */ + parms_arms_getssize_vcsr, /* getssize */ + parms_arms_nnz, /* getnnz */ + arms_free_vcsr, /* operator_free */ + arms_view_vcsr /* operator_view */ }; From e5509536c182374ff9a40420c9f9864859990501 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 20:07:46 +0200 Subject: [PATCH 46/71] Fix warning: '%s' directive writing up to 255 bytes into a region of size 249 --- lib/parms/src/DDPQ/sets.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 8e3db4e56..1ccf3adb3 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -11,7 +11,7 @@ void parms_errexit( char *f_str, ... ) { va_list argp; - char out1[256], out2[256]; + char out1[256], out2[512]; va_start(argp, f_str); vsprintf(out1, f_str, argp); From 3cdb8d0657430383c307f8328a820ce479306ba6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 20:41:02 +0200 Subject: [PATCH 47/71] Fix warning: extra tokens at end of #endif directive --- src/gen_modules_partitioning.F90 | 15 ++++++---- src/ice_fct.F90 | 50 ++++++++++++++++---------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90 index edf441e73..9c506dac9 100644 --- a/src/gen_modules_partitioning.F90 +++ b/src/gen_modules_partitioning.F90 @@ -93,8 +93,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI #else !For ECHAM coupled runs we use the old OASIS nameing scheme (prism / prism_proto) use mod_prism -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis implicit none integer, intent(in) :: COMM @@ -112,7 +114,8 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI call MPI_Barrier(COMM, error) call MPI_Finalize(error) endif -#else ! standalone +#else + ! standalone ! From here on the two coupled options !------------------------------------- @@ -136,8 +139,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI if (mype==0) print *, 'FESOM calls MPI_Finalize' call MPI_Finalize(error) -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis ! Regardless of standalone, OpenIFS oder ECHAM coupling, if we reach to this point ! we should be fine shutting the whole model down diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index a2ee681ed..8c42cc997 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -136,7 +136,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=0._WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0._WP -#endif /* (__oifs) */ +#endif END DO !$OMP END DO ! Velocities at nodes @@ -174,7 +174,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=rhs_ms(row)+sum(entries*m_snow(elnodes)) #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=rhs_temp(row)+sum(entries*ice_temp(elnodes)) -#endif /* (__oifs) */ +#endif END DO end do !$OMP END DO @@ -210,7 +210,7 @@ subroutine ice_fct_solve(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call ice_fem_fct(4, ice, partit, mesh) ! ice_temp -#endif /* (__oifs) */ +#endif end subroutine ice_fct_solve ! @@ -297,7 +297,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) m_templ(row)=(rhs_temp(row)+gamma*sum(mass_matrix(clo:clo2)* & ice_temp(location(1:cn))))/area(1,row) + & (1.0_WP-gamma)*ice_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO @@ -305,7 +305,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) call exchange_nod(m_icel,a_icel,m_snowl, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(m_templ, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end subroutine ice_solve_low_order @@ -372,7 +372,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=rhs_ms(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_temp(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -380,7 +380,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -407,7 +407,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_temp(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -424,7 +424,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -434,7 +434,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do end subroutine ice_solve_high_order @@ -569,7 +569,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) dm_temp(elnodes)))*(vol/area(1,elnodes(q)))/12.0_WP end do end if -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END DO @@ -646,7 +646,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) !$ACC END PARALLEL LOOP !$OMP END DO end if -#endif /* (__oifs) */ +#endif !___________________________________________________________________________ ! Sums of positive/negative fluxes to node row @@ -946,12 +946,12 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) #endif !$OMP END DO end if -#endif /* (__oifs) */ || defined (__ifsinterface) +#endif !$OMP END PARALLEL call exchange_nod(m_ice, a_ice, m_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(ice_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$ACC END DATA @@ -1126,13 +1126,13 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) rhs_ms(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0.0_WP -#endif /* (__oifs) */ +#endif rhs_mdiv(row)=0.0_WP rhs_adiv(row)=0.0_WP rhs_msdiv(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_tempdiv(row)=0.0_WP -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1180,7 +1180,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) cx3=vol*ice%ice_dt*c4*(sum(m_snow(elnodes))+m_snow(elnodes(n))+sum(entries2*m_snow(elnodes)))/12.0_WP #if defined (__oifs) || defined (__ifsinterface) cx4=vol*ice%ice_dt*c4*(sum(ice_temp(elnodes))+ice_temp(elnodes(n))+sum(entries2*ice_temp(elnodes)))/12.0_WP -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if defined(_OPENMP) && !defined(__openmp_reproducible) @@ -1212,7 +1212,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_temp(row)=rhs_temp(row)+tmp_sum+cx4 -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if !defined(DISABLE_OPENACC_ATOMICS) @@ -1232,7 +1232,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_tempdiv(row)=rhs_tempdiv(row)-cx4 -#endif /* (__oifs) */ +#endif #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(row)) #else @@ -1316,7 +1316,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row)=rhs_msdiv(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_tempdiv(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1326,7 +1326,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -1356,7 +1356,7 @@ subroutine ice_update_for_div(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_tempdiv(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1372,7 +1372,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row) = m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row) = m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1383,7 +1383,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do @@ -1396,7 +1396,7 @@ subroutine ice_update_for_div(ice, partit, mesh) m_snow(row) = m_snow(row)+dm_snow(row) #if defined (__oifs) || defined (__ifsinterface) ice_temp(row)= ice_temp(row)+dm_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO From 4ae7703dc810a6299f5841c11667b3b92c110a03 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 23:48:09 +0200 Subject: [PATCH 48/71] Improve FindNETCDF with extra search hints and handling of COMPONENTS --- cmake/FindNETCDF.cmake | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cmake/FindNETCDF.cmake b/cmake/FindNETCDF.cmake index 03b985d67..cbd823e8e 100644 --- a/cmake/FindNETCDF.cmake +++ b/cmake/FindNETCDF.cmake @@ -13,9 +13,13 @@ if(CMAKE_Fortran_COMPILER_LOADED) if(HAVE_Fortran_NETCDF) set(NETCDF_Fortran_INCLUDE_DIRECTORIES "") set(NETCDF_Fortran_LIBRARIES "") + set(NETCDF_Fortran_FOUND 1) else() - find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) + find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) find_library(NETCDF_Fortran_LIBRARIES netcdff HINTS ${NETCDF_Fortran_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_Fortran_INCLUDE_DIRECTORIES AND NETCDF_Fortran_LIBRARIES ) + set(NETCDF_Fortran_FOUND 1) + endif() endif() endif() @@ -27,14 +31,18 @@ if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED) if(HAVE_C_NETCDF) set(NETCDF_C_INCLUDE_DIRECTORIES "") set(NETCDF_C_LIBRARIES "") + set(NETCDF_C_FOUND 1) else() - find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) + find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) find_library(NETCDF_C_LIBRARIES netcdf HINTS ${NETCDF_C_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_C_INCLUDE_DIRECTORIES AND NETCDF_C_LIBRARIES ) + set(NETCDF_C_FOUND 1) + endif() endif() endif() if(CMAKE_CXX_COMPILER_LOADED) - find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_CXX_INCLUDE_DIRECTORIES) + find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR} ENV NETCDF_CXX_INCLUDE_DIRECTORIES) # the cray toolchain (e.g. hlrn) disables dynamic linking by default. to enable it at build time do e.g. "CRAYPE_LINK_TYPE=dynamic make". find_library(NETCDF_CXX_LIBRARIES NAMES netcdf_c++4 netcdf-cxx4 HINTS ${NETCDF_CXX_INCLUDE_DIRECTORIES}/../lib) if(NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_C_INCLUDE_DIRECTORIES) @@ -43,4 +51,21 @@ if(CMAKE_CXX_COMPILER_LOADED) if(NETCDF_CXX_LIBRARIES AND NETCDF_C_LIBRARIES) list(APPEND NETCDF_CXX_LIBRARIES ${NETCDF_C_LIBRARIES}) endif() -endif() \ No newline at end of file + if( NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_CXX_LIBRARIES ) + set(NETCDF_CXX_FOUND 1) + endif() +endif() + +if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS C) +endif() + +unset({CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) +foreach(COMP C CXX Fortran) + if("${COMP}" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + list(APPEND ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS NETCDF_${COMP}_FOUND) + endif() +endforeach() +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} HANDLE_COMPONENTS REQUIRED_VARS ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) + From d526ae43632798c227c1539957b2a319d8b7981b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 11 Oct 2023 02:52:06 +0300 Subject: [PATCH 49/71] Update lumi environment, showing that FindNETCDF works without intervention --- env/lumi/shell | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env/lumi/shell b/env/lumi/shell index 40a2cb047..aaf0f589c 100644 --- a/env/lumi/shell +++ b/env/lumi/shell @@ -11,10 +11,10 @@ module load cray-netcdf/4.8.1.5 export FC=ftn export CC=cc export CXX=cc -export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib -export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib $CC -v $FC -V $CXX -v From ae6958ef8a429f896802d936cd28af197accc81a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 23:08:13 +0000 Subject: [PATCH 50/71] Update atosecmwf environment, showing mpi compiler wrappers are no longer needed and FindNETCDF also works --- env/atosecmwf/shell | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/env/atosecmwf/shell b/env/atosecmwf/shell index 90f118607..97cd921a7 100644 --- a/env/atosecmwf/shell +++ b/env/atosecmwf/shell @@ -33,7 +33,5 @@ module load cmake/3.20.2 module load ninja/1.10.0 module load fcm/2019.05.0 -export NETCDF_DIR=$NETCDF4_DIR +export FC=ifort CC=icc CXX=icpc -export FC=mpif90 CC=mpicc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly -#export FC=mpif90 CC=gcc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly From 89fbf651ab6da5edf4bee5bbfed50a689cda0674 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 27 Oct 2023 10:38:55 +0200 Subject: [PATCH 51/71] addes missing OMP clauses --- src/ice_fct.F90 | 13 ++++++++----- src/oce_ale.F90 | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index a2ee681ed..129f0a5a4 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -504,7 +504,8 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) ! it takes memory and time. For every element ! we need its antidiffusive contribution to ! each of its 3 nodes - +!$OMP PARALLEL +!$OMP DO !$ACC DATA CREATE(icoef, elnodes) !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) @@ -513,20 +514,21 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) tmin(n) = 0.0_WP end do !$ACC END PARALLEL LOOP - +!$OMP END DO ! Auxiliary elemental operator (mass matrix- lumped mass matrix) !$ACC KERNELS icoef = 1 !$ACC END KERNELS - +!$OMP DO !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) do n=1,3 ! three upper nodes ! Cycle over rows row=elnodes(n) icoef(n,n)=-2 end do !$ACC END PARALLEL LOOP - +!$OMP END DO +!$OMP END PARALLEL !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, q, elem, elnodes, row, vol, flux, ae) !$OMP DO @@ -1117,7 +1119,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) ! Computes the rhs in a Taylor-Galerkin way (with upwind type of ! correction for the advection operator) ! In this version I tr to split divergent term off, so that FCT works without it. - +!$OMP PARALLEL DO !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) do row=1, myDim_nod2D !! row=myList_nod2D(m) @@ -1135,6 +1137,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) #endif /* (__oifs) */ end do !$ACC END PARALLEL LOOP +!$OMP END PARALLEL DO !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(diff, entries, um, vm, vol, dx, dy, n, q, row, elem, elnodes, c1, c2, c3, c4, cx1, cx2, cx3, cx4, entries2) !$OMP DO diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 3c25bb38c..973f81161 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2495,7 +2495,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) end do !$OMP END PARALLEL DO -!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax) +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, c1, c2) do n=1, myDim_nod2D+eDim_nod2D nzmin = ulevels_nod2D(n) nzmax = nlevels_nod2D(n)-1 From efbb3ed1412288fe2dbe4dcc14463e9b9e956cc0 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 27 Oct 2023 10:58:33 +0200 Subject: [PATCH 52/71] removed just introduced OMP statement im ice_fct (was not needed) --- src/ice_fct.F90 | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index 129f0a5a4..07f57531f 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -504,8 +504,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) ! it takes memory and time. For every element ! we need its antidiffusive contribution to ! each of its 3 nodes -!$OMP PARALLEL -!$OMP DO +!$OMP PARALLEL DO !$ACC DATA CREATE(icoef, elnodes) !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) @@ -514,21 +513,18 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) tmin(n) = 0.0_WP end do !$ACC END PARALLEL LOOP -!$OMP END DO +!$OMP END PARALLEL DO ! Auxiliary elemental operator (mass matrix- lumped mass matrix) !$ACC KERNELS icoef = 1 !$ACC END KERNELS -!$OMP DO !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) do n=1,3 ! three upper nodes ! Cycle over rows row=elnodes(n) icoef(n,n)=-2 end do !$ACC END PARALLEL LOOP -!$OMP END DO -!$OMP END PARALLEL !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, q, elem, elnodes, row, vol, flux, ae) !$OMP DO From fbecd21e0cb80ff7e9e170245cc2a25e0a882bc2 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Wed, 1 Nov 2023 14:52:45 +0100 Subject: [PATCH 53/71] added OMP for mEVP (aEVP is still not covered). tested for CORE2 mesh and works! --- src/ice_maEVP.F90 | 212 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 176 insertions(+), 36 deletions(-) diff --git a/src/ice_maEVP.F90 b/src/ice_maEVP.F90 index debe58873..b2900710e 100644 --- a/src/ice_maEVP.F90 +++ b/src/ice_maEVP.F90 @@ -130,6 +130,7 @@ subroutine stress_tensor_m(ice, partit, mesh) vale=1.0_WP/(ice%ellipse**2) det2=1.0_WP/(1.0_WP+ice%alpha_evp) det1=ice%alpha_evp*det2 +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes, dx, dy, msum, asum, eps1, eps2, pressure, delta, meancos, usum, vsum, r1, r2, r3, si1, si2) do elem=1,myDim_elem2D elnodes=elem2D_nodes(:,elem) !_______________________________________________________________________ @@ -184,6 +185,7 @@ subroutine stress_tensor_m(ice, partit, mesh) rdg_shear_elem(elem) = 0.5_WP*(delta - abs(eps11(elem)+eps22(elem))) #endif end do +!$OMP END PARALLEL DO ! Equations solved in terms of si1, si2, eps1, eps2 are (43)-(45) of ! Boullion et al Ocean Modelling 2013, but in an implicit mode: ! si1_{p+1}=det1*si1_p+det2*r1, where det1=alpha/(1+alpha) and det2=1/(1+alpha), @@ -232,15 +234,18 @@ subroutine ssh2rhs(ice, partit, mesh) !___________________________________________________________________________ val3=1.0_WP/3.0_WP +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(row, elem, elnodes, n, dx, dy, vol, meancos, aa, bb, p_ice) +!$OMP DO ! use rhs_m and rhs_a for storing the contribution from elevation: do row=1, myDim_nod2d rhs_a(row)=0.0_WP rhs_m(row)=0.0_WP end do - +!$OMP END DO !_____________________________________________________________________________ ! use floating sea ice for zlevel and zstar if (use_floatice .and. .not. trim(which_ale)=='linfs') then +!$OMP DO do elem=1,myDim_elem2d elnodes=elem2D_nodes(:,elem) !_______________________________________________________________________ @@ -263,10 +268,24 @@ subroutine ssh2rhs(ice, partit, mesh) bb=g*val3*vol aa=bb*sum(dx*(elevation(elnodes)+p_ice)) bb=bb*sum(dy*(elevation(elnodes)+p_ice)) - rhs_a(elnodes)=rhs_a(elnodes)-aa - rhs_m(elnodes)=rhs_m(elnodes)-bb + do n=1,3 +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(n))) +#else +!$OMP ORDERED +#endif + rhs_a(elnodes(n))=rhs_a(elnodes(n))-aa + rhs_m(elnodes(n))=rhs_m(elnodes(n))-bb +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(n))) +#else +!$OMP END ORDERED +#endif + end do end do +!$OMP END DO else +!$OMP DO do elem=1,myDim_elem2d elnodes=elem2D_nodes(:,elem) !_______________________________________________________________________ @@ -279,10 +298,24 @@ subroutine ssh2rhs(ice, partit, mesh) bb=g*val3*vol aa=bb*sum(dx*elevation(elnodes)) bb=bb*sum(dy*elevation(elnodes)) - rhs_a(elnodes)=rhs_a(elnodes)-aa - rhs_m(elnodes)=rhs_m(elnodes)-bb + do n=1,3 +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(n))) +#else +!$OMP ORDERED +#endif + rhs_a(elnodes(n))=rhs_a(elnodes(n))-aa + rhs_m(elnodes(n))=rhs_m(elnodes(n))-bb +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(n))) +#else +!$OMP END ORDERED +#endif + end do end do +!$OMP END DO end if +!$OMP END PARALLEL end subroutine ssh2rhs ! ! @@ -330,12 +363,15 @@ subroutine stress2rhs_m(ice, partit, mesh) !___________________________________________________________________________ val3=1.0_WP/3.0_WP - +!$OMP PARALLEL DO do row=1, myDim_nod2d u_rhs_ice(row)=0.0_WP v_rhs_ice(row)=0.0_WP end do +!$OMP END PARALLEL DO +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, k, row, dx, dy, vol, mf, aa, bb, mass, cluster_area, elevation_elem) +!$OMP DO do elem=1,myDim_elem2d elnodes=elem2D_nodes(:,elem) !_______________________________________________________________________ @@ -351,15 +387,26 @@ subroutine stress2rhs_m(ice, partit, mesh) do k=1,3 row=elnodes(k) +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(row)) +#else +!$OMP ORDERED +#endif u_rhs_ice(row)=u_rhs_ice(row) - vol* & (sigma11(elem)*dx(k)+sigma12(elem)*dy(k)) & -vol*sigma12(elem)*val3*mf !metrics v_rhs_ice(row)=v_rhs_ice(row) - vol* & (sigma12(elem)*dx(k)+sigma22(elem)*dy(k)) & +vol*sigma11(elem)*val3*mf ! metrics +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(row)) +#else +!$OMP END ORDERED +#endif end do end do - +!$OMP END DO +!$OMP DO do row=1, myDim_nod2d !_______________________________________________________________________ ! if cavity node skip it @@ -370,6 +417,8 @@ subroutine stress2rhs_m(ice, partit, mesh) u_rhs_ice(row)=(u_rhs_ice(row)*mass + rhs_a(row))/area(1,row) v_rhs_ice(row)=(v_rhs_ice(row)*mass + rhs_m(row))/area(1,row) end do +!$OMP END DO +!$OMP END PARALLEL end subroutine stress2rhs_m ! ! @@ -485,16 +534,18 @@ subroutine EVPdynamics_m(ice, partit, mesh) !NR inlined, to have all initialization in one place. ! call ssh2rhs - ! use rhs_m and rhs_a for storing the contribution from elevation: +!$OMP PARALLEL DO do row=1, myDim_nod2d rhs_a(row)=0.0_WP rhs_m(row)=0.0_WP end do - +!$OMP END PARALLEL DO !_____________________________________________________________________________ ! use floating sea ice for zlevel and zstar +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(el, elnodes, vol, dx, dy, p_ice, n, bb, aa) if (use_floatice .and. .not. trim(which_ale)=='linfs') then +!$OMP DO do el=1,myDim_elem2d elnodes=elem2D_nodes(:,el) @@ -518,12 +569,26 @@ subroutine EVPdynamics_m(ice, partit, mesh) bb=g*val3*vol aa=bb*sum(dx*(elevation(elnodes)+p_ice)) bb=bb*sum(dy*(elevation(elnodes)+p_ice)) - rhs_a(elnodes)=rhs_a(elnodes)-aa - rhs_m(elnodes)=rhs_m(elnodes)-bb + do n=1, 3 +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(n))) +#else +!$OMP ORDERED +#endif + rhs_a(elnodes(n))=rhs_a(elnodes(n))-aa + rhs_m(elnodes(n))=rhs_m(elnodes(n))-bb +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(n))) +#else +!$OMP END ORDERED +#endif + end do end do +!$OMP END DO !_____________________________________________________________________________ ! use levitating sea ice for linfs, zlevel and zstar else +!$OMP DO do el=1,myDim_elem2d elnodes=elem2D_nodes(:,el) !_______________________________________________________________________ @@ -536,13 +601,27 @@ subroutine EVPdynamics_m(ice, partit, mesh) bb=g*val3*vol aa=bb*sum(dx*elevation(elnodes)) bb=bb*sum(dy*elevation(elnodes)) - rhs_a(elnodes)=rhs_a(elnodes)-aa - rhs_m(elnodes)=rhs_m(elnodes)-bb + do n=1, 3 +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(n))) +#else +!$OMP ORDERED +#endif + rhs_a(elnodes(n))=rhs_a(elnodes(n))-aa + rhs_m(elnodes(n))=rhs_m(elnodes(n))-bb +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(n))) +#else +!$OMP END ORDERED +#endif + end do end do +!$OMP END DO end if - +!$OMP END PARALLEL !___________________________________________________________________________ ! precompute thickness (the inverse is needed) and mass (scaled by area) +!$OMP PARALLEL DO do i=1,myDim_nod2D inv_thickness(i) = 0._WP mass(i) = 0._WP @@ -565,9 +644,10 @@ subroutine EVPdynamics_m(ice, partit, mesh) ice_nod(i) = .true. endif enddo - +!$OMP END PARALLEL DO !___________________________________________________________________________ ! precompute pressure factor +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(el, elnodes, msum, asum) do el=1,myDim_elem2D elnodes=elem2D_nodes(:,el) pressure_fac(el) = 0._WP @@ -584,12 +664,13 @@ subroutine EVPdynamics_m(ice, partit, mesh) pressure_fac(el) = det2*ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) endif end do - +!$OMP END PARALLEL DO +!$OMP PARALLEL DO do row=1, myDim_nod2d u_rhs_ice(row)=0.0_WP v_rhs_ice(row)=0.0_WP end do - +!$OMP END PARALLEL DO !___________________________________________________________________________ ! Ice EVPdynamics Iteration main loop: #if defined (__icepack) @@ -603,6 +684,8 @@ subroutine EVPdynamics_m(ice, partit, mesh) ! New implementation following Boullion et al, Ocean Modelling 2013. ! SD, 30.07.2014 !_______________________________________________________________________ +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(el, i, ed, row, elnodes, dx, dy, meancos, eps1, eps2, delta, pressure, umod, drag, rhsu, rhsv, det, n) +!$OMP DO do el=1,myDim_elem2D if (ulevels(el)>1) cycle @@ -655,28 +738,59 @@ subroutine EVPdynamics_m(ice, partit, mesh) ! SD, 30.07.2014 !----------------------------------------------------------------- if (elnodes(1) <= myDim_nod2D) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(1))) +#else +!$OMP ORDERED +#endif u_rhs_ice(elnodes(1)) = u_rhs_ice(elnodes(1)) - elem_area(el)* & (sigma11(el)*dx(1)+sigma12(el)*(dy(1) + meancos)) !metrics v_rhs_ice(elnodes(1)) = v_rhs_ice(elnodes(1)) - elem_area(el)* & - (sigma12(el)*dx(1)+sigma22(el)*dy(1) - sigma11(el)*meancos) ! metrics + (sigma12(el)*dx(1)+sigma22(el)*dy(1) - sigma11(el)*meancos) !metrics +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(1))) +#else +!$OMP END ORDERED +#endif end if if (elnodes(2) <= myDim_nod2D) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(2))) +#else +!$OMP ORDERED +#endif u_rhs_ice(elnodes(2)) = u_rhs_ice(elnodes(2)) - elem_area(el)* & (sigma11(el)*dx(2)+sigma12(el)*(dy(2) + meancos)) !metrics v_rhs_ice(elnodes(2)) = v_rhs_ice(elnodes(2)) - elem_area(el)* & - (sigma12(el)*dx(2)+sigma22(el)*dy(2) - sigma11(el)*meancos) ! metrics + (sigma12(el)*dx(2)+sigma22(el)*dy(2) - sigma11(el)*meancos) !metrics end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(2))) +#else +!$OMP END ORDERED +#endif if (elnodes(3) <= myDim_nod2D) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(elnodes(3))) +#else +!$OMP ORDERED +#endif u_rhs_ice(elnodes(3)) = u_rhs_ice(elnodes(3)) - elem_area(el)* & (sigma11(el)*dx(3)+sigma12(el)*(dy(3) + meancos)) !metrics v_rhs_ice(elnodes(3)) = v_rhs_ice(elnodes(3)) - elem_area(el)* & - (sigma12(el)*dx(3)+sigma22(el)*dy(3) - sigma11(el)*meancos) ! metrics + (sigma12(el)*dx(3)+sigma22(el)*dy(3) - sigma11(el)*meancos) !metrics +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(elnodes(3))) +#else +!$OMP END ORDERED +#endif end if end if end do ! --> do el=1,myDim_elem2D - +!$OMP END DO +!$OMP DO do i=1, myDim_nod2d !___________________________________________________________________ if (ulevels_nod2D(i)>1) cycle @@ -702,15 +816,24 @@ subroutine EVPdynamics_m(ice, partit, mesh) v_ice_aux(i) = det*((1.0_WP+ice%beta_evp+drag)*rhsv -rdt*mesh%coriolis_node(i)*rhsu) end if end do ! --> do i=1, myDim_nod2d - +!$OMP END DO !_______________________________________________________________________ ! apply sea ice velocity boundary condition +!$OMP DO do ed=1,myDim_edge2D !___________________________________________________________________ ! apply coastal sea ice velocity boundary conditions - if(myList_edge2D(ed) > edge2D_in) then - u_ice_aux(edges(:,ed))=0.0_WP - v_ice_aux(edges(:,ed))=0.0_WP + if (myList_edge2D(ed) > edge2D_in) then + do n=1, 2 +#if defined(_OPENMP) + call omp_set_lock (partit%plock(edges(n, ed))) +#endif + u_ice_aux(edges(n,ed))=0.0_WP + v_ice_aux(edges(n,ed))=0.0_WP +#if defined(_OPENMP) + call omp_unset_lock(partit%plock(edges(n,ed))) +#endif + end do end if !___________________________________________________________________ @@ -718,26 +841,43 @@ subroutine EVPdynamics_m(ice, partit, mesh) if (use_cavity) then if ( (ulevels(edge_tri(1,ed))>1) .or. & ( edge_tri(2,ed)>0 .and. ulevels(edge_tri(2,ed))>1) ) then - u_ice_aux(edges(1:2,ed))=0.0_WP - v_ice_aux(edges(1:2,ed))=0.0_WP + do n=1, 2 +#if defined(_OPENMP) + call omp_set_lock (partit%plock(edges(n, ed))) +#endif + u_ice_aux(edges(n,ed))=0.0_WP + v_ice_aux(edges(n,ed))=0.0_WP +#if defined(_OPENMP) + call omp_unset_lock(partit%plock(edges(n,ed))) +#endif + end do end if end if end do ! --> do ed=1,myDim_edge2D - +!$OMP END DO !_______________________________________________________________________ +!$OMP MASTER call exchange_nod_begin(u_ice_aux, v_ice_aux, partit) - +!$OMP END MASTER +!$OMP BARRIER +!$OMP DO do row=1, myDim_nod2d - u_rhs_ice(row)=0.0_WP - v_rhs_ice(row)=0.0_WP + u_rhs_ice(row)=0.0_WP + v_rhs_ice(row)=0.0_WP end do - +!$OMP END DO +!$OMP MASTER call exchange_nod_end(partit) - +!$OMP END MASTER +!$OMP BARRIER +!$OMP END PARALLEL end do ! --> do shortstep=1, steps - u_ice=u_ice_aux - v_ice=v_ice_aux - +!$OMP PARALLEL DO + do row=1, myDim_nod2d+eDim_nod2D + u_ice(row)=u_ice_aux(row) + v_ice(row)=v_ice_aux(row) + end do +!$OMP END PARALLEL DO end subroutine EVPdynamics_m ! ! From aec5350f001ac208d48a2f73c2907683558dff7a Mon Sep 17 00:00:00 2001 From: Sebastian Beyer Date: Tue, 7 Nov 2023 14:10:31 +0100 Subject: [PATCH 54/71] fix OpenMP in EVPdynamics --- src/ice_maEVP.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ice_maEVP.F90 b/src/ice_maEVP.F90 index b2900710e..62fa9c0a0 100644 --- a/src/ice_maEVP.F90 +++ b/src/ice_maEVP.F90 @@ -764,12 +764,12 @@ subroutine EVPdynamics_m(ice, partit, mesh) (sigma11(el)*dx(2)+sigma12(el)*(dy(2) + meancos)) !metrics v_rhs_ice(elnodes(2)) = v_rhs_ice(elnodes(2)) - elem_area(el)* & (sigma12(el)*dx(2)+sigma22(el)*dy(2) - sigma11(el)*meancos) !metrics - end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(elnodes(2))) #else !$OMP END ORDERED #endif + end if if (elnodes(3) <= myDim_nod2D) then #if defined(_OPENMP) && !defined(__openmp_reproducible) From ac16e9c45a75119a1fac5dc5aeb6983d81c5a7fe Mon Sep 17 00:00:00 2001 From: suvarchal Date: Fri, 10 Nov 2023 11:23:16 +0100 Subject: [PATCH 55/71] consistent blowup check for salinty as clipped in oce_ale_tracer.F90 --- src/oce_ale_tracer.F90 | 3 ++- src/write_step_info.F90 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/oce_ale_tracer.F90 b/src/oce_ale_tracer.F90 index 34b9528ff..932110adf 100644 --- a/src/oce_ale_tracer.F90 +++ b/src/oce_ale_tracer.F90 @@ -274,7 +274,8 @@ subroutine solve_tracers_ale(ice, dynamics, tracers, partit, mesh) end do !$OMP END PARALLEL DO end if - + + ! TODO: do it only when it is coupled to atmosphere !___________________________________________________________________________ ! to avoid crash with high salinities when coupled to atmosphere ! --> if we do only where (tr_arr(:,:,2) < 3._WP ) we also fill up the bottom diff --git a/src/write_step_info.F90 b/src/write_step_info.F90 index 858dd1812..9b1087248 100644 --- a/src/write_step_info.F90 +++ b/src/write_step_info.F90 @@ -469,7 +469,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) !_______________________________________________________________ ! check salt if ( (tracers%data(2)%values(nz, n) /= tracers%data(2)%values(nz, n)) .or. & - tracers%data(2)%values(nz, n) <=3.0_WP .or. tracers%data(2)%values(nz, n)>=45.0_WP ) then + tracers%data(2)%values(nz, n) <3.0_WP .or. tracers%data(2)%values(nz, n) >45.0_WP ) then !$OMP CRITICAL found_blowup_loc=1 write(*,*) '___CHECK FOR BLOW UP___________ --> mstep=',istep From 683c6c40f4505a0d0711e15b02a5fc340e772bd1 Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Fri, 15 Dec 2023 13:27:26 +0100 Subject: [PATCH 56/71] moving node_contours into cpl_driver module, to use it. Replaces old angle based algorithm --- src/cpl_driver.F90 | 574 +++++++++++++++++------------------------- src/node_contours.F90 | 174 ------------- 2 files changed, 229 insertions(+), 519 deletions(-) mode change 100755 => 100644 src/cpl_driver.F90 delete mode 100644 src/node_contours.F90 diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 old mode 100755 new mode 100644 index 9e2829ae2..2be6c8278 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -2,16 +2,13 @@ module cpl_driver !====================================================================== ! - ! for coupling between the FESOM and an AOGCM using OASIS3-MCT + ! for coupling between the FESOM ocean ECHAM6 atmosphere using OASIS3-MCT ! !===================================================================== ! History : ! 09-09 (R. Redler, Germany) Original code ! 09-09 (K.Fieg, AWI Germany) Adjustment for FESOM ! 07-12 (D.Barbi, AWI Germany) Switch to ECHAM6.1 and OASIS3-MCT - ! 01-19 (J.Streffing, AWI Germany) Added OpenIFS coupling - ! 03-23 (J.Streffing, AWI Germany) Added corner point computation - ! for 1st order conserv remapping !---------------------------------------------------------------------- ! Modules used ! @@ -94,6 +91,182 @@ module cpl_driver contains + subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE o_PARAM + use g_comm_auto + use o_ARRAYS + use g_rotate_grid + + IMPLICIT NONE + type(t_mesh), intent(in), target :: mesh + type(t_partit), intent(inout), target :: partit + real(kind=WP), allocatable, intent(inout) :: my_x_corners(:,:) ! longitude node corners + real(kind=WP), allocatable, intent(inout) :: my_y_corners(:,:) ! latitude node corners + integer :: bEdge_left, bEdge_right + integer, dimension(2) :: belem_left, belem_right + integer :: edge_left, edge_right + integer :: n, ee, elem, nn, el(2), flag, nn1, nn2 + integer, allocatable, dimension(:) :: nedges, nelems, nedges1, nelems1, nedges2, nelems2 + real(kind=WP) :: this_x_coord, this_y_coord +include "associate_part_def.h" +include "associate_mesh_def.h" +include "associate_part_ass.h" +include "associate_mesh_ass.h" + + if (.not. allocated(array)) then + ALLOCATE(my_x_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) + ALLOCATE(my_y_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) + endif + do n=1, myDim_nod2D + ! find the type of node: internal or at boundary + bEdge_left =0 + belem_left =0 + bEdge_right=0 + belem_right=0 + + do ee=1, nod_in_elem2D_num(n) + elem=nod_in_elem2D(ee,n) + if (elem2D_nodes(1,elem)==n) then + edge_left=elem_edges(3,elem) + edge_right=elem_edges(2,elem) + elseif (elem2D_nodes(2,elem)==n) then + edge_left=elem_edges(1,elem) + edge_right=elem_edges(3,elem) + else + edge_left=elem_edges(2,elem) + edge_right=elem_edges(1,elem) + end if + if (myList_edge2D(edge_left)>edge2D_in) then + bEdge_left=bEdge_left+1 + belem_left(bEdge_left)=elem + end if + if (myList_edge2D(edge_right)>edge2D_in) then + bEdge_right=bEdge_right+1 + belem_right(bEdge_right)=elem + end if + end do + + ! now we have three cases + if (bEdge_left==0) then ! inner contour + elem=nod_in_elem2D(1, n) ! we can start from any + allocate(nedges(nod_in_elem2D_num(n))) + nedges=0 + allocate(nelems(nod_in_elem2D_num(n))) + nelems=0 + !!!!!!! inner_node_contour +include "node_contour_inner.h" + do nn=1, nod_in_elem2D_num(n) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + do nn=nod_in_elem2D_num(n)+1, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)) + my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)) + end do + deallocate(nedges, nelems) + end if + + if (bEdge_left==1) then ! standard boundary node + elem=belem_left(1) + allocate(nedges(nod_in_elem2D_num(n)+1)) + nedges=0 + allocate(nelems(nod_in_elem2D_num(n))) + nelems=0 + !!!!!!!boundary_node_contour +include "node_contour_boundary.h" + do nn=1, nod_in_elem2D_num(n) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + nn=nod_in_elem2D_num(n)+1 + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + nn=nod_in_elem2D_num(n)+2 + my_x_corners(n, nn)=coord_nod2D(1,n) + my_y_corners(n, nn)=coord_nod2D(2,n) + do nn=nod_in_elem2D_num(n)+3, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)+2) + my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)+2) + end do + !!!!!!! + deallocate(nedges, nelems) + end if + + if (bEdge_left==2) then ! strange boundary node + elem=belem_left(1) + allocate(nedges (nod_in_elem2D_num(n)+1)) + allocate(nedges1(nod_in_elem2D_num(n)+1)) + nedges =0 + nedges1=0 + allocate(nelems (nod_in_elem2D_num(n))) + allocate(nelems1(nod_in_elem2D_num(n))) + nelems=0 + nelems1=0 + !!!!!!!boundary_node_contour +include "node_contour_boundary.h" + where (nedges>0) + nedges1=nedges + end where + where (nelems>0) + nelems1=nelems + end where + nn1=nn + do nn=1, nn1 + call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems1(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + nn=nn1+1 + call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + nn=nn1+2 + my_x_corners(n, nn)=coord_nod2D(1,n) + my_y_corners(n, nn)=coord_nod2D(2,n) + !!!!!!! + elem=belem_left(2) + allocate(nedges2(nod_in_elem2D_num(n)+1)) + nedges =0 + nedges2=0 + allocate(nelems2(nod_in_elem2D_num(n))) + nelems =0 + nelems2=0 + !!!!!!!boundary_node_contour +include "node_contour_boundary.h" + where (nedges>0) + nedges2=nedges + end where + where (nelems>0) + nelems2=nelems + end where + nn2=nn + do nn=nn1+3, nn1+nn2+2 + call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call elem_center(nelems2(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + end do + nn=nn1+nn2+3 + call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + nn=nn1+nn2+4 + my_x_corners(n, nn)=coord_nod2D(1,n) + my_y_corners(n, nn)=coord_nod2D(2,n) + do nn=nn1+nn2+5, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nn1+nn2+4) + my_y_corners(n, nn)=my_y_corners(n, nn1+nn2+4) + end do + !!!!!!! + deallocate(nedges, nelems, nedges1, nelems1, nedges2, nelems2) + end if + end do + do n=1, myDim_nod2D + do nn=1, size(my_x_corners, 2) + this_x_coord=my_x_corners(n, nn) + this_y_coord=my_y_corners(n, nn) + call r2g(my_x_corners(n, nn), my_y_corners(n, nn), this_x_coord, this_y_coord) + end do + end do + my_x_corners=my_x_corners/rad + my_y_corners=my_y_corners/rad + end subroutine node_contours + subroutine cpl_oasis3mct_init(partit, localCommunicator ) USE MOD_PARTIT implicit none @@ -208,8 +381,8 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) integer :: il_flag logical :: new_points - integer :: i, j, k ! local loop indicees - integer :: l,m,n, done ! local loop indicees + integer :: i, j, k ! local loop indicees + integer :: l,m ! local loop indicees character(len=32) :: point_name ! name of the grid points @@ -218,46 +391,29 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) integer :: counts_from_all_pes(partit%npes) integer :: displs_from_all_pes(partit%npes) integer :: my_displacement - integer :: my_max_elem(partit%npes) - integer :: my_max_edge(partit%npes) - integer :: all_max_elem, all_max_edge, n_neg, n_pos - integer :: el(2), enodes(2), edge - - integer,allocatable :: unstr_mask(:,:), coastal_edge_list(:,:) - real(kind=WP) :: max_x ! max longitude on corners of control volume - real(kind=WP) :: min_x ! min longitude on corners of control volume - real(kind=WP) :: temp ! temp storage for corner sorting + + integer,allocatable :: unstr_mask(:,:) real(kind=WP) :: this_x_coord ! longitude coordinates real(kind=WP) :: this_y_coord ! latitude coordinates - real(kind=WP) :: this_x_corners ! longitude node corners - real(kind=WP) :: this_y_corners ! latitude node corners ! ! Corner data structure for a OASIS3-MCT Reglonlatvrt grid ! - real(kind=WP), allocatable :: pos_x(:) ! longitude to the right of dateline - real(kind=WP), allocatable :: pos_y(:) ! latitude to the right of dateline - real(kind=WP), allocatable :: neg_x(:) ! longitude to the left of dateline - real(kind=WP), allocatable :: neg_y(:) ! latitude to the left of dateline - real(kind=WP), allocatable :: temp_x_coord(:) ! longitude coordinates - real(kind=WP), allocatable :: temp_y_coord(:) ! longitude coordinates real(kind=WP), allocatable :: my_x_coords(:) ! longitude coordinates real(kind=WP), allocatable :: my_y_coords(:) ! latitude coordinates - real(kind=WP), allocatable :: angle(:,:) ! array for holding corner angle for sorting - real(kind=WP), allocatable :: my_x_corners(:,:) ! longitude node corners - real(kind=WP), allocatable :: my_y_corners(:,:) ! latitude node corners - real(kind=WP), allocatable :: coord_e_edge_center(:,:,:) ! edge center coords + real(kind=WP), allocatable :: all_x_coords(:, :) ! longitude coordinates real(kind=WP), allocatable :: all_y_coords(:, :) ! latitude coordinates - real(kind=WP), allocatable :: all_x_corners(:,:,:) ! longitude node corners - real(kind=WP), allocatable :: all_y_corners(:,:,:) ! latitude node corners real(kind=WP), allocatable :: all_area(:,:) - logical, allocatable :: coastal_nodes(:) + real(kind=WP), allocatable :: my_x_corners(:,:) ! local longitude node corners + real(kind=WP), allocatable :: my_y_corners(:,:) ! local latitude node corners + real(kind=WP), allocatable :: all_x_corners(:,:,:) ! global longitude node corners + real(kind=WP), allocatable :: all_y_corners(:,:,:) ! global latitude node corners -#include "associate_part_def.h" -#include "associate_mesh_def.h" -#include "associate_part_ass.h" -#include "associate_mesh_ass.h" +include "associate_part_def.h" +include "associate_mesh_def.h" +include "associate_part_ass.h" +include "associate_mesh_ass.h" #ifdef VERBOSE @@ -307,39 +463,13 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) my_displacement = SUM(counts_from_all_pes(1:mype)) endif - CALL MPI_BARRIER(MPI_COMM_FESOM, ierror) - - my_max_elem=0 - my_max_elem = maxval(nod_in_elem2D_num(1:myDim_nod2D)) - all_max_elem = 0 - call MPI_Allreduce(my_max_elem, all_max_elem, & - 1, MPI_INTEGER, MPI_MAX, & - MPI_COMM_FESOM, MPIerr) - - my_max_edge=0 - my_max_edge=maxval(nn_num) - all_max_edge=0 - call MPI_AllREDUCE( my_max_edge, all_max_edge, & - 1, MPI_INTEGER,MPI_MAX, & - MPI_COMM_FESOM, MPIerr) - - CALL MPI_BARRIER(MPI_COMM_FESOM, ierror) - - if (mype .eq. 0) then - print *, 'Max elements per node:', all_max_elem, 'Max edges per node:', all_max_edge - print *, 'FESOM before def partition' - endif - ig_paral(1) = 1 ! Apple Partition ig_paral(2) = my_displacement ! Global Offset ig_paral(3) = my_number_of_points ! Local Extent - ! For MPI_GATHERV we need the location of the local segment in the global vector - displs_from_all_pes(1) = 0 - do i = 2, npes - displs_from_all_pes(i) = SUM(counts_from_all_pes(1:(i-1))) - enddo - + if (mype .eq. 0) then + print *, 'FESOM before def partition' + endif CALL oasis_def_partition( part_id(1), ig_paral, ierror ) if (mype .eq. 0) then print *, 'FESOM after def partition' @@ -348,282 +478,51 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) print *, 'FESOM commRank def_partition failed' call oasis_abort(comp_id, 'cpl_oasis3mct_define_unstr', 'def_partition failed') endif - - ALLOCATE(coastal_nodes(number_of_all_points)) - ALLOCATE(angle(my_number_of_points,all_max_elem+all_max_edge)) - ALLOCATE(my_x_corners(my_number_of_points,all_max_elem+all_max_edge)) - ALLOCATE(my_y_corners(my_number_of_points,all_max_elem+all_max_edge)) - ALLOCATE(coord_e_edge_center(2,my_number_of_points, all_max_edge)) - - ! We need to know for every node if any of it's edges are coastal, because - ! in case they are the center point will be a corner of the nodal area - coastal_nodes=.False. - allocate (coastal_edge_list(my_number_of_points*2,my_number_of_points*2)) - do edge=1, myDim_edge2D - ! local indice of nodes that span up edge - enodes=edges(:,edge) - ! local index of element that contribute to edge - el=edge_tri(:,edge) - if(el(2)>0) then - ! Inner edge - continue - else - ! Boundary/coastal edge - coastal_nodes(enodes(1))=.True. - coastal_nodes(enodes(2))=.True. - coastal_edge_list(enodes(1),enodes(2))=edge - coastal_edge_list(enodes(2),enodes(1))=edge - end if - end do - - - ! For every node, loop over neighbours, calculate edge center as mean of node center and neighbour node center. - coord_e_edge_center=0 - do i = 1, my_number_of_points - ! if we are on coastal node, include node center n=1 as corner - if (coastal_nodes(i)==.True.) then - do n = 1, nn_num(i) - call edge_center(i, nn_pos(n,i), this_x_coord, this_y_coord, mesh) - call r2g(coord_e_edge_center(1,i,n), coord_e_edge_center(2,i,n), this_x_coord, this_y_coord) - end do - ! else we skip n=1 and use only the edge centers n=2:nn_num(i) - else - do n = 2, nn_num(i) - call edge_center(i, nn_pos(n,i), this_x_coord, this_y_coord, mesh) - call r2g(coord_e_edge_center(1,i,n-1), coord_e_edge_center(2,i,n-1), this_x_coord, this_y_coord) - end do - end if - end do - + ALLOCATE(my_x_coords(my_number_of_points)) ALLOCATE(my_y_coords(my_number_of_points)) + ALLOCATE(my_x_corners(myDim_nod2D, 25)) + ALLOCATE(my_y_corners(myDim_nod2D, 25)) - ! Obtain center coordinates as node center on open ocean and as mean of corners at coastline - do i = 1, my_number_of_points - ! Center coord as mean of corner coordiantes along coastline - if (coastal_nodes(i)==.True.) then - ! So we define temp_corner coordiantes - allocate(temp_x_coord(nod_in_elem2D_num(i)+nn_num(i))) - allocate(temp_y_coord(nod_in_elem2D_num(i)+nn_num(i))) - temp_x_coord=0 - temp_y_coord=0 - do j = 1, nod_in_elem2D_num(i) - temp_x_coord(j) = x_corners(i,j)*rad - temp_y_coord(j) = y_corners(i,j)*rad - end do - ! Loop over edges - do j = 1, nn_num(i) - ! We skip coastal edge center points for the new center point calculation - ! such that 1 element islands have the node center at the right angle - ! We only do so if n elements is > 2, to avoid having only 3 corners - if ((j>1) .and. (nod_in_elem2D_num(i) > 2)) then - edge = coastal_edge_list(i,nn_pos(j,i)) - ! if edge is coastal, we leave it out of the mean equation, replaced by the node center - if (edge>0) then - this_x_coord = coord_nod2D(1, i) - this_y_coord = coord_nod2D(2, i) - ! unrotate grid - call r2g(my_x_coords(i), my_y_coords(i), this_x_coord, this_y_coord) - temp_x_coord(j+nod_in_elem2D_num(i))=my_x_coords(i) - temp_y_coord(j+nod_in_elem2D_num(i))=my_y_coords(i) - ! case for only two elements, we need the real edge centers to ensure center coord - ! is inside polygon - else - temp_x_coord(j+nod_in_elem2D_num(i)) = coord_e_edge_center(1,i,j) - temp_y_coord(j+nod_in_elem2D_num(i)) = coord_e_edge_center(2,i,j) - end if - ! Open ocean case, we just use the corner coords - else - temp_x_coord(j+nod_in_elem2D_num(i)) = coord_e_edge_center(1,i,j) - temp_y_coord(j+nod_in_elem2D_num(i)) = coord_e_edge_center(2,i,j) - end if - end do - min_x = minval(temp_x_coord) - max_x = maxval(temp_x_coord) - ! if we are at dateline (fesom cell larger than pi) - if (max_x-min_x > pi) then - - ! set up separate data structures for the two hemispheres - n_pos=count(temp_x_coord>=0) - n_neg=count(temp_x_coord<0) - allocate(pos_x(n_pos)) - allocate(pos_y(n_pos)) - allocate(neg_x(n_neg)) - allocate(neg_y(n_neg)) - pos_x = 0 - pos_y = 0 - neg_x = 0 - neg_x = 0 - n=1 - do j = 1, size(temp_x_coord) - ! build separate corner vectors for the hemispheres - if (temp_x_coord(j) >= 0) then - pos_x(n) = temp_x_coord(j) - pos_y(n) = temp_y_coord(j) - n=n+1 - end if - end do - n=1 - do j = 1, size(temp_x_coord) - if (temp_x_coord(j) < 0) then - neg_x(n) = temp_x_coord(j) - neg_y(n) = temp_y_coord(j) - n=n+1 - end if - end do - ! if sum on right side of dateline are further from the dateline we shift the negative sum over to the right - if (-sum(pos_x)+pi*n_pos >= sum(neg_x)+pi*n_neg) then - this_x_coord = (sum(pos_x) + sum(neg_x) + 2*pi*n_neg) / (n_pos + n_neg) - this_y_coord = (sum(pos_y) + sum(neg_y)) / (n_pos + n_neg) - ! else we shift the positive sum over to the left side - else - this_x_coord = (sum(pos_x) - 2*pi*n_pos + sum(neg_x)) / (n_pos + n_neg) - this_y_coord = (sum(pos_y) + sum(neg_y)) / (n_pos + n_neg) - end if - deallocate(pos_x,pos_y,neg_x,neg_y) - ! max_x-min_x > pi -> we are not at dateline, just a normal mean is enough - else - this_x_coord = sum(temp_x_coord)/(size(temp_x_coord)) - this_y_coord = sum(temp_y_coord)/(size(temp_y_coord)) - end if - my_x_coords(i)=this_x_coord - my_y_coords(i)=this_y_coord - deallocate(temp_x_coord, temp_y_coord) - ! coastal_nodes(i)==.True. -> Node center on open ocean, we can use node center - else - this_x_coord = coord_nod2D(1, i) - this_y_coord = coord_nod2D(2, i) - ! unrotate grid - call r2g(my_x_coords(i), my_y_coords(i), this_x_coord, this_y_coord) - end if - end do - - ! Add the different corner types to single array in preparation for angle calculation do i = 1, my_number_of_points - ! First for element center based corners - do j = 1, nod_in_elem2D_num(i) - my_x_corners(i,j) = x_corners(i,j)*rad ! atan2 takes radian and elem corners come in grad - my_y_corners(i,j) = y_corners(i,j)*rad - end do - ! Then we repeat for edge center coordinate - ! The the coast j=1 is the node center - if (coastal_nodes(i)==.True.) then - do j = 1, nn_num(i) - my_x_corners(i,j+nod_in_elem2D_num(i)) = coord_e_edge_center(1,i,j) - my_y_corners(i,j+nod_in_elem2D_num(i)) = coord_e_edge_center(2,i,j) - end do - ! On open ocean we dont use the node center as corner, and thus have one less corner - else - do j = 1, nn_num(i)-1 - my_x_corners(i,j+nod_in_elem2D_num(i)) = coord_e_edge_center(1,i,j) - my_y_corners(i,j+nod_in_elem2D_num(i)) = coord_e_edge_center(2,i,j) - end do - end if - end do - - ! calculate angle between corners and center - do i = 1, my_number_of_points - if (coastal_nodes(i)==.True.) then - n=0 - else - n=1 - end if - do j = 1, nod_in_elem2D_num(i)+nn_num(i)-n - ! If they have different sign we are near the dateline and need to bring the corner onto - ! the same hemisphere as the center (only for angle calc, the coord for oasis remains as before) - ! Default: same sign -> normal atan2 - if (my_x_coords(i) <=0 .and. my_x_corners(i,j) <=0 .or. my_x_coords(i) >0 .and. my_x_corners(i,j) >0) then - angle(i,j) = atan2(my_x_corners(i,j) - my_x_coords(i), my_y_corners(i,j) - my_y_coords(i)) - else - ! at dateline center is on the right side - if (my_x_coords(i) >=pi/2) then - angle(i,j) = atan2(my_x_corners(i,j) + 2*pi - my_x_coords(i), my_y_corners(i,j) - my_y_coords(i)) - ! at dateline center is on the left side - else if (my_x_coords(i) <=-pi/2) then - angle(i,j) = atan2(my_x_corners(i,j) - 2*pi - my_x_coords(i), my_y_corners(i,j) - my_y_coords(i)) - ! at prime meridan -> also default - else - angle(i,j) = atan2(my_x_corners(i,j) - my_x_coords(i), my_y_corners(i,j) - my_y_coords(i)) - end if - end if - end do + this_x_coord = coord_nod2D(1, i) + this_y_coord = coord_nod2D(2, i) + call r2g(my_x_coords(i), my_y_coords(i), this_x_coord, this_y_coord) end do - ! Oasis requires corners sorted counterclockwise, so we sort by angle - do i = 1, my_number_of_points - if (coastal_nodes(i)==.True.) then - n=0 - else - n=1 - end if - do l = 1, nod_in_elem2D_num(i)+nn_num(i)-1-n - do m = l+1, nod_in_elem2D_num(i)+nn_num(i)-n - if (angle(i,l) < angle(i,m)) then - ! Swap angle - temp = angle(i,m) - angle(i,m) = angle(i,l) - angle(i,l) = temp - ! Swap lon - temp = my_x_corners(i,m) - my_x_corners(i,m) = my_x_corners(i,l) - my_x_corners(i,l) = temp - ! Swap lat - temp = my_y_corners(i,m) - my_y_corners(i,m) = my_y_corners(i,l) - my_y_corners(i,l) = temp - end if - end do - end do - end do - - ! We can have a variable number of corner points. - ! Luckly oasis can deal with that by just repeating the last one. - ! Note, we are only allowed to repeat one coordinate and - ! the last one is not an element center, but an edge center - do i = 1, my_number_of_points - do j = 1, all_max_elem+all_max_edge - if (coastal_nodes(i)==.True.) then - if (j < nod_in_elem2D_num(i)+nn_num(i)) then - my_y_corners(i,j)=my_y_corners(i,j) - my_x_corners(i,j)=my_x_corners(i,j) - else - my_y_corners(i,j)=my_y_corners(i,nod_in_elem2D_num(i)+nn_num(i)) - my_x_corners(i,j)=my_x_corners(i,nod_in_elem2D_num(i)+nn_num(i)) - end if - else - if (j < nod_in_elem2D_num(i)+nn_num(i)-1) then - my_y_corners(i,j)=my_y_corners(i,j) - my_x_corners(i,j)=my_x_corners(i,j) - else - my_y_corners(i,j)=my_y_corners(i,nod_in_elem2D_num(i)+nn_num(i)-1) - my_x_corners(i,j)=my_x_corners(i,nod_in_elem2D_num(i)+nn_num(i)-1) - end if - end if - end do - end do - - ! Oasis takes grad angles my_x_coords=my_x_coords/rad my_y_coords=my_y_coords/rad - my_x_corners=my_x_corners/rad - my_y_corners=my_y_corners/rad - + if (mype .eq. 0) then + print *, 'FESOM before corner computation' + endif + call node_contours(my_x_corners, my_y_corners, partit, mesh) + write(*,*) my_x_corners + if (mype .eq. 0) then + print *, 'FESOM after corner computation' + endif + if (mype .eq. localroot) then ALLOCATE(all_x_coords(number_of_all_points, 1)) ALLOCATE(all_y_coords(number_of_all_points, 1)) - ALLOCATE(all_x_corners(number_of_all_points, 1, all_max_elem+all_max_edge)) - ALLOCATE(all_y_corners(number_of_all_points, 1, all_max_elem+all_max_edge)) ALLOCATE(all_area(number_of_all_points, 1)) + ALLOCATE(all_x_corners(number_of_all_points, 1, 25)) + ALLOCATE(all_y_corners(number_of_all_points, 1, 25)) else ALLOCATE(all_x_coords(1, 1)) ALLOCATE(all_y_coords(1, 1)) - ALLOCATE(all_x_corners(1, 1, all_max_elem+all_max_edge)) - ALLOCATE(all_y_corners(1, 1, all_max_elem+all_max_edge)) ALLOCATE(all_area(1, 1)) + ALLOCATE(all_x_corners(1, 1, 1)) + ALLOCATE(all_y_corners(1, 1, 1)) endif + + displs_from_all_pes(1) = 0 + do i = 2, npes + displs_from_all_pes(i) = SUM(counts_from_all_pes(1:(i-1))) + enddo + if (mype .eq. 0) then print *, 'FESOM before 1st GatherV', displs_from_all_pes(npes), counts_from_all_pes(npes), number_of_all_points endif @@ -637,42 +536,33 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) counts_from_all_pes, displs_from_all_pes, MPI_DOUBLE_PRECISION, localroot, MPI_COMM_FESOM, ierror) if (mype .eq. 0) then - print *, 'FESOM before 3rd GatherV', displs_from_all_pes(npes), counts_from_all_pes(npes), number_of_all_points + print *, 'FESOM before 3rd GatherV' endif + CALL MPI_GATHERV(area(1,:), my_number_of_points, MPI_DOUBLE_PRECISION, all_area, & + counts_from_all_pes, displs_from_all_pes, MPI_DOUBLE_PRECISION, localroot, MPI_COMM_FESOM, ierror) - do j = 1, all_max_elem+all_max_edge - CALL MPI_GATHERV(my_x_corners(:,j), my_number_of_points, MPI_DOUBLE_PRECISION, all_x_corners(:,:,j), & + do j = 1, 25 + CALL MPI_GATHERV(my_x_corners(:,j), myDim_nod2D, MPI_DOUBLE_PRECISION, all_x_corners(:,:,j), & counts_from_all_pes, displs_from_all_pes, MPI_DOUBLE_PRECISION, localroot, MPI_COMM_FESOM, ierror) - CALL MPI_GATHERV(my_y_corners(:,j), my_number_of_points, MPI_DOUBLE_PRECISION, all_y_corners(:,:,j), & + CALL MPI_GATHERV(my_y_corners(:,j), myDim_nod2D, MPI_DOUBLE_PRECISION, all_y_corners(:,:,j), & counts_from_all_pes, displs_from_all_pes, MPI_DOUBLE_PRECISION, localroot, MPI_COMM_FESOM, ierror) end do - if (mype .eq. 0) then - print *, 'FESOM before 4th GatherV' - endif - CALL MPI_GATHERV(area(1,:), my_number_of_points, MPI_DOUBLE_PRECISION, all_area, & - counts_from_all_pes, displs_from_all_pes, MPI_DOUBLE_PRECISION, localroot, MPI_COMM_FESOM, ierror) - - if (mype .eq. 0) then - print *, 'FESOM after 4th GatherV' - endif - - CALL MPI_Barrier(MPI_COMM_FESOM, ierror) if (mype .eq. 0) then print *, 'FESOM after Barrier' endif if (mype .eq. localroot) then - print *, 'FESOM before start_grids_writing' + print *, 'FESOM before grid writing to oasis grid files' CALL oasis_start_grids_writing(il_flag) IF (il_flag .NE. 0) THEN - print *, 'FESOM before write grid' + print *, 'FESOM before write grid centers' CALL oasis_write_grid (grid_name, number_of_all_points, 1, all_x_coords(:,:), all_y_coords(:,:)) print *, 'FESOM before write corner' - CALL oasis_write_corner (grid_name, number_of_all_points, 1, all_max_elem+all_max_edge, all_x_corners(:,:,:), all_y_corners(:,:,:)) + CALL oasis_write_corner (grid_name, number_of_all_points, 1, 25, all_x_corners(:,:,:), all_y_corners(:,:,:)) ALLOCATE(unstr_mask(number_of_all_points, 1)) unstr_mask=0 @@ -688,10 +578,10 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) call oasis_terminate_grids_writing() print *, 'FESOM after terminate_grids_writing' endif !localroot + + DEALLOCATE(all_x_coords, all_y_coords, my_x_coords, my_y_coords) - DEALLOCATE(all_x_corners, all_y_corners, my_x_corners, my_y_corners, angle) - DEALLOCATE(coastal_nodes, coord_e_edge_center) !------------------------------------------------------------------ ! 3rd Declare the transient variables !------------------------------------------------------------------ @@ -808,9 +698,6 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) call exchange_roots(source_root, target_root, 1, partit%MPI_COMM_FESOM, MPI_COMM_WORLD) if (commRank) print *, 'FESOM source/target roots: ', source_root, target_root #endif - if (mype .eq. 0) then - print *, 'After enddef' - endif ! WAS VOM FOLGENDEN BRAUCHE ICH NOCH ??? @@ -818,9 +705,6 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) allocate(exfld(myDim_nod2D)) cplsnd=0. o2a_call_count=0 - if (mype .eq. 0) then - print *, 'Before last barrier' - endif CALL MPI_BARRIER(MPI_COMM_FESOM, ierror) if (mype .eq. 0) then diff --git a/src/node_contours.F90 b/src/node_contours.F90 deleted file mode 100644 index ad28e91c4..000000000 --- a/src/node_contours.F90 +++ /dev/null @@ -1,174 +0,0 @@ -subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) - USE MOD_MESH - USE MOD_PARTIT - USE MOD_PARSUP - USE o_PARAM - use g_comm_auto - use o_ARRAYS - use g_rotate_grid - - IMPLICIT NONE - type(t_mesh), intent(inout), target :: mesh - type(t_partit), intent(inout), target :: partit - real(kind=WP), allocatable, intent(inout) :: my_x_corners(:,:) ! longitude node corners - real(kind=WP), allocatable, intent(inout) :: my_y_corners(:,:) ! latitude node corners - integer :: bEdge_left, bEdge_right - integer, dimension(2) :: belem_left, belem_right - integer :: edge_left, edge_right - integer :: n, ee, elem, nn, el(2), flag, nn1, nn2 - integer, allocatable, dimension(:) :: nedges, nelems, nedges1, nelems1, nedges2, nelems2 - real(kind=WP) :: this_x_coord, this_y_coord -#include "associate_part_def.h" -#include "associate_mesh_def.h" -#include "associate_part_ass.h" -#include "associate_mesh_ass.h" - -ALLOCATE(my_x_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) -ALLOCATE(my_y_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) - -do n=1, myDim_nod2D - ! find the type of node: internal or at boundary - bEdge_left =0 - belem_left =0 - bEdge_right=0 - belem_right=0 - - do ee=1, nod_in_elem2D_num(n) - elem=nod_in_elem2D(ee,n) - if (elem2D_nodes(1,elem)==n) then - edge_left=elem_edges(3,elem) - edge_right=elem_edges(2,elem) - elseif (elem2D_nodes(2,elem)==n) then - edge_left=elem_edges(1,elem) - edge_right=elem_edges(3,elem) - else - edge_left=elem_edges(2,elem) - edge_right=elem_edges(1,elem) - end if - if (myList_edge2D(edge_left)>edge2D_in) then - bEdge_left=bEdge_left+1 - belem_left(bEdge_left)=elem - end if - if (myList_edge2D(edge_right)>edge2D_in) then - bEdge_right=bEdge_right+1 - belem_right(bEdge_right)=elem - end if - end do - -! now we have three cases - if (bEdge_left==0) then ! inner contour - elem=nod_in_elem2D(1, n) ! we can start from any - allocate(nedges(nod_in_elem2D_num(n))) - nedges=0 - allocate(nelems(nod_in_elem2D_num(n))) - nelems=0 - !!!!!!! inner_node_contour -#include "node_contour_inner.h" - do nn=1, nod_in_elem2D_num(n) - call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) - end do - do nn=nod_in_elem2D_num(n)+1, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)) - my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)) - end do - deallocate(nedges, nelems) - end if - - if (bEdge_left==1) then ! standard boundary node - elem=belem_left(1) - allocate(nedges(nod_in_elem2D_num(n)+1)) - nedges=0 - allocate(nelems(nod_in_elem2D_num(n))) - nelems=0 - !!!!!!!boundary_node_contour -#include "node_contour_boundary.h" - do nn=1, nod_in_elem2D_num(n) - call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) - end do - nn=nod_in_elem2D_num(n)+1 - call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - nn=nod_in_elem2D_num(n)+2 - my_x_corners(n, nn)=coord_nod2D(1,n) - my_y_corners(n, nn)=coord_nod2D(2,n) - do nn=nod_in_elem2D_num(n)+3, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)+2) - my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)+2) - end do - !!!!!!! - deallocate(nedges, nelems) - end if - - if (bEdge_left==2) then ! strange boundary node - elem=belem_left(1) - allocate(nedges (nod_in_elem2D_num(n)+1)) - allocate(nedges1(nod_in_elem2D_num(n)+1)) - nedges =0 - nedges1=0 - allocate(nelems (nod_in_elem2D_num(n))) - allocate(nelems1(nod_in_elem2D_num(n))) - nelems=0 - nelems1=0 - !!!!!!!boundary_node_contour -#include "node_contour_boundary.h" - where (nedges>0) - nedges1=nedges - end where - where (nelems>0) - nelems1=nelems - end where - nn1=nn - do nn=1, nn1 - call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems1(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) - end do - nn=nn1+1 - call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - nn=nn1+2 - my_x_corners(n, nn)=coord_nod2D(1,n) - my_y_corners(n, nn)=coord_nod2D(2,n) - !!!!!!! - elem=belem_left(2) - allocate(nedges2(nod_in_elem2D_num(n)+1)) - nedges =0 - nedges2=0 - allocate(nelems2(nod_in_elem2D_num(n))) - nelems =0 - nelems2=0 - !!!!!!!boundary_node_contour -#include "node_contour_boundary.h" - where (nedges>0) - nedges2=nedges - end where - where (nelems>0) - nelems2=nelems - end where - nn2=nn - do nn=nn1+3, nn1+nn2+2 - call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems2(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) - end do - nn=nn1+nn2+3 - call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - nn=nn1+nn2+4 - my_x_corners(n, nn)=coord_nod2D(1,n) - my_y_corners(n, nn)=coord_nod2D(2,n) - do nn=nn1+nn2+5, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nn1+nn2+4) - my_y_corners(n, nn)=my_y_corners(n, nn1+nn2+4) - end do - !!!!!!! - deallocate(nedges, nelems, nedges1, nelems1, nedges2, nelems2) - end if -end do -do n=1, myDim_nod2D - do nn=1, size(my_x_corners, 2) - this_x_coord=my_x_corners(n, nn) - this_y_coord=my_y_corners(n, nn) - call r2g(my_x_corners(n, nn), my_y_corners(n, nn), this_x_coord, this_y_coord) - end do -end do -my_x_corners=my_x_corners/rad -my_y_corners=my_y_corners/rad -end subroutine node_contours \ No newline at end of file From a22db37aff847889a7b7dda4b5d1731869b02f9f Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Mon, 18 Dec 2023 20:54:01 +0100 Subject: [PATCH 57/71] some debug output --- src/cpl_driver.F90 | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index 2be6c8278..1c6611bd2 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -110,18 +110,31 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) integer :: edge_left, edge_right integer :: n, ee, elem, nn, el(2), flag, nn1, nn2 integer, allocatable, dimension(:) :: nedges, nelems, nedges1, nelems1, nedges2, nelems2 - real(kind=WP) :: this_x_coord, this_y_coord + real(kind=WP) :: this_x_coord, this_y_coord + + real(kind=WP) :: temp_x_coord, temp_y_coord + logical :: node_of_interest + include "associate_part_def.h" include "associate_mesh_def.h" include "associate_part_ass.h" include "associate_mesh_ass.h" - if (.not. allocated(array)) then + if (.not. allocated(my_x_corners)) then ALLOCATE(my_x_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) + endif + if (.not. allocated(my_y_corners)) then ALLOCATE(my_y_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) endif do n=1, myDim_nod2D - ! find the type of node: internal or at boundary + node_of_interest=.False. + call r2g(temp_x_coord, temp_y_coord, coord_nod2D(1,n), coord_nod2D(2,n)) + if (temp_x_coord < 27 .AND. temp_x_coord > 26 .AND. temp_y_coord < 40 .AND. temp_y_coord > 38) THEN + print *, "found GP1" + print *, temp_x_coord, temp_y_coord + node_of_interest=.True. + end if + ! find the type/of node: internal or at boundary bEdge_left =0 belem_left =0 bEdge_right=0 @@ -497,7 +510,6 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh) print *, 'FESOM before corner computation' endif call node_contours(my_x_corners, my_y_corners, partit, mesh) - write(*,*) my_x_corners if (mype .eq. 0) then print *, 'FESOM after corner computation' endif From bae396d552732a84dec9dae20278f0fe02a0698d Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Mon, 18 Dec 2023 22:13:33 +0100 Subject: [PATCH 58/71] some debug output to hunt missing half of scalar volumes --- src/cpl_driver.F90 | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index 1c6611bd2..da746668e 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -129,7 +129,11 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) do n=1, myDim_nod2D node_of_interest=.False. call r2g(temp_x_coord, temp_y_coord, coord_nod2D(1,n), coord_nod2D(2,n)) - if (temp_x_coord < 27 .AND. temp_x_coord > 26 .AND. temp_y_coord < 40 .AND. temp_y_coord > 38) THEN + temp_x_coord=temp_x_coord/rad + temp_y_coord=temp_y_coord/rad + !node of interest: (26.37363839491671, 39.03160698980019) + !print *, temp_x_coord, temp_y_coord + if (temp_x_coord < 26.4 .AND. temp_x_coord > 26.3 .AND. temp_y_coord < 39.1 .AND. temp_y_coord > 39) THEN print *, "found GP1" print *, temp_x_coord, temp_y_coord node_of_interest=.True. @@ -160,6 +164,13 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) bEdge_right=bEdge_right+1 belem_right(bEdge_right)=elem end if + if (node_of_interest==.True.) then + print *, "ee:", ee + print *, "edge_left:", edge_left + print *, "edge_right:", edge_right + print *, "bEdge_left:", bEdge_left + print *, "bEdge_right:", bEdge_right + end if end do ! now we have three cases @@ -180,8 +191,16 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)) end do deallocate(nedges, nelems) + if (node_of_interest==.True.) then + print *, "case1" + print *, "n", n + print *, "nn", nn + print *, "my_x_corners(n, nn):", my_x_corners(n, nn) + print *, "my_y_corners(n, nn):", my_y_corners(n, nn) + end if end if + if (bEdge_left==1) then ! standard boundary node elem=belem_left(1) allocate(nedges(nod_in_elem2D_num(n)+1)) @@ -205,6 +224,13 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) end do !!!!!!! deallocate(nedges, nelems) + if (node_of_interest==.True.) then + print *, "case2" + print *, "n", n + print *, "nn", nn + print *, "my_x_corners(n, nn):", my_x_corners(n, nn) + print *, "my_y_corners(n, nn):", my_y_corners(n, nn) + end if end if if (bEdge_left==2) then ! strange boundary node @@ -267,6 +293,11 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) end do !!!!!!! deallocate(nedges, nelems, nedges1, nelems1, nedges2, nelems2) + if (node_of_interest==.True.) then + print *, "case3" + print *, "my_x_corners:", my_x_corners + print *, "my_y_corners:", my_y_corners + end if end if end do do n=1, myDim_nod2D @@ -274,6 +305,11 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) this_x_coord=my_x_corners(n, nn) this_y_coord=my_y_corners(n, nn) call r2g(my_x_corners(n, nn), my_y_corners(n, nn), this_x_coord, this_y_coord) + if (node_of_interest==.True.) then + print *, "final" + print *, "my_x_corners(n, nn)/rad:", my_x_corners(n, nn)/rad + print *, "my_y_corners(n, nn)/rad:", my_y_corners(n, nn)/rad + end if end do end do my_x_corners=my_x_corners/rad From 6da79a915d315072508eda5aa73dac0442b902a9 Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Tue, 19 Dec 2023 11:19:28 +0100 Subject: [PATCH 59/71] fixing indexing when filling and reordering corner fields; removing debug code --- src/cpl_driver.F90 | 63 +++++++--------------------------------------- 1 file changed, 9 insertions(+), 54 deletions(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index da746668e..fec645094 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -112,9 +112,6 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) integer, allocatable, dimension(:) :: nedges, nelems, nedges1, nelems1, nedges2, nelems2 real(kind=WP) :: this_x_coord, this_y_coord - real(kind=WP) :: temp_x_coord, temp_y_coord - logical :: node_of_interest - include "associate_part_def.h" include "associate_mesh_def.h" include "associate_part_ass.h" @@ -127,17 +124,6 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) ALLOCATE(my_y_corners(myDim_nod2D, 25)) !maxval(nod_in_elem2D_num, 1)*2+2)) endif do n=1, myDim_nod2D - node_of_interest=.False. - call r2g(temp_x_coord, temp_y_coord, coord_nod2D(1,n), coord_nod2D(2,n)) - temp_x_coord=temp_x_coord/rad - temp_y_coord=temp_y_coord/rad - !node of interest: (26.37363839491671, 39.03160698980019) - !print *, temp_x_coord, temp_y_coord - if (temp_x_coord < 26.4 .AND. temp_x_coord > 26.3 .AND. temp_y_coord < 39.1 .AND. temp_y_coord > 39) THEN - print *, "found GP1" - print *, temp_x_coord, temp_y_coord - node_of_interest=.True. - end if ! find the type/of node: internal or at boundary bEdge_left =0 belem_left =0 @@ -164,13 +150,6 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) bEdge_right=bEdge_right+1 belem_right(bEdge_right)=elem end if - if (node_of_interest==.True.) then - print *, "ee:", ee - print *, "edge_left:", edge_left - print *, "edge_right:", edge_right - print *, "bEdge_left:", bEdge_left - print *, "bEdge_right:", bEdge_right - end if end do ! now we have three cases @@ -186,18 +165,11 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) end do - do nn=nod_in_elem2D_num(n)+1, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)) - my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)) + do nn=nod_in_elem2D_num(n)*2+1, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)*2) + my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)*2) end do deallocate(nedges, nelems) - if (node_of_interest==.True.) then - print *, "case1" - print *, "n", n - print *, "nn", nn - print *, "my_x_corners(n, nn):", my_x_corners(n, nn) - print *, "my_y_corners(n, nn):", my_y_corners(n, nn) - end if end if @@ -215,22 +187,15 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) end do nn=nod_in_elem2D_num(n)+1 call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - nn=nod_in_elem2D_num(n)+2 - my_x_corners(n, nn)=coord_nod2D(1,n) - my_y_corners(n, nn)=coord_nod2D(2,n) - do nn=nod_in_elem2D_num(n)+3, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)+2) - my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)+2) +! nn=nod_in_elem2D_num(n)+2 + my_x_corners(n, (nn-1)*2+2)=coord_nod2D(1,n) + my_y_corners(n, (nn-1)*2+2)=coord_nod2D(2,n) + do nn=nod_in_elem2D_num(n)*2+3, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)*2+2) + my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)*2+2) end do !!!!!!! deallocate(nedges, nelems) - if (node_of_interest==.True.) then - print *, "case2" - print *, "n", n - print *, "nn", nn - print *, "my_x_corners(n, nn):", my_x_corners(n, nn) - print *, "my_y_corners(n, nn):", my_y_corners(n, nn) - end if end if if (bEdge_left==2) then ! strange boundary node @@ -293,11 +258,6 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) end do !!!!!!! deallocate(nedges, nelems, nedges1, nelems1, nedges2, nelems2) - if (node_of_interest==.True.) then - print *, "case3" - print *, "my_x_corners:", my_x_corners - print *, "my_y_corners:", my_y_corners - end if end if end do do n=1, myDim_nod2D @@ -305,11 +265,6 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) this_x_coord=my_x_corners(n, nn) this_y_coord=my_y_corners(n, nn) call r2g(my_x_corners(n, nn), my_y_corners(n, nn), this_x_coord, this_y_coord) - if (node_of_interest==.True.) then - print *, "final" - print *, "my_x_corners(n, nn)/rad:", my_x_corners(n, nn)/rad - print *, "my_y_corners(n, nn)/rad:", my_y_corners(n, nn)/rad - end if end do end do my_x_corners=my_x_corners/rad From 4a16537400b9b84e1e23df8a4272d82eaef0c170 Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 10:20:15 +0100 Subject: [PATCH 60/71] started to add support for clockwise / counter clockwise option in scalar geometry stuff --- src/cpl_driver.F90 | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index fec645094..d9fbb50f3 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -109,6 +109,8 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) integer, dimension(2) :: belem_left, belem_right integer :: edge_left, edge_right integer :: n, ee, elem, nn, el(2), flag, nn1, nn2 + integer :: current_pos + integer :: pos_increment=-1 ! counter clockwise is negative, otherwise +1! integer, allocatable, dimension(:) :: nedges, nelems, nedges1, nelems1, nedges2, nelems2 real(kind=WP) :: this_x_coord, this_y_coord @@ -161,13 +163,20 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) nelems=0 !!!!!!! inner_node_contour include "node_contour_inner.h" + if (pos_increment<0) then + current_pos=nod_in_elem2D_num(n)+nod_in_elem2D_num(n) + else + current_pos =1 + end if do nn=1, nod_in_elem2D_num(n) - call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment + call elem_center(nelems(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment end do - do nn=nod_in_elem2D_num(n)*2+1, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)*2) - my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)*2) + do nn=current_pos, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, current_pos-1) + my_y_corners(n, nn)=my_y_corners(n, current_pos-1) end do deallocate(nedges, nelems) end if From 8325e807794b1c2a4c7be3bafcc3ec00f6a3693b Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 10:38:41 +0100 Subject: [PATCH 61/71] bug fix in contour computation --- src/cpl_driver.F90 | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index d9fbb50f3..8a36e01ac 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -164,7 +164,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) !!!!!!! inner_node_contour include "node_contour_inner.h" if (pos_increment<0) then - current_pos=nod_in_elem2D_num(n)+nod_in_elem2D_num(n) + current_pos=2*nod_in_elem2D_num(n) else current_pos =1 end if @@ -174,6 +174,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) call elem_center(nelems(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) current_pos=current_pos+pos_increment end do + current_pos=2*nod_in_elem2D_num(n)+1 do nn=current_pos, size(my_x_corners, 2) my_x_corners(n, nn)=my_x_corners(n, current_pos-1) my_y_corners(n, nn)=my_y_corners(n, current_pos-1) @@ -190,18 +191,26 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) nelems=0 !!!!!!!boundary_node_contour include "node_contour_boundary.h" + if (pos_increment<0) then + current_pos=2*nod_in_elem2D_num(n)+1 + else + current_pos =1 + end if do nn=1, nod_in_elem2D_num(n) - call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment + call elem_center(nelems(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos, mesh) + current_pos=current_pos+pos_increment end do nn=nod_in_elem2D_num(n)+1 - call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) -! nn=nod_in_elem2D_num(n)+2 - my_x_corners(n, (nn-1)*2+2)=coord_nod2D(1,n) - my_y_corners(n, (nn-1)*2+2)=coord_nod2D(2,n) - do nn=nod_in_elem2D_num(n)*2+3, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nod_in_elem2D_num(n)*2+2) - my_y_corners(n, nn)=my_y_corners(n, nod_in_elem2D_num(n)*2+2) + call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment + my_x_corners(n, current_pos)=coord_nod2D(1,n) + my_y_corners(n, current_pos)=coord_nod2D(2,n) + current_pos=2*nod_in_elem2D_num(n)+1 + do nn=current_pos, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, current_pos-1) + my_y_corners(n, nn)=my_y_corners(n, current_pos-1) end do !!!!!!! deallocate(nedges, nelems) From efbae95c6d15c1b7d0a4077487815b5d08edf995 Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 10:41:40 +0100 Subject: [PATCH 62/71] a typo error fixed in scalar_geomenry_description --- src/cpl_driver.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index 8a36e01ac..c78f24a98 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -199,7 +199,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) do nn=1, nod_in_elem2D_num(n) call edge_center(edges(1, nedges(nn)), edges(2, nedges(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) current_pos=current_pos+pos_increment - call elem_center(nelems(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos, mesh) + call elem_center(nelems(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) current_pos=current_pos+pos_increment end do nn=nod_in_elem2D_num(n)+1 From 8f7bbf153284eb4db6c035ab909e0a69d4e1e21a Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 10:47:27 +0100 Subject: [PATCH 63/71] one more bug fix in scalar geometry --- src/cpl_driver.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index c78f24a98..a48117dbb 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -207,7 +207,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) current_pos=current_pos+pos_increment my_x_corners(n, current_pos)=coord_nod2D(1,n) my_y_corners(n, current_pos)=coord_nod2D(2,n) - current_pos=2*nod_in_elem2D_num(n)+1 + current_pos=2*nod_in_elem2D_num(n)+2 do nn=current_pos, size(my_x_corners, 2) my_x_corners(n, nn)=my_x_corners(n, current_pos-1) my_y_corners(n, nn)=my_y_corners(n, current_pos-1) From 4265af0b3821cf023b6f3a435bd096529b1ac9ed Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 10:58:38 +0100 Subject: [PATCH 64/71] again bug in geometry desctiption --- src/cpl_driver.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index a48117dbb..bc14ee8a6 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -192,7 +192,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) !!!!!!!boundary_node_contour include "node_contour_boundary.h" if (pos_increment<0) then - current_pos=2*nod_in_elem2D_num(n)+1 + current_pos=2*nod_in_elem2D_num(n)+2 !one more for the node n itself else current_pos =1 end if From 37be3919e1c5af709874567888e8e14aa47a4879 Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 11:09:36 +0100 Subject: [PATCH 65/71] extended grid description geometry to the case with 4 boundary edges --- src/cpl_driver.F90 | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index bc14ee8a6..ac6cb775f 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -192,7 +192,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) !!!!!!!boundary_node_contour include "node_contour_boundary.h" if (pos_increment<0) then - current_pos=2*nod_in_elem2D_num(n)+2 !one more for the node n itself + current_pos=2*nod_in_elem2D_num(n)+2 !one more for the node n itself also we have 2 boundary edges else current_pos =1 end if @@ -226,6 +226,11 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) allocate(nelems1(nod_in_elem2D_num(n))) nelems=0 nelems1=0 + if (pos_increment<0) then + current_pos=2*nod_in_elem2D_num(n)+4 !two more for the node n itself also we have 4 boundary edges + else + current_pos =1 + end if !!!!!!!boundary_node_contour include "node_contour_boundary.h" where (nedges>0) @@ -236,14 +241,18 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) end where nn1=nn do nn=1, nn1 - call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems1(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment + call elem_center(nelems1(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment end do nn=nn1+1 - call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call edge_center(edges(1, nedges1(nn)), edges(2, nedges1(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment nn=nn1+2 - my_x_corners(n, nn)=coord_nod2D(1,n) - my_y_corners(n, nn)=coord_nod2D(2,n) + my_x_corners(n, current_pos)=coord_nod2D(1,n) + my_y_corners(n, current_pos)=coord_nod2D(2,n) + current_pos=current_pos+pos_increment !!!!!!! elem=belem_left(2) allocate(nedges2(nod_in_elem2D_num(n)+1)) @@ -262,17 +271,20 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) end where nn2=nn do nn=nn1+3, nn1+nn2+2 - call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) - call elem_center(nelems2(nn), my_x_corners(n, (nn-1)*2+2), my_y_corners(n, (nn-1)*2+2), mesh) + call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment + call elem_center(nelems2(nn), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment end do nn=nn1+nn2+3 - call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, (nn-1)*2+1), my_y_corners(n, (nn-1)*2+1), mesh) + call edge_center(edges(1, nedges2(nn)), edges(2, nedges2(nn)), my_x_corners(n, current_pos), my_y_corners(n, current_pos), mesh) + current_pos=current_pos+pos_increment nn=nn1+nn2+4 my_x_corners(n, nn)=coord_nod2D(1,n) my_y_corners(n, nn)=coord_nod2D(2,n) - do nn=nn1+nn2+5, size(my_x_corners, 2) - my_x_corners(n, nn)=my_x_corners(n, nn1+nn2+4) - my_y_corners(n, nn)=my_y_corners(n, nn1+nn2+4) + do nn=current_pos, size(my_x_corners, 2) + my_x_corners(n, nn)=my_x_corners(n, current_pos-1) + my_y_corners(n, nn)=my_y_corners(n, current_pos-1) end do !!!!!!! deallocate(nedges, nelems, nedges1, nelems1, nedges2, nelems2) From 53c1d1b8df67fcf2e8fb20b86ba0530c78e78f56 Mon Sep 17 00:00:00 2001 From: sidorenko1 Date: Wed, 20 Dec 2023 11:17:43 +0100 Subject: [PATCH 66/71] fix in scalar geometry computation --- src/cpl_driver.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 index ac6cb775f..91f2225d9 100644 --- a/src/cpl_driver.F90 +++ b/src/cpl_driver.F90 @@ -207,7 +207,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) current_pos=current_pos+pos_increment my_x_corners(n, current_pos)=coord_nod2D(1,n) my_y_corners(n, current_pos)=coord_nod2D(2,n) - current_pos=2*nod_in_elem2D_num(n)+2 + current_pos=2*nod_in_elem2D_num(n)+3 do nn=current_pos, size(my_x_corners, 2) my_x_corners(n, nn)=my_x_corners(n, current_pos-1) my_y_corners(n, nn)=my_y_corners(n, current_pos-1) @@ -282,6 +282,7 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh) nn=nn1+nn2+4 my_x_corners(n, nn)=coord_nod2D(1,n) my_y_corners(n, nn)=coord_nod2D(2,n) + current_pos=2*nod_in_elem2D_num(n)+5 do nn=current_pos, size(my_x_corners, 2) my_x_corners(n, nn)=my_x_corners(n, current_pos-1) my_y_corners(n, nn)=my_y_corners(n, current_pos-1) From 52423a74407165df993e1c4be747af8388767a26 Mon Sep 17 00:00:00 2001 From: Pengyang Song Date: Thu, 18 Jan 2024 13:38:57 +0100 Subject: [PATCH 67/71] Missing values should be -999; 1 is also a valid value. --- src/fvom_init.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fvom_init.F90 b/src/fvom_init.F90 index eac201990..30ee8f5f6 100755 --- a/src/fvom_init.F90 +++ b/src/fvom_init.F90 @@ -888,7 +888,7 @@ subroutine find_levels(mesh) !___________________________________________________________ ! loop over neighbouring triangles do i=1,nneighb - if (elems(i)>1) then + if (elems(i)>0) then if (nlevels(elems(i))>=nz) then !count neighbours count_neighb_open=count_neighb_open+1 From cddac6022157aed45e7bc24ac1dfb250588bf9f2 Mon Sep 17 00:00:00 2001 From: patrickscholz Date: Fri, 19 Jan 2024 13:46:08 +0100 Subject: [PATCH 68/71] Update namelist.cvmix Make people aware that if they change the idemix surface/bottom forcing path they also might need to change the variable name thats in the file!!! --- config/namelist.cvmix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/namelist.cvmix b/config/namelist.cvmix index 18a90c979..2bf06d887 100644 --- a/config/namelist.cvmix +++ b/config/namelist.cvmix @@ -26,7 +26,9 @@ idemix_mu0 = 0.33333333 ! dissipation parameter (dimensionless) idemix_sforcusage = 0.2 idemix_n_hor_iwe_prop_iter = 5 ! iterations for contribution from horiz. wave propagation idemix_surforc_file = '/work/ollie/clidyn/forcing/IDEMIX/fourier_smooth_2005_cfsr_inert_rgrid.nc' +idemix_surforc_vname= 'var706' idemix_botforc_file = '/work/ollie/clidyn/forcing/IDEMIX/tidal_energy_gx1v6_20090205_rgrid.nc' +idemix_botforc_vname= 'wave_dissipation' / ! namelist for PP From 06ffacf54ac40a87409e01808372921179cafe88 Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Mon, 22 Jan 2024 17:45:15 +0100 Subject: [PATCH 69/71] adding find oasis for oasis compiled with yac. also adding switch to select it instead of regular find oasis --- CMakeLists.txt | 1 + cmake/FindOASISYAC.cmake | 19 +++++++++++++++++++ src/CMakeLists.txt | 13 ++++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 cmake/FindOASISYAC.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ec4985d5..a3a37e1dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs F set(CRAY OFF CACHE BOOL "compile with cray ftn") set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea ice column physics.") set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results") +set(OASIS_WITH_YAC OFF CACHE BOOL "Useing a version of OASIS compiled with YAC instead of SCRIP for interpolation?") #set(VERBOSE OFF CACHE BOOL "toggle debug output") #add_subdirectory(oasis3-mct/lib/psmile) diff --git a/cmake/FindOASISYAC.cmake b/cmake/FindOASISYAC.cmake new file mode 100644 index 000000000..b7927f62c --- /dev/null +++ b/cmake/FindOASISYAC.cmake @@ -0,0 +1,19 @@ +find_path(OASIS_Fortran_INCLUDE_DIRECTORIES mod_oasis.mod HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile) +find_library(OASIS_Fortran_LIBRARIES psmile HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile) + +find_path(MCT_Fortran_INCLUDE_DIRECTORIES mct_mod.mod HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile/mct) +find_library(MCT_Fortran_LIBRARIES mct HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile/mct) + +find_path(MPEU_Fortran_INCLUDE_DIRECTORIES m_mpout.mod HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile/mct) +find_library(MPEU_Fortran_LIBRARIES mpeu HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile/mct) + +find_path(SCRIP_Fortran_INCLUDE_DIRECTORIES remap_bicubic_reduced.mod HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile/scrip) +find_library(SCRIP_Fortran_LIBRARIES scrip HINTS ${TOPLEVEL_DIR}/../oasis/build/lib/psmile/scrip) + +find_path(YAC_Fortran_INCLUDE_DIRECTORIES mo_yac_utils.mod HINTS ${TOPLEVEL_DIR}/../oasis/lib/yac/include) +find_library(YACUTILS_Fortran_LIBRARIES yac_utils HINTS ${TOPLEVEL_DIR}/../oasis/lib/yac/lib) +find_library(YACCORE_Fortran_LIBRARIES yac_core HINTS ${TOPLEVEL_DIR}/../oasis/lib/yac/lib) + +find_path(YAXT_Fortran_INCLUDE_DIRECTORIES yaxt.mod HINTS ${TOPLEVEL_DIR}/../oasis/lib/yaxt/include) +find_library(YAXT_Fortran_LIBRARIES yaxt HINTS ${TOPLEVEL_DIR}/../oasis/lib/yaxt/lib) +find_library(YAXTC_Fortran_LIBRARIES yaxt_c HINTS ${TOPLEVEL_DIR}/../oasis/lib/yaxt/lib) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fb02eab70..66690c753 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -123,7 +123,14 @@ if(${DISABLE_MULTITHREADING}) target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_MULTITHREADING) endif() if(${FESOM_COUPLED}) - include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindOASIS.cmake) + message(STATUS "FESOM is coupled") + if(${OASIS_WITH_YAC}) + message(STATUS "Looking for OASIS with YAC lib") + include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindOASISYAC.cmake) + else() + message(STATUS "Looking for OASIS") + include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindOASIS.cmake) + endif() target_compile_definitions(${PROJECT_NAME} PRIVATE __oasis) endif() if(${OIFS_COUPLED}) @@ -192,8 +199,11 @@ endif() target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES}) target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTORIES} ${MPEU_Fortran_INCLUDE_DIRECTORIES}) target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) +target_include_directories(${PROJECT_NAME} PRIVATE ${YAC_Fortran_INCLUDE_DIRECTORIES} ${YAXT_Fortran_INCLUDE_DIRECTORIES}) target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${YACUTILS_Fortran_LIBRARIES} ${YACCORE_Fortran_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${YAXT_Fortran_LIBRARIES} ${YAXTC_Fortran_LIBRARIES}) target_link_libraries(${PROJECT_NAME} async_threads_cpp) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) @@ -202,6 +212,7 @@ if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) endif() + set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") if(${BUILD_FESOM_AS_LIBRARY}) install(TARGETS ${PROJECT_NAME} DESTINATION "${FESOM_INSTALL_PREFIX}/lib") From c8073a0fab3ef5b31717191add0cca44c0948448 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 24 Jan 2024 23:14:04 +0100 Subject: [PATCH 70/71] Fix typo (inizialization/initialization) in gen_surface_forcing.F90 --- src/gen_surface_forcing.F90 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gen_surface_forcing.F90 b/src/gen_surface_forcing.F90 index 2ffb26c43..529ce37d1 100644 --- a/src/gen_surface_forcing.F90 +++ b/src/gen_surface_forcing.F90 @@ -30,7 +30,7 @@ MODULE g_sbf !! we assume that all NetCDF files have identical grid and time variable !! !! public: - !! sbc_ini -- inizialization atmpospheric forcing + !! sbc_ini -- initialization atmpospheric forcing !! sbc_do -- provide a sbc (surface boundary conditions) each time step !! USE MOD_MESH @@ -904,7 +904,7 @@ SUBROUTINE sbc_ini(partit, mesh) !!--------------------------------------------------------------------- !! *** ROUTINE sbc_ini *** !! - !! ** Purpose : inizialization of ocean forcing + !! ** Purpose : initialization of ocean forcing !! ** Method : !! ** Action : !!---------------------------------------------------------------------- @@ -945,13 +945,13 @@ SUBROUTINE sbc_ini(partit, mesh) READ( nm_sbc_unit, nml=nam_sbc, iostat=iost ) close( nm_sbc_unit ) - if (mype==0) write(*,*) "Start: Ocean forcing inizialization." + if (mype==0) write(*,*) "Start: Ocean forcing initialization." rdate = real(julday(yearnew,1,1)) rdate = rdate+real(daynew-1,WP)+timenew/86400._WP idate = int(rdate) if (mype==0) then - write(*,*) "Start: Ocean forcing inizialization." + write(*,*) "Start: Ocean forcing initialization." write(*,*) "Surface boundary conditions parameters:" end if @@ -1098,7 +1098,7 @@ SUBROUTINE sbc_ini(partit, mesh) end if end if - if (mype==0) write(*,*) "DONE: Ocean forcing inizialization." + if (mype==0) write(*,*) "DONE: Ocean forcing initialization." if (mype==0) write(*,*) 'Parts of forcing data (only constant in time fields) are read' ! call read_runoff_mapper("/p/project/chhb19/streffing1/input/runoff-mapper/runoff_maps_D3.nc", "arrival_point_id", 1.0_WP, partit, mesh) END SUBROUTINE sbc_ini From 0f64ece9cbc4bcdf8706059f0d3a9005586f3feb Mon Sep 17 00:00:00 2001 From: Pengyang Song Date: Mon, 29 Jan 2024 21:04:20 +0100 Subject: [PATCH 71/71] In cavity the model applies linfs, eta should not be always zero --- src/oce_ale.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 973f81161..ffc16afd9 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1938,7 +1938,6 @@ subroutine compute_hbar_ale(dynamics, partit, mesh) !$OMP PARALLEL DO do n=1,myDim_nod2D - if (ulevels_nod2D(n) > 1) cycle ! --> if cavity node hbar == hbar_old hbar(n)=hbar_old(n)+ssh_rhs_old(n)*dt/areasvol(ulevels_nod2D(n),n) end do !$OMP END PARALLEL DO @@ -3165,7 +3164,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) ! rigid lid. !$OMP PARALLEL DO do node=1, myDim_nod2D+eDim_nod2D - if (ulevels_nod2D(node)==1) eta_n(node)=alpha*hbar(node)+(1.0_WP-alpha)*hbar_old(node) + eta_n(node)=alpha*hbar(node)+(1.0_WP-alpha)*hbar_old(node) end do !$OMP END PARALLEL DO ! --> eta_(n)